From cd6fa0708e2409a3e772e46b81df0e0a5bd564fa Mon Sep 17 00:00:00 2001 From: Viktor Adam Date: Mon, 11 Jun 2018 00:18:00 +0100 Subject: [PATCH] Support loading templates from network (HTTP) --- examples/modernized/stack-templated.yml | 519 ++++++++++++++++++++++++ pkg/template/consts.go | 10 + pkg/template/defaults.go | 2 +- pkg/template/hook.go | 14 +- pkg/template/session.go | 12 +- pkg/template/template.go | 20 +- pkg/template/template_test.go | 128 ++++++ pkg/template/types.go | 1 + 8 files changed, 695 insertions(+), 11 deletions(-) create mode 100644 examples/modernized/stack-templated.yml create mode 100644 pkg/template/consts.go create mode 100644 pkg/template/template_test.go diff --git a/examples/modernized/stack-templated.yml b/examples/modernized/stack-templated.yml new file mode 100644 index 0000000..0c4ff24 --- /dev/null +++ b/examples/modernized/stack-templated.yml @@ -0,0 +1,519 @@ +version: '3.5' + +# extension field for common Podlike settings and templates +x-podlike-templates: + + # the pod template enables logging from the controller + - &with-logging-enabled + inline: | + pod: + image: rycus86/podlike + command: -logs + volumes: + - /var/run/docker.sock:/var/run/docker.sock + + # expose the Traefik dashboard for each task + - &with-traefik-debug-port + inline: | + pod: + ports: + - :8080 + + # expose the Consul UI for each task + - &with-consul-debug-port + inline: | + pod: + ports: + - :8500 + + # use the transformers to add the appropriate log volume to the main component + - &app-template + inline: | + app: + volumes: + - {{ .Service.Name }}_logs:/var/log/apps + environment: + - LOG_PATH=/var/log/apps/{{ .Service.Name }}-app.log + + + # the template that add the Traefik reverse proxy component + - &proxy-component + inline: | + proxy: + image: traefik + command: > + --accesslog --accesslog.filepath=/var/log/apps/{{ .Service.Name }}-proxy.access.log + --traefiklog --traefiklog.filepath=/var/log/apps/{{ .Service.Name }}-proxy.log + --consulcatalog --consulcatalog.watch + --consulcatalog.constraints='tag=={{ .Service.Name }}' + --consulcatalog.endpoint=127.0.0.1:8500 + --consulcatalog.frontendrule='PathPrefix: /{{ "{{.ServiceName}}" }}' + --tracing.jaeger --tracing.servicename={{ .Service.Name }} + --api --api.dashboard --metrics.prometheus + volumes: + - {{ .Service.Name }}_logs:/var/log/apps + + # the template for the OpenTracing compatible Jaeger agent's component + - &tracing-component + inline: | + tracing: + image: jaegertracing/jaeger-agent + environment: + COLLECTOR_HOST_PORT: jaeger-collector:14267 + + # the template for the Consul agent to register in service discovery + - &service-discovery-component + inline: | + sd: + image: consul + command: agent -join=sd-server -enable-script-checks + environment: + CONSUL_BIND_INTERFACE: eth2 + CONSUL_LOCAL_CONFIG: | + { + "services": [ + {{ range $idx, $config := .Args.traefik }} + {{ if gt $idx 0 }} , {{ end }} + { + "name": "{{ .name }}", + "tags": [ + {{ range $ti, $tag := .tags }} + {{ if gt $ti 0 }} , {{ end }} "traefik.{{ $tag }}" + {{ end }} + ], + {{ if .address }} + "address": "{{ .address }}", + {{ end }} + "port": {{ .port }}, + "checks": [ + { + "args": ["sh", "-c", "{{ .check }}"], + "interval": "2s", + "status": "passing" + } + ] + } + {{ end}} + ] + } + + - &logging-component + inline: | + logging: + image: fluent/fluent-bit + command: > + /fluent-bit/bin/fluent-bit + -i tail -p 'path=/var/log/apps/{{ .Service.Name }}-app.log' -t '{{ .Service.Name }}.app' + -i tail -p 'path=/var/log/apps/{{ .Service.Name }}-proxy.access.log' -t '{{ .Service.Name }}.proxy.access' + -i tail -p 'path=/var/log/apps/{{ .Service.Name }}-proxy.log' -t '{{ .Service.Name }}.proxy.out' + -o forward -p 'host=logging-server' -m '*' -v + volumes: + - {{ .Service.Name }}_logs:/var/log/apps + +services: + + aggregator: + image: rycus86/sample-flask-base + command: | + python -c " + import os + import logging + import requests + from flask import Flask, request, redirect + + logging.basicConfig(filename=os.environ.get('LOG_PATH', '/dev/stdout'), level='INFO') + + app = Flask(__name__) + + @app.route('/') + def serve(): + incoming_headers = {k: v for k, v in request.headers} + + data_response = requests.get('http://localhost/data/fetch', headers=incoming_headers) + + if data_response.status_code != 200: + return 'Oops, no data', 500, {} + + render_response = requests.post('http://localhost/renderer/render', + json=data_response.json(), headers=incoming_headers) + + if render_response.status_code != 200: + return 'Oops, failed to render', 500, {} + + return render_response.text + + @app.route('/update', methods=['POST']) + def update(): + incoming_headers = {k: v for k, v in request.headers} + + data_response = requests.post('http://localhost/data/set', + data=request.form, headers=incoming_headers) + + if data_response.status_code != 200: + return 'Oops, update failed', 500, {} + + return redirect(data_response.json().get('returnPath')) + + app.run(host='127.0.0.1', port=5000, threaded=True) + " + x-podlike: + transformer: + - <<: *app-template + templates: + - <<: *proxy-component + - <<: *tracing-component + - <<: *service-discovery-component + - <<: *logging-component + pod: + - <<: *with-logging-enabled + - <<: *with-traefik-debug-port + - <<: *with-consul-debug-port + args: + traefik: + - name: entry + port: 80 + check: pgrep python + tags: + - 'tags=external' + - 'frontend.rule=PathPrefix: /; AddPrefix: /entry' + - name: local-aggregator + port: 5000 + address: 127.0.0.1 + check: pgrep python + tags: + - 'tags=aggregator' + - 'frontend.rule=PathPrefixStrip: /entry' + - name: aggregator-metrics + port: 8080 + check: pgrep traefik + tags: + - 'prom.metrics=enabled' + + data-server: + image: rycus86/sample-flask-base + command: | + python -c " + import os + import json + import base64 + import logging + import requests + from flask import Flask, request, jsonify + + logging.basicConfig(filename=os.environ.get('LOG_PATH', '/dev/stdout'), level='INFO') + + app = Flask(__name__) + + def do_request(url, method='get', data=None, fail_on_error=True): + headers = {k: v for k, v in request.headers + if k.lower() not in ('content-type', 'content-length')} + # make sure the `Content-Type` and `Content-Length` headers are set by requests + + response = requests.request(method, url, data=data, headers=headers) + + if response.status_code != 200: + if fail_on_error: + raise Exception('Request failed: %s' % response) + else: + return None + + return response.text + + @app.route('/fetch') + def render(): + static_config = do_request('http://localhost/static-files/config.json') + static_styles = do_request('http://localhost/static-files/styles.css') + + kv_data = do_request('http://localhost/kv/data?recurse=true', fail_on_error=False) + + result = dict(json.loads(static_config)) + result['data'] = {} + + if kv_data: + for item in json.loads(kv_data): + key, value = item.get('Key'), item.get('Value') + if key and value: + key = key.split('/')[-1] + value = base64.b64decode(value) + + result['data'][key] = value + + result['styles'] = static_styles + + return jsonify(result) + + @app.route('/set', methods=['POST']) + def set_parameter(): + for name, value in request.form.items(): + do_request('http://localhost/kv/data/%s' % name, method='put', data=value) + + static_config = do_request('http://localhost/static-files/config.json') + + return jsonify(returnPath=json.loads(static_config).get('returnPath')) + + app.run(host='127.0.0.1', port=5000, threaded=True) + " + x-podlike: + transformer: + - <<: *app-template + templates: + - <<: *proxy-component + - <<: *tracing-component + - <<: *service-discovery-component + - <<: *logging-component + pod: + - <<: *with-logging-enabled + - <<: *with-traefik-debug-port + - <<: *with-consul-debug-port + args: + traefik: + - name: data + port: 80 + check: pgrep python + tags: + - 'tags=aggregator' + - name: local-data + port: 5000 + address: 127.0.0.1 + check: pgrep python + tags: + - 'tags=data-server' + - 'frontend.rule=PathPrefixStrip: /data' + - name: data-metrics + port: 8080 + check: pgrep traefik + tags: + - 'prom.metrics=enabled' + - name: kv-consul + port: 8500 + address: 127.0.0.1 + check: pgrep python + tags: + - 'tags=data-server' + - 'frontend.rule=PathPrefix: /kv; ReplacePathRegex: ^/kv/(.*) /v1/kv/$$1' + + renderer: + image: rycus86/sample-flask-base + command: | + python -c ' + import os + import logging + from flask import Flask, request, render_template_string + + logging.basicConfig(filename=os.environ.get("LOG_PATH", "/dev/stdout"), level="INFO") + + app = Flask(__name__) + + @app.route("/render", methods=["POST"]) + def render(): + return render_template_string(""" + +

Example form

+ {% for name in names %} +
+
+ {{ name }}: + + +
+
+ {% endfor %} + + """, **request.get_json()) + + app.run(host="127.0.0.1", port=5000, threaded=True) + ' + x-podlike: + transformer: + - <<: *app-template + templates: + - <<: *proxy-component + - <<: *tracing-component + - <<: *service-discovery-component + - <<: *logging-component + pod: + - <<: *with-logging-enabled + - <<: *with-traefik-debug-port + - <<: *with-consul-debug-port + args: + traefik: + - name: renderer + port: 80 + check: pgrep python + tags: + - 'tags=aggregator' + - name: local-renderer + port: 5000 + address: 127.0.0.1 + check: pgrep python + tags: + - 'tags=renderer' + - 'frontend.rule=PathPrefixStrip: /renderer' + - name: renderer-metrics + port: 8080 + check: pgrep traefik + tags: + - 'prom.metrics=enabled' + + # web server for static files + + static-files: + image: nginx + configs: + - source: static-config + target: /tmp/static-config.json + - source: static-styles + target: /tmp/static-styles.css + x-podlike: + templates: + - <<: *service-discovery-component + copy: + - inline: | + app: + /tmp/static-config.json: /usr/share/nginx/html/config.json + /tmp/static-styles.css: /usr/share/nginx/html/styles.css + pod: + - <<: *with-logging-enabled + - <<: *with-consul-debug-port + args: + traefik: + - name: static-files + port: 80 + check: pgrep nginx + tags: + - 'tags=data-server' + - 'frontend.rule=PathPrefixStrip: /static-files' + + # external router + + router: + image: traefik + command: > + --api --api.dashboard --metrics.prometheus + --tracing.jaeger --tracing.servicename=router + --tracing.jaeger.localagenthostport=jaeger-stack-agent:6831 + --consulcatalog --consulcatalog.watch + --consulcatalog.constraints='tag==external' + --consulcatalog.endpoint=sd-server:8500 + --consulcatalog.frontendrule='PathPrefix: /{{.ServiceName}}' + ports: + - 8080:8080 + - 80:80 + + # main service discovery server + + sd-server: + image: consul + environment: + - CONSUL_BIND_INTERFACE=eth2 + ports: + - 8500:8500 + + # EFK stack (used for logs and traces) + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4 + environment: + ES_JAVA_OPTS: '-Xms512m -Xmx512m' + discovery.type: single-node + http.host: 0.0.0.0 + transport.host: 127.0.0.1 + + kibana: + image: docker.elastic.co/kibana/kibana-oss:6.2.4 + ports: + - 5601:5601 + healthcheck: + # hack: use the healthcheck to auto-create the index pattern in Kibana + test: | + (curl -fs "http://127.0.0.1:5601/api/saved_objects/?type=index-pattern&per_page=3" | grep -v '"total":0') || \ + ((curl -fs -XPOST -H "Content-Type: application/json" -H "kbn-xsrf: anything" \ + "http://127.0.0.1:5601/api/saved_objects/index-pattern/fluent-bit" \ + -d"{\"attributes\":{\"title\":\"fluent-bit\",\"timeFieldName\":\"@timestamp\"}}") && \ + (curl -fs -XPOST -H "Content-Type: application/json" -H "kbn-xsrf: anything" \ + "http://127.0.0.1:5601/api/kibana/settings/defaultIndex" \ + -d"{\"value\":\"fluent-bit\"}")) + interval: 3s + timeout: 15s + start_period: 1m + + logging-server: + image: fluent/fluent-bit:0.13.0 + command: > + /fluent-bit/bin/fluent-bit + -i forward -o stdout + -o es -p 'Host=elasticsearch' -p 'Include_Tag_Key=on' -p 'Tag_Key=@log_name' + -m '*' -v + + # Jaeger tracing + + jaeger-stack-agent: + image: jaegertracing/jaeger-agent + environment: + COLLECTOR_HOST_PORT: jaeger-collector:14267 + + jaeger-collector: + image: jaegertracing/jaeger-collector + environment: + SPAN_STORAGE_TYPE: elasticsearch + ES_SERVER_URLS: http://elasticsearch:9200 + + jaeger-query: + image: jaegertracing/jaeger-query + environment: + SPAN_STORAGE_TYPE: elasticsearch + ES_SERVER_URLS: http://elasticsearch:9200 + ports: + - 16686:16686 + + # Prometheus + Grafana for metrics + + prometheus: + image: prom/prometheus:v2.2.1 + configs: + - source: prometheus-config + target: /etc/prometheus/prometheus.yml + + grafana: + image: grafana/grafana:5.1.3 + configs: + - source: grafana-config + target: /etc/grafana/grafana.ini + - source: grafana-datasource + target: /etc/grafana/provisioning/datasources/default.yaml + - source: grafana-dashboard-config + target: /etc/grafana/provisioning/dashboards/default.yaml + - source: grafana-dashboard-example + target: /var/lib/grafana/dashboards/example.json + ports: + - 3000:3000 + +volumes: + aggregator_logs: + name: 'aggregator_logs_{{.Task.ID}}' + labels: + com.github.rycus86.podlike.volume-ref: aggregator_logs + data_server_logs: + name: 'data_server_logs_{{.Task.ID}}' + labels: + com.github.rycus86.podlike.volume-ref: data_server_logs + renderer_logs: + name: 'renderer_logs_{{.Task.ID}}' + labels: + com.github.rycus86.podlike.volume-ref: renderer_logs + +configs: + static-config: + file: ./static-content/config.json + static-styles: + file: ./static-content/styles.css + prometheus-config: + file: ./prometheus/config.yml + grafana-config: + file: ./grafana/config.ini + grafana-datasource: + file: ./grafana/datasource.yaml + grafana-dashboard-config: + file: ./grafana/dashboard.yaml + grafana-dashboard-example: + file: ./grafana/dashboards/example.json diff --git a/pkg/template/consts.go b/pkg/template/consts.go new file mode 100644 index 0000000..d2ee5bc --- /dev/null +++ b/pkg/template/consts.go @@ -0,0 +1,10 @@ +package template + +const ( + XPodlikeExtension = "x-podlike" + ArgsProperty = "args" + ServicesProperty = "services" + + TypeInline = "inline" + TypeHttp = "http" +) diff --git a/pkg/template/defaults.go b/pkg/template/defaults.go index f4af4ef..70fefd7 100644 --- a/pkg/template/defaults.go +++ b/pkg/template/defaults.go @@ -21,7 +21,7 @@ pod: func getTagForPod() string { v := version.Parse() - if v.Tag == version.DEFAULT_VERSION { + if v.Tag == version.DEFAULT_VERSION || v.Tag == "master" { return "latest" } else { return v.Tag diff --git a/pkg/template/hook.go b/pkg/template/hook.go index e5d4275..e9bb7d7 100644 --- a/pkg/template/hook.go +++ b/pkg/template/hook.go @@ -31,7 +31,7 @@ func podTemplateHookFunc() mapstructure.DecodeHookFunc { item := reflect.ValueOf(data).Interface() if m, ok := item.(map[string]interface{}); ok { - if inline, ok := m["inline"]; ok { + if inline, ok := m[TypeInline]; ok { if t == reflect.TypeOf(podTemplate{}) { return podTemplate{Template: inline.(string), Inline: true}, nil @@ -41,6 +41,18 @@ func podTemplateHookFunc() mapstructure.DecodeHookFunc { {Template: inline.(string), Inline: true}, }, nil } + + } else if httpSource, ok := m[TypeHttp]; ok { + + if t == reflect.TypeOf(podTemplate{}) { + return podTemplate{Template: httpSource.(string), Http: true}, nil + + } else if t.Kind() == reflect.Slice && t.Elem() == reflect.TypeOf(podTemplate{}) { + return []podTemplate{ + {Template: httpSource.(string), Http: true}, + }, nil + } + } } diff --git a/pkg/template/session.go b/pkg/template/session.go index 3d96bd4..1c40c8c 100644 --- a/pkg/template/session.go +++ b/pkg/template/session.go @@ -17,17 +17,17 @@ func (ts *transformSession) prepareConfiguration() { } func (ts *transformSession) collectTopLevelConfigurations(configFile types.ConfigFile) { - if configSection, ok := configFile.Config["x-podlike"]; ok { + if configSection, ok := configFile.Config[XPodlikeExtension]; ok { globalConfig, ok := configSection.(map[string]interface{}) if !ok { panic("top level x-podlike config is not a mapping") } // extract the top level global arguments first - if args, ok := globalConfig["args"]; ok { + if args, ok := globalConfig[ArgsProperty]; ok { if mArgs, ok := args.(map[string]interface{}); ok { mergeRecursively(ts.Args, mArgs) - delete(globalConfig, "args") + delete(globalConfig, ArgsProperty) } else { panic("template args is not a mapping") } @@ -56,7 +56,7 @@ func (ts *transformSession) collectTopLevelConfigurations(configFile types.Confi } func (ts *transformSession) collectServiceLevelConfigurations(configFile types.ConfigFile) { - services, ok := configFile.Config["services"] + services, ok := configFile.Config[ServicesProperty] if !ok { return // ok, some YAMLs might only define volumes and such } @@ -72,14 +72,14 @@ func (ts *transformSession) collectServiceLevelConfigurations(configFile types.C panic(fmt.Sprintf("service definition is not a mapping for %s", serviceName)) } - configSection, ok := mDefinition["x-podlike"] + configSection, ok := mDefinition[XPodlikeExtension] if !ok { continue } if v := schema.Version(configFile.Config); versions.LessThan(v, "3.7") { // we have to delete the extension key below schema version 3.7 - delete(mDefinition, "x-podlike") + delete(mDefinition, XPodlikeExtension) } var config transformConfiguration diff --git a/pkg/template/template.go b/pkg/template/template.go index 063d5d6..3115686 100644 --- a/pkg/template/template.go +++ b/pkg/template/template.go @@ -3,6 +3,8 @@ package template import ( "bytes" "gopkg.in/yaml.v2" + "io/ioutil" + "net/http" "path" "text/template" ) @@ -38,8 +40,8 @@ func (t *podTemplate) render(tc *transformConfiguration) map[string]interface{} } func (t *podTemplate) prepareTemplate(workingDir string) *template.Template { - name := "inline" - if !t.Inline { + name := TypeInline + if !t.Inline && !t.Http { name = path.Base(t.Template) } @@ -52,11 +54,23 @@ func (t *podTemplate) prepareTemplate(workingDir string) *template.Template { var err error - if t.Inline { + if t.Http { + if resp, err := http.Get(t.Template); err == nil && resp.StatusCode == 200 { + defer resp.Body.Close() + + if content, err := ioutil.ReadAll(resp.Body); err == nil { + tmpl, err = tmpl.Parse(string(content)) + } + } + + } else if t.Inline { tmpl, err = tmpl.Parse(t.Template) + } else { tmpl, err = tmpl.ParseFiles(path.Join(workingDir, t.Template)) + } + if err != nil { panic(err) } diff --git a/pkg/template/template_test.go b/pkg/template/template_test.go new file mode 100644 index 0000000..b0f4ebe --- /dev/null +++ b/pkg/template/template_test.go @@ -0,0 +1,128 @@ +package template + +import ( + "github.com/docker/cli/cli/compose/types" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "testing" +) + +func TestRender_File(t *testing.T) { + tmplFile, err := ioutil.TempFile("", "template") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmplFile.Name()) + + if _, err := tmplFile.WriteString(` +component: + image: sample/{{ .Service.Labels.CompName }}:{{ .Args.TargetTag }} +`); err != nil { + t.Fatal(err) + } + if err := tmplFile.Close(); err != nil { + t.Fatal(err) + } + + tmpl := podTemplate{ + Template: tmplFile.Name(), + } + + rendered := tmpl.render(&transformConfiguration{ + Service: &types.ServiceConfig{ + Labels: types.Labels{ + "CompName": "testing", + }, + }, + Args: map[string]interface{}{ + "TargetTag": "0.1.2", + }, + Session: &transformSession{}, + }) + + if comp, ok := rendered["component"]; !ok { + t.Error("Root key not found") + } else if mComp, ok := comp.(map[string]interface{}); !ok { + t.Error("Invalid root key") + } else if image, ok := mComp["image"]; !ok { + t.Error("Image key not found") + } else if image != "sample/testing:0.1.2" { + t.Error("Invalid image value found") + } +} + +func TestRender_Inline(t *testing.T) { + tmpl := podTemplate{ + Template: ` +component: + image: sample/{{ .Service.Labels.CompName }}:{{ .Args.TargetTag }} +`, + Inline: true, + } + + rendered := tmpl.render(&transformConfiguration{ + Service: &types.ServiceConfig{ + Labels: types.Labels{ + "CompName": "testing", + }, + }, + Args: map[string]interface{}{ + "TargetTag": "0.1.2", + }, + Session: &transformSession{}, + }) + + if comp, ok := rendered["component"]; !ok { + t.Error("Root key not found") + } else if mComp, ok := comp.(map[string]interface{}); !ok { + t.Error("Invalid root key") + } else if image, ok := mComp["image"]; !ok { + t.Error("Image key not found") + } else if image != "sample/testing:0.1.2" { + t.Error("Invalid image value found") + } +} + +func TestRender_Http(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI != "/template/testing" { + t.Fatal("Invalid template request") + } + + w.WriteHeader(200) + w.Write([]byte(` +component: + image: sample/{{ .Service.Labels.CompName }}:{{ .Args.TargetTag }} +`)) + })) + defer server.Close() + + tmpl := podTemplate{ + Template: server.URL + "/template/testing", + Http: true, + } + + rendered := tmpl.render(&transformConfiguration{ + Service: &types.ServiceConfig{ + Labels: types.Labels{ + "CompName": "testing", + }, + }, + Args: map[string]interface{}{ + "TargetTag": "0.1.2", + }, + Session: &transformSession{}, + }) + + if comp, ok := rendered["component"]; !ok { + t.Error("Root key not found") + } else if mComp, ok := comp.(map[string]interface{}); !ok { + t.Error("Invalid root key") + } else if image, ok := mComp["image"]; !ok { + t.Error("Image key not found") + } else if image != "sample/testing:0.1.2" { + t.Error("Invalid image value found") + } +} diff --git a/pkg/template/types.go b/pkg/template/types.go index 6b251ce..c89e33e 100644 --- a/pkg/template/types.go +++ b/pkg/template/types.go @@ -27,6 +27,7 @@ type transformConfiguration struct { type podTemplate struct { Template string Inline bool + Http bool } type templateVars struct {