From 1171d0c55b9ca1d02567c89a18544161bbde9e0c Mon Sep 17 00:00:00 2001 From: Ali Tavakoli Date: Fri, 15 Oct 2021 19:52:00 -0400 Subject: [PATCH] template: Add include function The include function allows templates to be composed of other templates. Fixes #305 --- docs/templates.md | 51 ++++++++++++++ template/funcs.go | 43 ++++++++++-- template/render_test.go | 68 +++++++++++++++++++ template/template.go | 32 ++++++++- .../composition_services_template.nomad | 13 ++++ .../composition_task_template.nomad | 17 +++++ .../test-fixtures/composition_templated.nomad | 26 +++++++ .../recursive_include_template_1.nomad | 1 + .../recursive_include_template_2.nomad | 1 + 9 files changed, 247 insertions(+), 5 deletions(-) create mode 100644 template/test-fixtures/composition_services_template.nomad create mode 100644 template/test-fixtures/composition_task_template.nomad create mode 100644 template/test-fixtures/composition_templated.nomad create mode 100644 template/test-fixtures/recursive_include_template_1.nomad create mode 100644 template/test-fixtures/recursive_include_template_2.nomad diff --git a/docs/templates.md b/docs/templates.md index 6a90c32a5..361d12670 100644 --- a/docs/templates.md +++ b/docs/templates.md @@ -184,6 +184,57 @@ yaml: ``` +#### include + +Reads the entire contents of the specified template file and renders it into the +current template, using the specified data. + +This function can be used in templates that are themselves included from other +templates, but cyclic includes are not supported. + +Example contents of "/etc/myapp/templates/docker-task.nomad": +``` + task "[[.name]]" { + driver = "docker" + config { + image = "[[.image]]" + } + ... + } +``` + +Example main template: +``` +job "myapp" { + group "mygroup" { +[[ include "/etc/myapp/templates/docker-task.nomad" .task ]] + } +} +``` + +Example varables file: +```yaml +task: + name: mytask + image: registry/mytask:v1.1 +``` + +Render: +``` +job "myapp" { + group "maingroup" { + task "mytask" { + driver = "docker" + config { + image = "registry/mytask:v1.1" + } + ... + } + } +} +``` + + #### loop Accepts varying parameters and differs its behavior based on those parameters as detailed below. diff --git a/template/funcs.go b/template/funcs.go index db600e714..9a9e8af83 100644 --- a/template/funcs.go +++ b/template/funcs.go @@ -1,6 +1,7 @@ package template import ( + "bytes" "encoding/json" "errors" "fmt" @@ -22,11 +23,11 @@ import ( // funcMap builds the template functions and passes the consulClient where this // is required. -func funcMap(consulClient *consul.Client) template.FuncMap { +func funcMap(t *tmpl) template.FuncMap { r := template.FuncMap{ - "consulKey": consulKeyFunc(consulClient), - "consulKeyExists": consulKeyExistsFunc(consulClient), - "consulKeyOrDefault": consulKeyOrDefaultFunc(consulClient), + "consulKey": consulKeyFunc(t.consulClient), + "consulKeyExists": consulKeyExistsFunc(t.consulClient), + "consulKeyOrDefault": consulKeyOrDefaultFunc(t.consulClient), "env": envFunc(), "fileContents": fileContents(), "loop": loop, @@ -42,6 +43,9 @@ func funcMap(consulClient *consul.Client) template.FuncMap { "toLower": toLower, "toUpper": toUpper, + //Nested templates. + "include": includeFunc(t), + // Maths. "add": add, "subtract": subtract, @@ -303,6 +307,37 @@ func fileContents() func(string) (string, error) { } } +func includeFunc(t *tmpl) func(string, interface{}) (string, error) { + return func(tmplPath string, data interface{}) (string, error) { + if tmplPath == "" { + return "", fmt.Errorf("include: empty template path") + } + if t.callStackContains(tmplPath) { + stack := strings.Join(append(t.callStack, tmplPath), "\n calls: ") + return "", fmt.Errorf("include: cyclic include detected in template '%s':\n%s", tmplPath, stack) + } + + tmplContents, err := ioutil.ReadFile(tmplPath) + if err != nil { + return "", err + } + innerTmpl, err := t.newTemplate().Parse(string(tmplContents)) + if err != nil { + return "", fmt.Errorf("include: unable to parse template '%s': %w", tmplPath, err) + } + + t.pushCall(tmplPath) + defer t.popCall() + + var out bytes.Buffer + err = innerTmpl.Execute(&out, data) + if err != nil { + return "", fmt.Errorf("include: unable to execute template '%s': %w", tmplPath, err) + } + return out.String(), nil + } +} + func add(b, a interface{}) (interface{}, error) { av := reflect.ValueOf(a) bv := reflect.ValueOf(b) diff --git a/template/render_test.go b/template/render_test.go index 09bfd6853..4e92b105c 100644 --- a/template/render_test.go +++ b/template/render_test.go @@ -2,6 +2,7 @@ package template import ( "os" + "strings" "testing" nomad "github.com/hashicorp/nomad/api" @@ -116,3 +117,70 @@ func TestTemplater_RenderTemplate(t *testing.T) { t.Fatalf("expected %s but got %v", testEnvValue, *job.TaskGroups[0].Name) } } + +func findService(task *nomad.Task, portLabel string) (*nomad.Service, bool) { + for _, service := range task.Services { + if portLabel == service.PortLabel { + return service, true + } + } + return nil, false +} + +// Test templates composed of other templates via the include function. +func TestTemplater_RenderTemplateInclude(t *testing.T) { + compositionTasks := []struct { + Name string + Image string + Memory uint64 + Services map[string]int // name: port + }{ + {"task1", "registry/task1:v1.1", 250, map[string]int{"http": 80, "https": 443}}, + {"task2", "registry/task2:v1.2", 300, map[string]int{"metrics": 8080}}, + } + + fVars := map[string]interface{}{ + "tasks": compositionTasks, + } + + job, err := RenderJob("test-fixtures/composition_templated.nomad", []string{"test-fixtures/test.yaml"}, "", &fVars) + if err != nil { + t.Fatal(err) + } + for i, expectedTask := range compositionTasks { + actualTask := job.TaskGroups[0].Tasks[i] + if actualTask.Name != expectedTask.Name { + t.Fatalf("expected %s but got %v", expectedTask.Name, actualTask.Name) + } + actualTaskImage := actualTask.Config["image"].(string) + if actualTaskImage != expectedTask.Image { + t.Fatalf("expected %s but got %v", expectedTask.Image, actualTaskImage) + } + + actualTaskPorts := actualTask.Config["port_map"].([]map[string]interface{})[0] + for portName, expectedPort := range expectedTask.Services { + if actualPort, ok := actualTaskPorts[portName]; !ok { + t.Fatalf("expected %s in port_map of task %v", portName, expectedTask.Name) + } else if actualPort.(int) != expectedPort { + t.Fatalf("expected port_map[%s]=%v but got %v", portName, expectedPort, actualPort) + } + + actualService, found := findService(actualTask, portName) + if !found { + t.Fatalf("expected %s in services of task %v", portName, expectedTask.Name) + } + expectedServiceName := "global-" + portName + "-check" + if actualService.Name != expectedServiceName { + t.Fatalf("expected service %s but got %v", expectedServiceName, actualService.Name) + } + } + } + + // Test that cyclic includes are detected as an error. + job, err = RenderJob("test-fixtures/recursive_include_template_1.nomad", nil, "", &fVars) + if err == nil { + t.Fatalf("expected error on cyclic includes") + } else if !strings.Contains(err.Error(), "cyclic include detected") { + t.Fatalf("expected error to contain 'cyclic include detected' but got %v", err) + } +} diff --git a/template/template.go b/template/template.go index 6f767f5da..39d644eed 100644 --- a/template/template.go +++ b/template/template.go @@ -13,6 +13,9 @@ type tmpl struct { flagVariables *map[string]interface{} jobTemplateFile string variableFiles []string + + // callStack contains the current stack of template calls. Not threadsafe. + callStack []string } const ( @@ -29,6 +32,33 @@ func (t *tmpl) newTemplate() *template.Template { tmpl := template.New("jobTemplate") tmpl.Delims(leftDelim, rightDelim) tmpl.Option("missingkey=zero") - tmpl.Funcs(funcMap(t.consulClient)) + tmpl.Funcs(funcMap(t)) return tmpl } + +// pushCall pushes a template path to the call stack. +func (t *tmpl) pushCall(tmplPath string) { + t.callStack = append(t.callStack, tmplPath) +} + +// popCall pops & returns the current top template path from the call stack. +// The bool return value is true iff there was an item to return in the stack. +func (t *tmpl) popCall() (string, bool) { + l := len(t.callStack) + if l == 0 { + return "", false + } + var top string + t.callStack, top = t.callStack[:l-1], t.callStack[l-1] + return top, true +} + +// callStackContains returns true iff tmplPath was pushed but not yet popped. +func (t *tmpl) callStackContains(tmplPath string) bool { + for _, call := range t.callStack { + if tmplPath == call { + return true + } + } + return false +} diff --git a/template/test-fixtures/composition_services_template.nomad b/template/test-fixtures/composition_services_template.nomad new file mode 100644 index 000000000..44f11dfff --- /dev/null +++ b/template/test-fixtures/composition_services_template.nomad @@ -0,0 +1,13 @@ +[[range $name, $port := . -]] +service { + name = "global-[[$name]]-check" + tags = ["global"] + port = "[[$name]]" + check { + name = "alive" + type = "tcp" + interval = "10s" + timeout = "2s" + } +} +[[- end]] diff --git a/template/test-fixtures/composition_task_template.nomad b/template/test-fixtures/composition_task_template.nomad new file mode 100644 index 000000000..6297e06f7 --- /dev/null +++ b/template/test-fixtures/composition_task_template.nomad @@ -0,0 +1,17 @@ +task "[[.Name]]" { + driver = "docker" + config { + image = "[[.Image]]" + port_map = { + [[range $name, $port := .Services -]] + [[$name]] = [[$port]][[end]] + } + } + + resources { + cpu = 500 + memory = [[.Memory]] + } + + [[include "test-fixtures/composition_services_template.nomad" .Services]] +} diff --git a/template/test-fixtures/composition_templated.nomad b/template/test-fixtures/composition_templated.nomad new file mode 100644 index 000000000..245371f68 --- /dev/null +++ b/template/test-fixtures/composition_templated.nomad @@ -0,0 +1,26 @@ +job "composedJob" { + datacenters = ["dc1"] + type = "service" + update { + max_parallel = 1 + min_healthy_time = "10s" + healthy_deadline = "1m" + auto_revert = true + } + + group "composedGroup" { + count = 1 + restart { + attempts = 10 + interval = "5m" + delay = "25s" + mode = "delay" + } + ephemeral_disk { + size = 300 + } +[[range $task := .tasks -]] +[[include "test-fixtures/composition_task_template.nomad" $task | indent 2]] +[[end]] + } +} diff --git a/template/test-fixtures/recursive_include_template_1.nomad b/template/test-fixtures/recursive_include_template_1.nomad new file mode 100644 index 000000000..54db767d4 --- /dev/null +++ b/template/test-fixtures/recursive_include_template_1.nomad @@ -0,0 +1 @@ +[[include "test-fixtures/recursive_include_template_2.nomad" .]] diff --git a/template/test-fixtures/recursive_include_template_2.nomad b/template/test-fixtures/recursive_include_template_2.nomad new file mode 100644 index 000000000..df50841e1 --- /dev/null +++ b/template/test-fixtures/recursive_include_template_2.nomad @@ -0,0 +1 @@ +[[include "test-fixtures/recursive_include_template_1.nomad" .]]