template: Add include function

docs/templates.md

@@ -184,6 +184,57 @@ yaml:
 ```
 
 
+#### include
+
+Reads the entire contents of the specified template file and renders it into the
+current template, using the specified data.
+
+This function can be used in templates that are themselves included from other
+templates, but cyclic includes are not supported.
+
+Example contents of "/etc/myapp/templates/docker-task.nomad":
+```
+    task "[[.name]]" {
+      driver = "docker"
+      config {
+        image = "[[.image]]"
+      }
+      ...
+    }
+```
+
+Example main template:
+```
+job "myapp" {
+  group "mygroup" {
+[[ include "/etc/myapp/templates/docker-task.nomad" .task ]]
+  }
+}
+```
+
+Example varables file:
+```yaml
+task:
+  name: mytask
+  image: registry/mytask:v1.1
+```
+
+Render:
+```
+job "myapp" {
+  group "maingroup" {
+    task "mytask" {
+      driver = "docker"
+      config {
+        image = "registry/mytask:v1.1"
+      }
+      ...
+    }
+  }
+}
+```
+
+
 #### loop
 
 Accepts varying parameters and differs its behavior based on those parameters as detailed below.

template/funcs.go

@@ -1,6 +1,7 @@
 package template
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -22,11 +23,11 @@ import (
 
 // funcMap builds the template functions and passes the consulClient where this
 // is required.
-func funcMap(consulClient *consul.Client) template.FuncMap {
+func funcMap(t *tmpl) template.FuncMap {
 	r := template.FuncMap{
-		"consulKey":          consulKeyFunc(consulClient),
-		"consulKeyExists":    consulKeyExistsFunc(consulClient),
-		"consulKeyOrDefault": consulKeyOrDefaultFunc(consulClient),
+		"consulKey":          consulKeyFunc(t.consulClient),
+		"consulKeyExists":    consulKeyExistsFunc(t.consulClient),
+		"consulKeyOrDefault": consulKeyOrDefaultFunc(t.consulClient),
 		"env":                envFunc(),
 		"fileContents":       fileContents(),
 		"loop":               loop,
@@ -42,6 +43,9 @@ func funcMap(consulClient *consul.Client) template.FuncMap {
 		"toLower":            toLower,
 		"toUpper":            toUpper,
 
+		//Nested templates.
+		"include": includeFunc(t),
+
 		// Maths.
 		"add":      add,
 		"subtract": subtract,
@@ -303,6 +307,37 @@ func fileContents() func(string) (string, error) {
 	}
 }
 
+func includeFunc(t *tmpl) func(string, interface{}) (string, error) {
+	return func(tmplPath string, data interface{}) (string, error) {
+		if tmplPath == "" {
+			return "", fmt.Errorf("include: empty template path")
+		}
+		if t.callStackContains(tmplPath) {
+			stack := strings.Join(append(t.callStack, tmplPath), "\n  calls: ")
+			return "", fmt.Errorf("include: cyclic include detected in template '%s':\n%s", tmplPath, stack)
+		}
+
+		tmplContents, err := ioutil.ReadFile(tmplPath)
+		if err != nil {
+			return "", err
+		}
+		innerTmpl, err := t.newTemplate().Parse(string(tmplContents))
+		if err != nil {
+			return "", fmt.Errorf("include: unable to parse template '%s': %w", tmplPath, err)
+		}
+
+		t.pushCall(tmplPath)
+		defer t.popCall()
+
+		var out bytes.Buffer
+		err = innerTmpl.Execute(&out, data)
+		if err != nil {
+			return "", fmt.Errorf("include: unable to execute template '%s': %w", tmplPath, err)
+		}
+		return out.String(), nil
+	}
+}
+
 func add(b, a interface{}) (interface{}, error) {
 	av := reflect.ValueOf(a)
 	bv := reflect.ValueOf(b)

template/render_test.go

@@ -2,6 +2,7 @@ package template
 
 import (
 	"os"
+	"strings"
 	"testing"
 
 	nomad "github.com/hashicorp/nomad/api"
@@ -116,3 +117,70 @@ func TestTemplater_RenderTemplate(t *testing.T) {
 		t.Fatalf("expected %s but got %v", testEnvValue, *job.TaskGroups[0].Name)
 	}
 }
+
+func findService(task *nomad.Task, portLabel string) (*nomad.Service, bool) {
+	for _, service := range task.Services {
+		if portLabel == service.PortLabel {
+			return service, true
+		}
+	}
+	return nil, false
+}
+
+// Test templates composed of other templates via the include function.
+func TestTemplater_RenderTemplateInclude(t *testing.T) {
+	compositionTasks := []struct {
+		Name     string
+		Image    string
+		Memory   uint64
+		Services map[string]int // name: port
+	}{
+		{"task1", "registry/task1:v1.1", 250, map[string]int{"http": 80, "https": 443}},
+		{"task2", "registry/task2:v1.2", 300, map[string]int{"metrics": 8080}},
+	}
+
+	fVars := map[string]interface{}{
+		"tasks": compositionTasks,
+	}
+
+	job, err := RenderJob("test-fixtures/composition_templated.nomad", []string{"test-fixtures/test.yaml"}, "", &fVars)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for i, expectedTask := range compositionTasks {
+		actualTask := job.TaskGroups[0].Tasks[i]
+		if actualTask.Name != expectedTask.Name {
+			t.Fatalf("expected %s but got %v", expectedTask.Name, actualTask.Name)
+		}
+		actualTaskImage := actualTask.Config["image"].(string)
+		if actualTaskImage != expectedTask.Image {
+			t.Fatalf("expected %s but got %v", expectedTask.Image, actualTaskImage)
+		}
+
+		actualTaskPorts := actualTask.Config["port_map"].([]map[string]interface{})[0]
+		for portName, expectedPort := range expectedTask.Services {
+			if actualPort, ok := actualTaskPorts[portName]; !ok {
+				t.Fatalf("expected %s in port_map of task %v", portName, expectedTask.Name)
+			} else if actualPort.(int) != expectedPort {
+				t.Fatalf("expected port_map[%s]=%v but got %v", portName, expectedPort, actualPort)
+			}
+
+			actualService, found := findService(actualTask, portName)
+			if !found {
+				t.Fatalf("expected %s in services of task %v", portName, expectedTask.Name)
+			}
+			expectedServiceName := "global-" + portName + "-check"
+			if actualService.Name != expectedServiceName {
+				t.Fatalf("expected service %s but got %v", expectedServiceName, actualService.Name)
+			}
+		}
+	}
+
+	// Test that cyclic includes are detected as an error.
+	job, err = RenderJob("test-fixtures/recursive_include_template_1.nomad", nil, "", &fVars)
+	if err == nil {
+		t.Fatalf("expected error on cyclic includes")
+	} else if !strings.Contains(err.Error(), "cyclic include detected") {
+		t.Fatalf("expected error to contain 'cyclic include detected' but got %v", err)
+	}
+}

template/template.go

@@ -13,6 +13,9 @@ type tmpl struct {
 	flagVariables   *map[string]interface{}
 	jobTemplateFile string
 	variableFiles   []string
+
+	// callStack contains the current stack of template calls. Not threadsafe.
+	callStack []string
 }
 
 const (
@@ -29,6 +32,33 @@ func (t *tmpl) newTemplate() *template.Template {
 	tmpl := template.New("jobTemplate")
 	tmpl.Delims(leftDelim, rightDelim)
 	tmpl.Option("missingkey=zero")
-	tmpl.Funcs(funcMap(t.consulClient))
+	tmpl.Funcs(funcMap(t))
 	return tmpl
 }
+
+// pushCall pushes a template path to the call stack.
+func (t *tmpl) pushCall(tmplPath string) {
+	t.callStack = append(t.callStack, tmplPath)
+}
+
+// popCall pops & returns the current top template path from the call stack.
+// The bool return value is true iff there was an item to return in the stack.
+func (t *tmpl) popCall() (string, bool) {
+	l := len(t.callStack)
+	if l == 0 {
+		return "", false
+	}
+	var top string
+	t.callStack, top = t.callStack[:l-1], t.callStack[l-1]
+	return top, true
+}
+
+// callStackContains returns true iff tmplPath was pushed but not yet popped.
+func (t *tmpl) callStackContains(tmplPath string) bool {
+	for _, call := range t.callStack {
+		if tmplPath == call {
+			return true
+		}
+	}
+	return false
+}

template/test-fixtures/composition_services_template.nomad

@@ -0,0 +1,13 @@
+[[range $name, $port := . -]]
+service {
+  name = "global-[[$name]]-check"
+  tags = ["global"]
+  port = "[[$name]]"
+  check {
+    name     = "alive"
+    type     = "tcp"
+    interval = "10s"
+    timeout  = "2s"
+  }
+}
+[[- end]]

template/test-fixtures/composition_task_template.nomad

@@ -0,0 +1,17 @@
+task "[[.Name]]" {
+  driver = "docker"
+  config {
+    image = "[[.Image]]"
+    port_map = {
+      [[range $name, $port := .Services -]]
+      [[$name]] = [[$port]][[end]]
+    }
+  }
+
+  resources {
+    cpu    = 500
+    memory = [[.Memory]]
+  }
+
+  [[include "test-fixtures/composition_services_template.nomad" .Services]]
+}

template/test-fixtures/composition_templated.nomad

@@ -0,0 +1,26 @@
+job "composedJob" {
+  datacenters = ["dc1"]
+  type = "service"
+  update {
+    max_parallel     = 1
+    min_healthy_time = "10s"
+    healthy_deadline = "1m"
+    auto_revert      = true
+  }
+
+  group "composedGroup" {
+    count = 1
+    restart {
+      attempts = 10
+      interval = "5m"
+      delay = "25s"
+      mode = "delay"
+    }
+    ephemeral_disk {
+      size = 300
+    }
+[[range $task := .tasks -]]
+[[include "test-fixtures/composition_task_template.nomad" $task | indent 2]]
+[[end]]
+  }
+}

template/test-fixtures/recursive_include_template_1.nomad

@@ -0,0 +1 @@
+[[include "test-fixtures/recursive_include_template_2.nomad" .]]

template/test-fixtures/recursive_include_template_2.nomad

@@ -0,0 +1 @@
+[[include "test-fixtures/recursive_include_template_1.nomad" .]]