diff --git a/atmos.yaml b/atmos.yaml index 34fa1ff58..9ccde11b8 100644 --- a/atmos.yaml +++ b/atmos.yaml @@ -322,3 +322,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/cmd/cmd_utils.go b/cmd/cmd_utils.go index 438b4a24f..8f70fe126 100644 --- a/cmd/cmd_utils.go +++ b/cmd/cmd_utils.go @@ -219,7 +219,7 @@ func executeCustomCommand( // process the component stack config and expose it in {{ .ComponentConfig.xxx.yyy.zzz }} Go template variables if commandConfig.ComponentConfig.Component != "" && commandConfig.ComponentConfig.Stack != "" { // Process Go templates in the command's 'component_config.component' - component, err := u.ProcessTmpl(cliConfig, fmt.Sprintf("component-config-component-%d", i), commandConfig.ComponentConfig.Component, data, false) + component, err := u.ProcessTmpl(fmt.Sprintf("component-config-component-%d", i), commandConfig.ComponentConfig.Component, data, false) if err != nil { u.LogErrorAndExit(err) } @@ -229,7 +229,7 @@ func executeCustomCommand( } // Process Go templates in the command's 'component_config.stack' - stack, err := u.ProcessTmpl(cliConfig, fmt.Sprintf("component-config-stack-%d", i), commandConfig.ComponentConfig.Stack, data, false) + stack, err := u.ProcessTmpl(fmt.Sprintf("component-config-stack-%d", i), commandConfig.ComponentConfig.Stack, data, false) if err != nil { u.LogErrorAndExit(err) } @@ -271,7 +271,7 @@ func executeCustomCommand( value = strings.TrimRight(res, "\r\n") } else { // Process Go templates in the values of the command's ENV vars - value, err = u.ProcessTmpl(cliConfig, fmt.Sprintf("env-var-%d", i), value, data, false) + value, err = u.ProcessTmpl(fmt.Sprintf("env-var-%d", i), value, data, false) if err != nil { u.LogErrorAndExit(err) } @@ -293,7 +293,7 @@ func executeCustomCommand( // Process Go templates in the command's steps. // Steps support Go templates and have access to {{ .ComponentConfig.xxx.yyy.zzz }} Go template variables - commandToRun, err := u.ProcessTmpl(cliConfig, fmt.Sprintf("step-%d", i), step, data, false) + commandToRun, err := u.ProcessTmpl(fmt.Sprintf("step-%d", i), step, data, false) if err != nil { u.LogErrorAndExit(err) } diff --git a/examples/quick-start/Dockerfile b/examples/quick-start/Dockerfile index 791976422..6891f4e63 100644 --- a/examples/quick-start/Dockerfile +++ b/examples/quick-start/Dockerfile @@ -6,10 +6,10 @@ ARG GEODESIC_OS=debian # https://atmos.tools/ # https://github.com/cloudposse/atmos # https://github.com/cloudposse/atmos/releases -ARG ATMOS_VERSION=1.68.0 +ARG ATMOS_VERSION=1.70.0 # Terraform: https://github.com/hashicorp/terraform/releases -ARG TF_VERSION=1.7.5 +ARG TF_VERSION=1.8.0 FROM cloudposse/geodesic:${GEODESIC_VERSION}-${GEODESIC_OS} diff --git a/examples/quick-start/atmos.yaml b/examples/quick-start/atmos.yaml index 7821376fd..631b39eb5 100644 --- a/examples/quick-start/atmos.yaml +++ b/examples/quick-start/atmos.yaml @@ -255,3 +255,5 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml b/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml index 9814c1444..7f64a9e1c 100644 --- a/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml +++ b/examples/quick-start/rootfs/usr/local/etc/atmos/atmos.yaml @@ -254,3 +254,5 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/examples/quick-start/stacks/orgs/acme/_defaults.yaml b/examples/quick-start/stacks/orgs/acme/_defaults.yaml index 5a3547178..4c1e42a5d 100644 --- a/examples/quick-start/stacks/orgs/acme/_defaults.yaml +++ b/examples/quick-start/stacks/orgs/acme/_defaults.yaml @@ -4,15 +4,16 @@ vars: terraform: vars: tags: + # https://atmos.tools/core-concepts/stacks/templating atmos_component: "{{ .atmos_component }}" atmos_stack: "{{ .atmos_stack }}" atmos_manifest: "{{ .atmos_stack_file }}" terraform_workspace: "{{ .workspace }}" - # Examples of using the Gomplate and Sprig functions - # https://docs.gomplate.ca/functions/strings - atmos_component_description: "{{ strings.Title .atmos_component }} component {{ .vars.name | strings.Quote }} provisioned in the stack {{ .atmos_stack | strings.Quote }}" + # Examples of using the Sprig and Gomplate functions # https://masterminds.github.io/sprig/os.html provisioned_by_user: '{{ env "USER" }}' + # https://docs.gomplate.ca/functions/strings + atmos_component_description: "{{ strings.Title .atmos_component }} component {{ .vars.name | strings.Quote }} provisioned in the stack {{ .atmos_stack | strings.Quote }}" # Terraform backend configuration # https://atmos.tools/core-concepts/components/terraform-backends diff --git a/examples/quick-start/stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json b/examples/quick-start/stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json index d7bd762a8..445e9c7f2 100644 --- a/examples/quick-start/stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json +++ b/examples/quick-start/stacks/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json @@ -362,6 +362,9 @@ }, "atlantis": { "$ref": "#/definitions/atlantis" + }, + "templates": { + "$ref": "#/definitions/templates" } }, "required": [], @@ -743,6 +746,12 @@ "description": "Providers section", "additionalProperties": true, "title": "providers" + }, + "templates": { + "type": "object", + "description": "Templates section", + "additionalProperties": true, + "title": "templates" } } } diff --git a/examples/tests/atmos.yaml b/examples/tests/atmos.yaml index 60c93e193..321c7a33c 100644 --- a/examples/tests/atmos.yaml +++ b/examples/tests/atmos.yaml @@ -380,3 +380,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml b/examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml index bf5f842f8..fbe95303b 100644 --- a/examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml +++ b/examples/tests/rootfs/usr/local/etc/atmos/atmos.yaml @@ -626,3 +626,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/examples/tests/stacks/catalog/terraform/eks_cluster_tmpl_hierarchical.yaml b/examples/tests/stacks/catalog/terraform/eks_cluster_tmpl_hierarchical.yaml index 491b03979..473ad5056 100644 --- a/examples/tests/stacks/catalog/terraform/eks_cluster_tmpl_hierarchical.yaml +++ b/examples/tests/stacks/catalog/terraform/eks_cluster_tmpl_hierarchical.yaml @@ -8,9 +8,6 @@ import: region: "{{ .region }}" environment: "{{ .environment }}" - # `Go` templates in the import path - - path: "orgs/cp/{{ .tenant }}/{{ .stage }}/_defaults" - components: terraform: # Parameterize Atmos component name diff --git a/examples/tests/stacks/orgs/cp/_defaults.yaml b/examples/tests/stacks/orgs/cp/_defaults.yaml index 5985d03a0..fbee39c1f 100644 --- a/examples/tests/stacks/orgs/cp/_defaults.yaml +++ b/examples/tests/stacks/orgs/cp/_defaults.yaml @@ -80,6 +80,17 @@ components: helmfile: { } settings: + templates: + settings: + gomplate: + timeout: 20 # 20 seconds timeout to execute the datasources + # https://docs.gomplate.ca/datasources + datasources: + ip: + url: "https://api.ipify.org?format=json" + headers: + accept: + - "application/json" spacelift: workspace_enabled: false autodeploy: false diff --git a/examples/tests/stacks/orgs/cp/tenant1/test1/us-west-1.yaml b/examples/tests/stacks/orgs/cp/tenant1/test1/us-west-1.yaml index db5268469..922bce37a 100644 --- a/examples/tests/stacks/orgs/cp/tenant1/test1/us-west-1.yaml +++ b/examples/tests/stacks/orgs/cp/tenant1/test1/us-west-1.yaml @@ -1,4 +1,6 @@ import: + - path: mixins/region/us-west-1 + - path: orgs/cp/tenant1/test1/_defaults # This import with the provided hierarchical context will dynamically generate # a new Atmos component `eks-blue/cluster` in the `tenant1-uw1-test-1` stack diff --git a/internal/exec/describe_affected_utils.go b/internal/exec/describe_affected_utils.go index b97affa65..046729989 100644 --- a/internal/exec/describe_affected_utils.go +++ b/internal/exec/describe_affected_utils.go @@ -1122,7 +1122,7 @@ func addAffectedSpaceliftAdminStack( var adminStackContextPrefix string if cliConfig.Stacks.NameTemplate != "" { - adminStackContextPrefix, err = u.ProcessTmpl(cliConfig, "spacelift-admin-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + adminStackContextPrefix, err = u.ProcessTmpl("spacelift-admin-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return nil, err } @@ -1158,7 +1158,7 @@ func addAffectedSpaceliftAdminStack( var contextPrefix string if cliConfig.Stacks.NameTemplate != "" { - contextPrefix, err = u.ProcessTmpl(cliConfig, "spacelift-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + contextPrefix, err = u.ProcessTmpl("spacelift-stack-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return nil, err } diff --git a/internal/exec/describe_stacks.go b/internal/exec/describe_stacks.go index 15cd3b62d..b32ab2c05 100644 --- a/internal/exec/describe_stacks.go +++ b/internal/exec/describe_stacks.go @@ -1,8 +1,10 @@ package exec import ( + "errors" "fmt" c "github.com/cloudposse/atmos/pkg/convert" + "github.com/mitchellh/mapstructure" "strings" "github.com/spf13/cobra" @@ -208,7 +210,7 @@ func ExecuteDescribeStacks( // Stack name if cliConfig.Stacks.NameTemplate != "" { - stackName, err = u.ProcessTmpl(cliConfig, "describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + stackName, err = u.ProcessTmpl("describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return nil, err } @@ -263,16 +265,29 @@ func ExecuteDescribeStacks( return nil, err } - componentSectionProcessed, err := u.ProcessTmpl(cliConfig, "describe-stacks-all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true) + var settingsSectionStruct schema.Settings + err = mapstructure.Decode(settingsSection, &settingsSectionStruct) if err != nil { return nil, err } - componentSectionConverted, err := c.YAMLToMapOfInterfaces(componentSectionProcessed) + componentSectionProcessed, err := u.ProcessTmplWithDatasources(cliConfig, settingsSectionStruct, "describe-stacks-all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true) if err != nil { return nil, err } + componentSectionConverted, err := c.YAMLToMapOfInterfaces(componentSectionProcessed) + if err != nil { + if !cliConfig.Templates.Settings.Enabled { + if strings.Contains(componentSectionStr, "{{") || strings.Contains(componentSectionStr, "}}") { + errorMessage := "the stack manifests contain Go templates, but templating is disabled in atmos.yaml in 'templates.settings.enabled'\n" + + "to enable templating, refer to https://atmos.tools/core-concepts/stacks/templating" + err = errors.Join(err, errors.New(errorMessage)) + } + } + u.LogErrorAndExit(err) + } + componentSection = c.MapsOfInterfacesToMapsOfStrings(componentSectionConverted) // Add sections @@ -369,7 +384,7 @@ func ExecuteDescribeStacks( // Stack name if cliConfig.Stacks.NameTemplate != "" { - stackName, err = u.ProcessTmpl(cliConfig, "describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + stackName, err = u.ProcessTmpl("describe-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return nil, err } @@ -416,16 +431,29 @@ func ExecuteDescribeStacks( return nil, err } - componentSectionProcessed, err := u.ProcessTmpl(cliConfig, "describe-stacks-all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true) + var settingsSectionStruct schema.Settings + err = mapstructure.Decode(settingsSection, &settingsSectionStruct) if err != nil { return nil, err } - componentSectionConverted, err := c.YAMLToMapOfInterfaces(componentSectionProcessed) + componentSectionProcessed, err := u.ProcessTmplWithDatasources(cliConfig, settingsSectionStruct, "describe-stacks-all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true) if err != nil { return nil, err } + componentSectionConverted, err := c.YAMLToMapOfInterfaces(componentSectionProcessed) + if err != nil { + if !cliConfig.Templates.Settings.Enabled { + if strings.Contains(componentSectionStr, "{{") || strings.Contains(componentSectionStr, "}}") { + errorMessage := "the stack manifests contain Go templates, but templating is disabled in atmos.yaml in 'templates.settings.enabled'\n" + + "to enable templating, refer to https://atmos.tools/core-concepts/stacks/templating" + err = errors.Join(err, errors.New(errorMessage)) + } + } + u.LogErrorAndExit(err) + } + componentSection = c.MapsOfInterfacesToMapsOfStrings(componentSectionConverted) // Add sections diff --git a/internal/exec/spacelift_utils.go b/internal/exec/spacelift_utils.go index dd87f8651..6598ed834 100644 --- a/internal/exec/spacelift_utils.go +++ b/internal/exec/spacelift_utils.go @@ -103,7 +103,7 @@ func BuildSpaceliftStackNameFromComponentConfig( context.Component = strings.Replace(configAndStacksInfo.ComponentFromArg, "/", "-", -1) if cliConfig.Stacks.NameTemplate != "" { - contextPrefix, err = u.ProcessTmpl(cliConfig, "name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + contextPrefix, err = u.ProcessTmpl("name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return "", err } diff --git a/internal/exec/stack_utils.go b/internal/exec/stack_utils.go index 13894dea9..8b557e3f8 100644 --- a/internal/exec/stack_utils.go +++ b/internal/exec/stack_utils.go @@ -17,7 +17,7 @@ func BuildTerraformWorkspace(cliConfig schema.CliConfiguration, configAndStacksI var tmpl string if cliConfig.Stacks.NameTemplate != "" { - tmpl, err = u.ProcessTmpl(cliConfig, "terraform-workspace-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + tmpl, err = u.ProcessTmpl("terraform-workspace-stacks-name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return "", err } @@ -36,7 +36,7 @@ func BuildTerraformWorkspace(cliConfig schema.CliConfiguration, configAndStacksI // Terraform workspace can be overridden per component using `metadata.terraform_workspace_pattern` or `metadata.terraform_workspace_template` or `metadata.terraform_workspace` if terraformWorkspaceTemplate, terraformWorkspaceTemplateExist := componentMetadata["terraform_workspace_template"].(string); terraformWorkspaceTemplateExist { - tmpl, err = u.ProcessTmpl(cliConfig, "terraform-workspace-template", terraformWorkspaceTemplate, configAndStacksInfo.ComponentSection, false) + tmpl, err = u.ProcessTmpl("terraform-workspace-template", terraformWorkspaceTemplate, configAndStacksInfo.ComponentSection, false) if err != nil { return "", err } diff --git a/internal/exec/utils.go b/internal/exec/utils.go index 33fcb3760..e419db7ce 100644 --- a/internal/exec/utils.go +++ b/internal/exec/utils.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/hashicorp/terraform-config-inspect/tfconfig" + "github.com/mitchellh/mapstructure" "github.com/spf13/cobra" cfg "github.com/cloudposse/atmos/pkg/config" @@ -347,7 +348,7 @@ func ProcessStacks( configAndStacksInfo.ComponentEnvList = u.ConvertEnvVars(configAndStacksInfo.ComponentEnvSection) if cliConfig.Stacks.NameTemplate != "" { - tmpl, err2 := u.ProcessTmpl(cliConfig, "name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) + tmpl, err2 := u.ProcessTmpl("name-template", cliConfig.Stacks.NameTemplate, configAndStacksInfo.ComponentSection, false) if err2 != nil { continue } @@ -535,13 +536,20 @@ func ProcessStacks( configAndStacksInfo.ComponentSection["deps"] = componentDeps configAndStacksInfo.ComponentSection["deps_all"] = componentDepsAll - // Process `Go` templates in sections + // Process `Go` templates in Atmos manifest sections componentSectionStr, err := u.ConvertToYAML(configAndStacksInfo.ComponentSection) if err != nil { return configAndStacksInfo, err } - componentSectionProcessed, err := u.ProcessTmpl(cliConfig, "all-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true) + var settingsSectionStruct schema.Settings + + err = mapstructure.Decode(configAndStacksInfo.ComponentSettingsSection, &settingsSectionStruct) + if err != nil { + return configAndStacksInfo, err + } + + componentSectionProcessed, err := u.ProcessTmplWithDatasources(cliConfig, settingsSectionStruct, "all-atmos-sections", componentSectionStr, configAndStacksInfo.ComponentSection, true) if err != nil { // If any error returned from the templates processing, log it and exit u.LogErrorAndExit(err) @@ -549,11 +557,19 @@ func ProcessStacks( componentSectionConverted, err := c.YAMLToMapOfInterfaces(componentSectionProcessed) if err != nil { - return configAndStacksInfo, err + if !cliConfig.Templates.Settings.Enabled { + if strings.Contains(componentSectionStr, "{{") || strings.Contains(componentSectionStr, "}}") { + errorMessage := "the stack manifests contain Go templates, but templating is disabled in atmos.yaml in 'templates.settings.enabled'\n" + + "to enable templating, refer to https://atmos.tools/core-concepts/stacks/templating" + err = errors.Join(err, errors.New(errorMessage)) + } + } + u.LogErrorAndExit(err) } configAndStacksInfo.ComponentSection = c.MapsOfInterfacesToMapsOfStrings(componentSectionConverted) + // Process Atmos manifest sections if i, ok := configAndStacksInfo.ComponentSection[cfg.ProvidersSectionName].(map[any]any); ok { configAndStacksInfo.ComponentProvidersSection = i } diff --git a/internal/exec/vendor_utils.go b/internal/exec/vendor_utils.go index e5fd10743..70931ed74 100644 --- a/internal/exec/vendor_utils.go +++ b/internal/exec/vendor_utils.go @@ -284,7 +284,7 @@ func ExecuteAtmosVendorInternal( // Parse 'source' template if s.Version != "" { - uri, err = u.ProcessTmpl(cliConfig, fmt.Sprintf("source-%d-%s", indexSource, s.Version), s.Source, s, false) + uri, err = u.ProcessTmpl(fmt.Sprintf("source-%d-%s", indexSource, s.Version), s.Source, s, false) if err != nil { return err } @@ -318,7 +318,7 @@ func ExecuteAtmosVendorInternal( var target string // Parse 'target' template if s.Version != "" { - target, err = u.ProcessTmpl(cliConfig, fmt.Sprintf("target-%d-%d-%s", indexSource, indexTarget, s.Version), tgt, s, false) + target, err = u.ProcessTmpl(fmt.Sprintf("target-%d-%d-%s", indexSource, indexTarget, s.Version), tgt, s, false) if err != nil { return err } diff --git a/pkg/atlantis/atmos.yaml b/pkg/atlantis/atmos.yaml index e75f97488..b2de0b448 100644 --- a/pkg/atlantis/atmos.yaml +++ b/pkg/atlantis/atmos.yaml @@ -358,3 +358,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/pkg/aws/atmos.yaml b/pkg/aws/atmos.yaml index e59cd6699..d350dcd70 100644 --- a/pkg/aws/atmos.yaml +++ b/pkg/aws/atmos.yaml @@ -358,3 +358,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/pkg/component/atmos.yaml b/pkg/component/atmos.yaml index e75f97488..b2de0b448 100644 --- a/pkg/component/atmos.yaml +++ b/pkg/component/atmos.yaml @@ -358,3 +358,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/pkg/config/config.go b/pkg/config/config.go index d4c77cb7f..06fdb9561 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -74,7 +74,8 @@ var ( Enabled: true, }, Gomplate: schema.TemplatesSettingsGomplate{ - Enabled: true, + Enabled: true, + Datasources: make(map[string]schema.TemplatesSettingsGomplateDatasource), }, }, }, diff --git a/pkg/describe/atmos.yaml b/pkg/describe/atmos.yaml index e75f97488..b2de0b448 100644 --- a/pkg/describe/atmos.yaml +++ b/pkg/describe/atmos.yaml @@ -358,3 +358,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/pkg/generate/atmos.yaml b/pkg/generate/atmos.yaml index e75f97488..b2de0b448 100644 --- a/pkg/generate/atmos.yaml +++ b/pkg/generate/atmos.yaml @@ -358,3 +358,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/pkg/merge/merge.go b/pkg/merge/merge.go index 0b5d2ee3d..b4b530305 100644 --- a/pkg/merge/merge.go +++ b/pkg/merge/merge.go @@ -1,7 +1,7 @@ package merge import ( - u "github.com/cloudposse/atmos/pkg/utils" + "github.com/fatih/color" "github.com/imdario/mergo" "gopkg.in/yaml.v2" ) @@ -25,13 +25,15 @@ func MergeWithOptions(inputs []map[any]any, appendSlice, sliceDeepCopy bool) (ma // so `mergo` does not have access to the original pointers yamlCurrent, err := yaml.Marshal(current) if err != nil { - u.LogError(err) + c := color.New(color.FgRed) + _, _ = c.Fprintln(color.Error, err.Error()+"\n") return nil, err } var dataCurrent map[any]any if err = yaml.Unmarshal(yamlCurrent, &dataCurrent); err != nil { - u.LogError(err) + c := color.New(color.FgRed) + _, _ = c.Fprintln(color.Error, err.Error()+"\n") return nil, err } @@ -52,7 +54,8 @@ func MergeWithOptions(inputs []map[any]any, appendSlice, sliceDeepCopy bool) (ma } if err = mergo.Merge(&merged, dataCurrent, opts...); err != nil { - u.LogError(err) + c := color.New(color.FgRed) + _, _ = c.Fprintln(color.Error, err.Error()+"\n") return nil, err } } diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index 6070f8f1a..8efffc5a8 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -37,8 +37,15 @@ type TemplatesSettingsSprig struct { Enabled bool `yaml:"enabled" json:"enabled" mapstructure:"enabled"` } +type TemplatesSettingsGomplateDatasource struct { + Url string `yaml:"url" json:"url" mapstructure:"url"` + Headers map[string][]string `yaml:"headers" json:"headers" mapstructure:"headers"` +} + type TemplatesSettingsGomplate struct { - Enabled bool `yaml:"enabled" json:"enabled" mapstructure:"enabled"` + Enabled bool `yaml:"enabled" json:"enabled" mapstructure:"enabled"` + Timeout int `yaml:"timeout" json:"timeout" mapstructure:"timeout"` + Datasources map[string]TemplatesSettingsGomplateDatasource `yaml:"datasources" json:"datasources" mapstructure:"datasources"` } type Terraform struct { @@ -460,8 +467,9 @@ type Dependent struct { type SettingsSpacelift map[any]any type Settings struct { - DependsOn DependsOn `yaml:"depends_on" json:"depends_on" mapstructure:"depends_on"` - Spacelift SettingsSpacelift `yaml:"spacelift" json:"spacelift" mapstructure:"spacelift"` + DependsOn DependsOn `yaml:"depends_on,omitempty" json:"depends_on,omitempty" mapstructure:"depends_on"` + Spacelift SettingsSpacelift `yaml:"spacelift,omitempty" json:"spacelift,omitempty" mapstructure:"spacelift"` + Templates Templates `yaml:"templates,omitempty" json:"templates,omitempty" mapstructure:"templates"` } // ConfigSourcesStackDependency defines schema for sources of config sections diff --git a/pkg/spacelift/atmos.yaml b/pkg/spacelift/atmos.yaml index e75f97488..b2de0b448 100644 --- a/pkg/spacelift/atmos.yaml +++ b/pkg/spacelift/atmos.yaml @@ -358,3 +358,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/pkg/stack/stack_processor.go b/pkg/stack/stack_processor.go index 3fd940839..d61795f28 100644 --- a/pkg/stack/stack_processor.go +++ b/pkg/stack/stack_processor.go @@ -196,17 +196,28 @@ func ProcessYAMLConfigFile( } } - // Process `Go` templates in the imported stack manifest using the provided context + stackManifestTemplatesProcessed := stackYamlConfig + stackManifestTemplatesErrorMessage := "" + + // Process `Go` templates in the imported stack manifest using the provided `context` + // https://atmos.tools/core-concepts/stacks/imports#go-templates-in-imports if !skipTemplatesProcessingInImports && len(context) > 0 { - stackYamlConfig, err = u.ProcessTmpl(cliConfig, relativeFilePath, stackYamlConfig, context, ignoreMissingTemplateValues) + stackManifestTemplatesProcessed, err = u.ProcessTmpl(relativeFilePath, stackYamlConfig, context, ignoreMissingTemplateValues) if err != nil { - return nil, nil, nil, err + if cliConfig.Logs.Level == u.LogLevelTrace || cliConfig.Logs.Level == u.LogLevelDebug { + stackManifestTemplatesErrorMessage = fmt.Sprintf("\n\n%s", stackYamlConfig) + } + e := fmt.Errorf("invalid stack manifest '%s'\n%v%s", relativeFilePath, err, stackManifestTemplatesErrorMessage) + return nil, nil, nil, e } } - stackConfigMap, err := c.YAMLToMapOfInterfaces(stackYamlConfig) + stackConfigMap, err := c.YAMLToMapOfInterfaces(stackManifestTemplatesProcessed) if err != nil { - e := fmt.Errorf("invalid stack manifest '%s'\n%v", relativeFilePath, err) + if cliConfig.Logs.Level == u.LogLevelTrace || cliConfig.Logs.Level == u.LogLevelDebug { + stackManifestTemplatesErrorMessage = fmt.Sprintf("\n\n%s", stackYamlConfig) + } + e := fmt.Errorf("invalid stack manifest '%s'\n%v%s", relativeFilePath, err, stackManifestTemplatesErrorMessage) return nil, nil, nil, e } diff --git a/pkg/stack/stack_processor_test.go b/pkg/stack/stack_processor_test.go index 9861bf6e0..39e9a5877 100644 --- a/pkg/stack/stack_processor_test.go +++ b/pkg/stack/stack_processor_test.go @@ -26,8 +26,22 @@ func TestStackProcessor(t *testing.T) { processStackDeps := true processComponentDeps := true + cliConfig := schema.CliConfiguration{ + Templates: schema.Templates{ + Settings: schema.TemplatesSettings{ + Enabled: true, + Sprig: schema.TemplatesSettingsSprig{ + Enabled: true, + }, + Gomplate: schema.TemplatesSettingsGomplate{ + Enabled: true, + }, + }, + }, + } + var listResult, mapResult, _, err = ProcessYAMLConfigFiles( - schema.CliConfiguration{}, + cliConfig, stacksBasePath, terraformComponentsBasePath, helmfileComponentsBasePath, diff --git a/pkg/stack/stack_processor_utils.go b/pkg/stack/stack_processor_utils.go index 89c717b15..2660eafe2 100644 --- a/pkg/stack/stack_processor_utils.go +++ b/pkg/stack/stack_processor_utils.go @@ -235,6 +235,7 @@ func sectionContainsAnyNotEmptySections(section map[any]any, sectionsToCheck []s // CreateComponentStackMap accepts a config file and creates a map of component-stack dependencies func CreateComponentStackMap( + cliConfig schema.CliConfiguration, stacksBasePath string, terraformComponentsBasePath string, helmfileComponentsBasePath string, @@ -266,7 +267,7 @@ func CreateComponentStackMap( if !isDirectory && isYaml { config, _, _, err := ProcessYAMLConfigFile( - schema.CliConfiguration{}, + cliConfig, stacksBasePath, p, map[string]map[any]any{}, diff --git a/pkg/utils/log_utils.go b/pkg/utils/log_utils.go index 9f2daccb0..89078db3f 100644 --- a/pkg/utils/log_utils.go +++ b/pkg/utils/log_utils.go @@ -110,7 +110,7 @@ func log(cliConfig schema.CliConfiguration, logColor *color.Color, message strin color.Red("%s\n", err) } } else { - f, err := os.OpenFile(cliConfig.Logs.File, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0644) + f, err := os.OpenFile(cliConfig.Logs.File, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) if err != nil { color.Red("%s\n", err) return diff --git a/pkg/utils/template_utils.go b/pkg/utils/template_utils.go index f1770681d..3bc3daa5f 100644 --- a/pkg/utils/template_utils.go +++ b/pkg/utils/template_utils.go @@ -5,17 +5,51 @@ import ( "context" "text/template" "text/template/parse" + "time" "github.com/Masterminds/sprig/v3" "github.com/hairyhenderson/gomplate/v3" + "github.com/hairyhenderson/gomplate/v3/data" + "github.com/mitchellh/mapstructure" "github.com/samber/lo" + "github.com/cloudposse/atmos/pkg/merge" "github.com/cloudposse/atmos/pkg/schema" ) // ProcessTmpl parses and executes Go templates -func ProcessTmpl( +func ProcessTmpl(tmplName string, tmplValue string, tmplData any, ignoreMissingTemplateValues bool) (string, error) { + t, err := template.New(tmplName).Funcs(sprig.FuncMap()).Parse(tmplValue) + if err != nil { + return "", err + } + + // Control the behavior during execution if a map is indexed with a key that is not present in the map + // If the template context (`tmplData`) does not provide all the required variables, the following errors would be thrown: + // template: catalog/terraform/eks_cluster_tmpl_hierarchical.yaml:17:12: executing "catalog/terraform/eks_cluster_tmpl_hierarchical.yaml" at <.flavor>: map has no entry for key "flavor" + // template: catalog/terraform/eks_cluster_tmpl_hierarchical.yaml:12:36: executing "catalog/terraform/eks_cluster_tmpl_hierarchical.yaml" at <.stage>: map has no entry for key "stage" + + option := "missingkey=error" + + if ignoreMissingTemplateValues { + option = "missingkey=default" + } + + t.Option(option) + + var res bytes.Buffer + err = t.Execute(&res, tmplData) + if err != nil { + return "", err + } + + return res.String(), nil +} + +// ProcessTmplWithDatasources parses and executes Go templates with datasources +func ProcessTmplWithDatasources( cliConfig schema.CliConfiguration, + settingsSection schema.Settings, tmplName string, tmplValue string, tmplData any, @@ -25,16 +59,66 @@ func ProcessTmpl( return tmplValue, nil } - // Add Gomplate and Sprig functions + // Add Gomplate and Sprig functions and datasources funcs := make(map[string]any) + // Gomplate functions and datasources if cliConfig.Templates.Settings.Gomplate.Enabled { - funcs = lo.Assign(funcs, gomplate.CreateFuncs(context.Background(), nil)) + // Merge the datasources from `atmos.yaml` and from the `settings.templates.settings` section in stack manifests + var cliConfigDatasources map[any]any + var stackManifestDatasources map[any]any + + err := mapstructure.Decode(cliConfig.Templates.Settings.Gomplate.Datasources, &cliConfigDatasources) + if err != nil { + return "", err + } + + err = mapstructure.Decode(settingsSection.Templates.Settings.Gomplate.Datasources, &stackManifestDatasources) + if err != nil { + return "", err + } + + merged, err := merge.Merge([]map[any]any{cliConfigDatasources, stackManifestDatasources}) + if err != nil { + return "", err + } + + var datasources map[string]schema.TemplatesSettingsGomplateDatasource + err = mapstructure.Decode(merged, &datasources) + if err != nil { + return "", err + } + + // If timeout is not provided in `atmos.yaml` nor in `settings.templates.settings` stack manifest, use 5 seconds + timeoutSeconds, _ := lo.Coalesce(cliConfig.Templates.Settings.Gomplate.Timeout, settingsSection.Templates.Settings.Gomplate.Timeout, 5) + + ctx, cancelFunc := context.WithTimeout(context.TODO(), time.Second*time.Duration(timeoutSeconds)) + defer cancelFunc() + + d := data.Data{} + d.Ctx = ctx + + for k, v := range datasources { + _, err := d.DefineDatasource(k, v.Url) + if err != nil { + return "", err + } + + // Add datasource headers + if len(v.Headers) > 0 { + d.Sources[k].Header = v.Headers + } + } + + funcs = lo.Assign(funcs, gomplate.CreateFuncs(ctx, &d)) } + + // Sprig functions if cliConfig.Templates.Settings.Sprig.Enabled { funcs = lo.Assign(funcs, sprig.FuncMap()) } + // Process the template t, err := template.New(tmplName).Funcs(funcs).Parse(tmplValue) if err != nil { return "", err @@ -53,6 +137,7 @@ func ProcessTmpl( t.Option(option) + // Execute the template var res bytes.Buffer err = t.Execute(&res, tmplData) if err != nil { diff --git a/pkg/validate/atmos.yaml b/pkg/validate/atmos.yaml index e75f97488..b2de0b448 100644 --- a/pkg/validate/atmos.yaml +++ b/pkg/validate/atmos.yaml @@ -358,3 +358,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/pkg/vender/atmos.yaml b/pkg/vender/atmos.yaml index e75f97488..b2de0b448 100644 --- a/pkg/vender/atmos.yaml +++ b/pkg/vender/atmos.yaml @@ -358,3 +358,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/pkg/workflow/atmos.yaml b/pkg/workflow/atmos.yaml index e75f97488..b2de0b448 100644 --- a/pkg/workflow/atmos.yaml +++ b/pkg/workflow/atmos.yaml @@ -358,3 +358,6 @@ templates: # https://docs.gomplate.ca gomplate: enabled: true + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: {} diff --git a/website/docs/cli/configuration.mdx b/website/docs/cli/configuration.mdx index a884cadbc..f7385700c 100644 --- a/website/docs/cli/configuration.mdx +++ b/website/docs/cli/configuration.mdx @@ -40,7 +40,7 @@ names/paths (double-star/globstar `**` is supported as well) If `atmos.yaml` is not found in any of the searched locations, Atmos will use the default CLI configuration: -```yaml +```yaml title="atmos.yaml" base_path: "." components: terraform: @@ -127,7 +127,7 @@ base_path: "." Specify the default behaviors for components. -```yaml +```yaml title="atmos.yaml" components: terraform: # Can also be set using 'ATMOS_COMPONENTS_TERRAFORM_BASE_PATH' ENV var, or '--terraform-dir' command-line argument @@ -169,7 +169,7 @@ components: Define the stack name pattern or template and specify where to find stacks. -```yaml +```yaml title="atmos.yaml" stacks: # Can also be set using 'ATMOS_STACKS_BASE_PATH' ENV var, or '--config-dir' and '--stacks-dir' command-line arguments # Supports both absolute and relative paths @@ -304,7 +304,7 @@ Refer to [Atmos Design Patterns](/design-patterns) for the examples on how to co ## Workflows -```yaml +```yaml title="atmos.yaml" workflows: # Can also be set using 'ATMOS_WORKFLOWS_BASE_PATH' ENV var, or '--workflows-dir' command-line arguments # Supports both absolute and relative paths @@ -322,7 +322,7 @@ Then developers can just run `atmos help` and discover all available commands. Here are some examples to play around with to get started. -```yaml +```yaml title="atmos.yaml" # Custom CLI commands commands: - name: tf @@ -513,7 +513,7 @@ commands: ## Integrations -```yaml +```yaml title="atmos.yaml" # Integrations integrations: @@ -581,7 +581,7 @@ integrations: Configure the paths where to find OPA and JSON Schema files. -```yaml +```yaml title="atmos.yaml" # Validation schemas (for validating atmos stacks and components) schemas: # https://json-schema.org @@ -610,7 +610,7 @@ schemas: Logs are configured in the `logs` section: -```yaml +```yaml title="atmos.yaml" logs: # Can also be set using 'ATMOS_LOGS_FILE' ENV var, or '--logs-file' command-line argument file: "/dev/stdout" @@ -642,7 +642,7 @@ native commands like `terraform apply` or `describe stacks`, as well as [Atmos C For example: -```yaml +```yaml title="atmos.yaml" # CLI command aliases aliases: # Aliases for Atmos native commands @@ -695,7 +695,9 @@ For example: ## Templates Atmos supports [Go templates](https://pkg.go.dev/text/template) in stack manifests. -[Sprig Functions](https://masterminds.github.io/sprig/) and [Gomplate Functions](https://docs.gomplate.ca/functions/) + +[Sprig Functions](https://masterminds.github.io/sprig/), [Gomplate Functions](https://docs.gomplate.ca/functions/) +and [Gomplate Datasources](https://docs.gomplate.ca/datasources/) are supported as well. :::tip @@ -704,7 +706,7 @@ For more details, refer to [Atmos Stack Manifest Templating](/core-concepts/stac
-```yaml +```yaml title="atmos.yaml" # https://pkg.go.dev/text/template templates: settings: @@ -713,8 +715,28 @@ templates: sprig: enabled: true # https://docs.gomplate.ca + # https://docs.gomplate.ca/functions gomplate: enabled: true + # Timeout in seconds to execute the datasources + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: + # 'http' datasource + # https://docs.gomplate.ca/datasources/#using-file-datasources + ip: + url: "https://api.ipify.org?format=json" + # https://docs.gomplate.ca/datasources/#sending-http-headers + # https://docs.gomplate.ca/usage/#--datasource-header-h + headers: + accept: + - "application/json" + # 'file' datasources + # https://docs.gomplate.ca/datasources/#using-file-datasources + config-1: + url: "./config1.json" + config-2: + url: "file:///config2.json" ``` - `templates.settings.enabled` - a boolean flag to enable/disable the processing of `Go` templates in Atmos stack manifests. @@ -724,7 +746,41 @@ templates: in Atmos stack manifests - `templates.settings.gomplate.enabled` - a boolean flag to enable/disable the [Gomplate Functions](https://docs.gomplate.ca/functions/) - in Atmos stack manifests + and [Gomplate Datasources](https://docs.gomplate.ca/datasources) in Atmos stack manifests + +- `templates.settings.gomplate.timeout` - timeout in seconds to execute [Gomplate Datasources](https://docs.gomplate.ca/datasources) + +- `templates.settings.gomplate.datasources` - a map of [Gomplate Datasource](https://docs.gomplate.ca/datasources) definitions: + + - The keys of the map are the datasource names, which are used in `Go` templates in Atmos stack manifests. + For example: + + ```yaml + terraform: + vars: + tags: + provisioned_by_ip: '{{ (datasource "ip").ip }}' + config1_tag: '{{ (datasource "config-1").tag }}' + config2_service_name: '{{ (datasource "config-2").service.name }}' + ``` + + - The values of the map are the datasource definitions with the following schema: + + - `url` - the [Datasource URL](https://docs.gomplate.ca/datasources/#url-format) + + - `headers` - a map of [HTTP request headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers) for + the [`http` datasource](https://docs.gomplate.ca/datasources/#sending-http-headers). + The keys of the map are the header names. The values of the map are lists of values for the header. + + The following configuration will result in the + [`accept: application/json`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept) HTTP header + being sent with the HTTP request to the datasource: + + ```yaml + headers: + accept: + - "application/json" + ```
diff --git a/website/docs/core-concepts/stacks/imports.md b/website/docs/core-concepts/stacks/imports.md index 1d8e57729..e2fbfd089 100644 --- a/website/docs/core-concepts/stacks/imports.md +++ b/website/docs/core-concepts/stacks/imports.md @@ -55,7 +55,7 @@ The `import` section supports the following two formats: - a list of paths to the imported files, for example: - ```yaml title=stacks/orgs/cp/tenant1/test1/us-east-2.yaml + ```yaml title="stacks/orgs/cp/tenant1/test1/us-east-2.yaml" import: - mixins/region/us-east-2 - orgs/cp/tenant1/test1/_defaults @@ -122,9 +122,9 @@ Stack configurations can be templatized and then reused with different settings For example, we can define the following configuration for EKS Atmos components in the `catalog/terraform/eks_cluster_tmpl.yaml` template file: -```yaml title=stacks/catalog/terraform/eks_cluster_tmpl.yaml +```yaml title="stacks/catalog/terraform/eks_cluster_tmpl.yaml" # Imports can also be parameterized using `Go` templates -import: [ ] +import: [] components: terraform: @@ -153,7 +153,7 @@ sections (`vars`, `settings`, `env`, `backend`, etc.), and even the `import` sec Then we can import the template into a top-level stack multiple times providing different context variables to each import: -```yaml title=stacks/orgs/cp/tenant1/test1/us-west-2.yaml +```yaml title="stacks/orgs/cp/tenant1/test1/us-west-2.yaml" import: - path: "mixins/region/us-west-2" - path: "orgs/cp/tenant1/test1/_defaults" @@ -228,7 +228,7 @@ This will allow you to parameterize the entire chain of stack configurations and For example, let's create the configuration `stacks/catalog/terraform/eks_cluster_tmpl_hierarchical.yaml` with the following content: -```yaml title=stacks/catalog/terraform/eks_cluster_tmpl_hierarchical.yaml +```yaml title="stacks/catalog/terraform/eks_cluster_tmpl_hierarchical.yaml" import: # Use `region_tmpl` `Go` template and provide `context` for it. # This can also be done by using `Go` templates in the import path itself. @@ -263,7 +263,7 @@ components: Then we can import the template into a top-level stack multiple times providing different context variables to each import and to the imports for the entire inheritance chain (which `catalog/terraform/eks_cluster_tmpl_hierarchical` imports itself): -```yaml title=stacks/orgs/cp/tenant1/test1/us-west-1.yaml +```yaml title="stacks/orgs/cp/tenant1/test1/us-west-1.yaml" import: # This import with the provided hierarchical context will dynamically generate diff --git a/website/docs/core-concepts/stacks/templating.md b/website/docs/core-concepts/stacks/templating.md index 809b18848..491d4f7e2 100644 --- a/website/docs/core-concepts/stacks/templating.md +++ b/website/docs/core-concepts/stacks/templating.md @@ -6,15 +6,27 @@ id: templating --- Atmos supports [Go templates](https://pkg.go.dev/text/template) in stack manifests. -[Sprig Functions](https://masterminds.github.io/sprig/) and [Gomplate Functions](https://docs.gomplate.ca/functions/) + +[Sprig Functions](https://masterminds.github.io/sprig/), [Gomplate Functions](https://docs.gomplate.ca/functions/) +and [Gomplate Datasources](https://docs.gomplate.ca/datasources/) are supported as well. ## Configuration +Templating in Atmos stack manifests can be configured in the following places: + +- In the `templates.settings` section in `atmos.yaml` [CLI config file](/cli/configuration) + +- In the `settings.templates.settings` section in [Atmos stack manifests](/core-concepts/stacks). + The `settings.templates.settings` section can be defined globally per organization, tenant, account, or per component. + Atmos deep-merges the configurations from all scopes into the final result using [inheritance](/core-concepts/components/inheritance). + +### Configuring templating in `atmos.yaml` CLI config file + Templating in Atmos stack manifests is configured in the `atmos.yaml` [CLI config file](/cli/configuration) in the -`templates` section: +`templates.settings` section: -```yaml +```yaml title="atmos.yaml" # https://pkg.go.dev/text/template templates: settings: @@ -23,18 +35,84 @@ templates: sprig: enabled: true # https://docs.gomplate.ca + # https://docs.gomplate.ca/functions gomplate: enabled: true + # Timeout in seconds to execute the datasources + timeout: 5 + # https://docs.gomplate.ca/datasources + datasources: + # 'http' datasource + # https://docs.gomplate.ca/datasources/#using-file-datasources + ip: + url: "https://api.ipify.org?format=json" + # https://docs.gomplate.ca/datasources/#sending-http-headers + # https://docs.gomplate.ca/usage/#--datasource-header-h + headers: + accept: + - "application/json" + # 'file' datasources + # https://docs.gomplate.ca/datasources/#using-file-datasources + config-1: + url: "./config1.json" + config-2: + url: "file:///config2.json" + # `aws+smp` AWS Systems Manager Parameter Store datasource + # https://docs.gomplate.ca/datasources/#using-awssmp-datasources + secret-1: + url: "aws+smp:///path/to/secret" + # `aws+sm` AWS Secrets Manager datasource + # https://docs.gomplate.ca/datasources/#using-awssm-datasource + secret-2: + url: "aws+sm:///path/to/secret" + # `s3` datasource + # https://docs.gomplate.ca/datasources/#using-s3-datasources + s3-config: + url: "s3://mybucket/config/config.json" ``` -- `templates.settings.enabled` - a boolean flag to enable/disable the processing of `Go` templates in Atmos stack manifests. +- `templates.settings.enabled` - a boolean flag to enable/disable the processing of `Go` templates in Atmos stack manifests. If set to `false`, Atmos will not process `Go` templates in stack manifests - `templates.settings.sprig.enabled` - a boolean flag to enable/disable the [Sprig Functions](https://masterminds.github.io/sprig/) in Atmos stack manifests -- `templates.settings.gomplate.enabled` - a boolean flag to enable/disable the [Gomplate Functions](https://docs.gomplate.ca/functions/) - in Atmos stack manifests +- `templates.settings.gomplate.enabled` - a boolean flag to enable/disable the [Gomplate Functions](https://docs.gomplate.ca/functions/) + and [Gomplate Datasources](https://docs.gomplate.ca/datasources) in Atmos stack manifests + +- `templates.settings.gomplate.timeout` - timeout in seconds to execute [Gomplate Datasources](https://docs.gomplate.ca/datasources) + +- `templates.settings.gomplate.datasources` - a map of [Gomplate Datasource](https://docs.gomplate.ca/datasources) definitions: + + - The keys of the map are the datasource names, which are used in `Go` templates in Atmos stack manifests. + For example: + + ```yaml + terraform: + vars: + tags: + provisioned_by_ip: '{{ (datasource "ip").ip }}' + config1_tag: '{{ (datasource "config-1").tag }}' + config2_service_name: '{{ (datasource "config-2").service.name }}' + ``` + + - The values of the map are the datasource definitions with the following schema: + + - `url` - the [Datasource URL](https://docs.gomplate.ca/datasources/#url-format) + + - `headers` - a map of [HTTP request headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers) for + the [`http` datasource](https://docs.gomplate.ca/datasources/#sending-http-headers). + The keys of the map are the header names. The values of the map are lists of values for the header. + + The following configuration will result in the + [`accept: application/json`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept) HTTP header + being sent with the HTTP request to the datasource: + + ```yaml + headers: + accept: + - "application/json" + ```
@@ -56,6 +134,64 @@ functions.
+### Configuring templating in Atmos stack manifests + +The `settings.templates.settings` section can be defined globally per organization, tenant, account, or per component. +Atmos deep-merges the configurations from all scopes into the final result using [inheritance](/core-concepts/components/inheritance). + +For example, define [Gomplate Datasources](https://docs.gomplate.ca/datasources/) for the entire organization in the +`stacks/orgs/acme/_defaults.yaml` stack manifest: + +```yaml title="stacks/orgs/acme/_defaults.yaml" +settings: + templates: + settings: + gomplate: + # 7 seconds timeout to execute the datasources + timeout: 7 + # https://docs.gomplate.ca/datasources + datasources: + # 'file' datasources + # https://docs.gomplate.ca/datasources/#using-file-datasources + config-1: + url: "./my-config1.json" + config-3: + url: "file:///config3.json" +``` + +Atmos deep-merges the configurations from the `settings.templates.settings` section in [Atmos stack manifests](/core-concepts/stacks) +with the `templates.settings` section in `atmos.yaml` [CLI config file](/cli/configuration) using [inheritance](/core-concepts/components/inheritance). + +The `settings.templates.settings` section in [Atmos stack manifests](/core-concepts/stacks) takes precedence over +the `templates.settings` section in `atmos.yaml` [CLI config file](/cli/configuration), allowing you to define the global +`datasources` in `atmos.yaml` and then add or override `datasources` in Atmos stack manifests for the entire organization, +tenant, account, or per component. + +For example, taking into account the configurations described above in `atmos.yaml` [CLI config file](/cli/configuration) +and in the `stacks/orgs/acme/_defaults.yaml` stack manifest, the final `datasources` map will look like this: + +```yaml +gomplate: + timeout: 7 + datasources: + ip: + url: "https://api.ipify.org?format=json" + headers: + accept: + - "application/json" + config-1: + url: "./my-config1.json" + config-2: + url: "file:///config2.json" + config-3: + url: "file:///config3.json" +``` + +Note that the `config-1` datasource from `atmos.yaml` was overridden with the `config-1` datasource from the +`stacks/orgs/acme/_defaults.yaml` stack manifest. The `timeout` attribute was overridden as well. + +You can now use the `datasources` in `Go` templates in all Atmos sections that support `Go` templates. + ## Atmos sections supporting `Go` templates You can use `Go` templates in the following Atmos sections to refer to values in the same or other sections: @@ -116,11 +252,16 @@ component: terraform_workspace: "{{ .workspace }}" assumed_role: "{{ .providers.aws.assume_role }}" description: "{{ .atmos_component }} component provisioned in {{ .atmos_stack }} stack by assuming IAM role {{ .providers.aws.assume_role }}" - # Examples of using the Gomplate and Sprig functions - # https://docs.gomplate.ca/functions/strings - atmos_component_description: "{{ strings.Title .atmos_component }} component {{ .vars.name | strings.Quote }} provisioned in the stack {{ .atmos_stack | strings.Quote }}" + # Examples of using the Sprig and Gomplate functions and datasources # https://masterminds.github.io/sprig/os.html provisioned_by_user: '{{ env "USER" }}' + # https://docs.gomplate.ca/functions/strings + atmos_component_description: "{{ strings.Title .atmos_component }} component {{ .vars.name | strings.Quote }} provisioned in the stack {{ .atmos_stack | strings.Quote }}" + # https://docs.gomplate.ca/datasources + provisioned_by_ip: '{{ (datasource "ip").ip }}' + config1_tag: '{{ (datasource "config-1").tag }}' + config2_service_name: '{{ (datasource "config-2").service.name }}' + config3_team_name: '{{ (datasource "config-3").team.name }}' ``` When executing Atmos commands like `atmos describe component` and `atmos terraform plan/apply`, Atmos processes all the template tokens @@ -156,8 +297,12 @@ vars: atmos_component_description: Vpc component "common" provisioned in the stack "plat-ue2-dev" atmos_manifest: orgs/acme/plat/dev/us-east-2 atmos_stack: plat-ue2-dev + config1_tag: test1 + config2_service_name: service1 + config3_team_name: my-team description: vpc component provisioned in plat-ue2-dev stack by assuming IAM role provisioned_by_user: + provisioned_by_ip: 167.38.132.237 region: us-east-2 terraform_workspace: plat-ue2-dev ``` @@ -186,7 +331,7 @@ terraform: The tags will be processed and automatically added to all the resources provisioned in the infrastructure. -## Excluding templates from processing by Atmos +## Excluding templates in stack manifest from processing by Atmos If you need to provide `Go` templates to external systems (e.g. ArgoCD or Datadog) verbatim and prevent Atmos from processing the templates, use **double curly braces + backtick + double curly braces** instead of just **double curly braces**: @@ -287,3 +432,107 @@ chart_values: template-github-commit-status: message: '{{ printf "Application {{ .app.metadata.name }} is now running new version." }}' ``` + +## Excluding templates in imports from processing by Atmos + +If you are using [`Go` Templates in Imports](/core-concepts/stacks/imports#go-templates-in-imports) and `Go` templates +in stack manifests in the same Atmos manifest, take into account that in this case Atmos will do `Go` +template processing two times (two passes): + + - When importing the manifest and processing the template tokens using the variables from the provided `context` object + - After finding the component in the stack as the final step in the processing pipeline + +
+ +For example, we can define the following configuration in the `stacks/catalog/eks/eks_cluster.tmpl` template file: + +```yaml title="stacks/catalog/eks/eks_cluster.tmpl" +components: + terraform: + eks/cluster: + metadata: + component: eks/cluster + vars: + enabled: "{{ .enabled }}" + name: "{{ .name }}" + tags: + atmos_component: "{{ .atmos_component }}" + atmos_stack: "{{ .atmos_stack }}" + terraform_workspace: "{{ .workspace }}" +``` + +
+ +Then we import the template into a top-level stack providing the context variables for the import in the `context` object: + +```yaml title="stacks/orgs/acme/plat/prod/us-east-2.yaml" +import: + - path: "catalog/eks/eks_cluster.tmpl" + context: + enabled: true + name: prod-eks +``` + +Atmos will process the import and replace the template tokens using the variables from the `context`. +Since the `context` does not provide the variables for the template tokens in `tags`, the following manifest will be +generated: + +```yaml +components: + terraform: + eks/cluster: + metadata: + component: eks/cluster + vars: + enabled: true + name: prod-eks + tags: + atmos_component: + atmos_stack: + terraform_workspace: +``` + +
+ +The second pass of template processing will not replace the tokens in `tags` because they are already processed in the +first pass (importing) and the values `` are generated. + +To deal with this, use **double curly braces + backtick + double curly braces** instead of just **double curly braces** +in `tags` to prevent Atmos from processing the templates in the first pass and instead process them in the second pass: + +```yaml title="stacks/catalog/eks/eks_cluster.tmpl" +components: + terraform: + eks/cluster: + metadata: + component: eks/cluster + vars: + enabled: "{{ .enabled }}" + name: "{{ .name }}" + tags: + atmos_component: "{{`{{ .atmos_component }}`}}" + atmos_stack: "{{`{{ .atmos_stack }}`}}" + terraform_workspace: "{{`{{ .workspace }}`}}" +``` + +
+ +Atmos will first process the import and replace the template tokens using the variables from the `context`. +Then in the second pass the tokens in `tags` will be replaced with the correct values. + +It will generate the following manifest: + +```yaml +components: + terraform: + eks/cluster: + metadata: + component: eks/cluster + vars: + enabled: true + name: prod-eks + tags: + atmos_component: eks/cluster + atmos_stack: plat-ue2-prod + terraform_workspace: plat-ue2-prod +``` diff --git a/website/docs/integrations/atlantis.mdx b/website/docs/integrations/atlantis.mdx index a53346627..33e43372b 100644 --- a/website/docs/integrations/atlantis.mdx +++ b/website/docs/integrations/atlantis.mdx @@ -686,7 +686,7 @@ on: branches: [ main ] env: - ATMOS_VERSION: 1.68.0 + ATMOS_VERSION: 1.70.0 ATMOS_CLI_CONFIG_PATH: ./ jobs: diff --git a/website/docs/integrations/github-actions/setup-atmos.md b/website/docs/integrations/github-actions/setup-atmos.md index 19f03cb81..1adc73ce6 100644 --- a/website/docs/integrations/github-actions/setup-atmos.md +++ b/website/docs/integrations/github-actions/setup-atmos.md @@ -27,5 +27,5 @@ jobs: uses: cloudposse/github-action-setup-atmos with: # Make sure to pin to the latest version of atmos - atmos_version: 1.68.0 + atmos_version: 1.70.0 ``` diff --git a/website/static/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json b/website/static/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json index d7bd762a8..445e9c7f2 100644 --- a/website/static/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json +++ b/website/static/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json @@ -362,6 +362,9 @@ }, "atlantis": { "$ref": "#/definitions/atlantis" + }, + "templates": { + "$ref": "#/definitions/templates" } }, "required": [], @@ -743,6 +746,12 @@ "description": "Providers section", "additionalProperties": true, "title": "providers" + }, + "templates": { + "type": "object", + "description": "Templates section", + "additionalProperties": true, + "title": "templates" } } }