From 39d33c722a6d468df72f86d491171a0feba1d4fb Mon Sep 17 00:00:00 2001 From: Andriy Knysh Date: Thu, 17 Aug 2023 14:33:27 -0400 Subject: [PATCH] Update Spacelift deps. Update `atmos validate stacks` command. Update docs (#418) * updates * update docs * updates * updates * updates * updates * updates * updates * updates * updates --- examples/complete/Dockerfile | 4 +- go.mod | 2 +- go.sum | 4 +- internal/exec/describe_dependents.go | 2 +- internal/exec/stack_utils.go | 10 +- pkg/spacelift/spacelift_stack_processor.go | 75 ++++++++++++-- .../spacelift_stack_processor_test.go | 8 +- pkg/stack/stack_processor.go | 48 ++++++--- website/docs/core-concepts/stacks/imports.md | 78 +++++++++++++-- website/docs/integrations/spacelift.md | 98 ++++++++++++++++--- 10 files changed, 281 insertions(+), 48 deletions(-) diff --git a/examples/complete/Dockerfile b/examples/complete/Dockerfile index 3d99b1e95..182e9b116 100644 --- a/examples/complete/Dockerfile +++ b/examples/complete/Dockerfile @@ -3,10 +3,10 @@ ARG GEODESIC_VERSION=2.2.4 ARG GEODESIC_OS=debian # atmos: https://github.com/cloudposse/atmos -ARG ATMOS_VERSION=1.42.0 +ARG ATMOS_VERSION=1.44.0 # Terraform: https://github.com/hashicorp/terraform/releases -ARG TF_VERSION=1.5.3 +ARG TF_VERSION=1.5.5 FROM cloudposse/geodesic:${GEODESIC_VERSION}-${GEODESIC_OS} diff --git a/go.mod b/go.mod index a2a7c8bbc..b22485265 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/hashicorp/go-getter v1.7.2 github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/hcl/v2 v2.17.0 - github.com/hashicorp/terraform-config-inspect v0.0.0-20230614215431-f32df32a01cd + github.com/hashicorp/terraform-config-inspect v0.0.0-20230808231734-f15f31bf62b3 github.com/imdario/mergo v0.3.13 github.com/json-iterator/go v1.1.12 github.com/mitchellh/go-homedir v1.1.0 diff --git a/go.sum b/go.sum index 125626ed4..1b4b73959 100644 --- a/go.sum +++ b/go.sum @@ -442,8 +442,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.17.0 h1:z1XvSUyXd1HP10U4lrLg5e0JMVz6CPaJvAgxM0KNZVY= github.com/hashicorp/hcl/v2 v2.17.0/go.mod h1:gJyW2PTShkJqQBKpAmPO3yxMxIuoXkOF2TpqXzrQyx4= -github.com/hashicorp/terraform-config-inspect v0.0.0-20230614215431-f32df32a01cd h1:1uPcotqoL4TjcGKlgIe7OFSRplf7BMVtUjekwmCrvuM= -github.com/hashicorp/terraform-config-inspect v0.0.0-20230614215431-f32df32a01cd/go.mod h1:l8HcFPm9cQh6Q0KSWoYPiePqMvRFenybP1CH2MjKdlg= +github.com/hashicorp/terraform-config-inspect v0.0.0-20230808231734-f15f31bf62b3 h1:1uP3RA50ayEcTrHJtHaqubpW66KkXKIYXHP1+79dbMc= +github.com/hashicorp/terraform-config-inspect v0.0.0-20230808231734-f15f31bf62b3/go.mod h1:l8HcFPm9cQh6Q0KSWoYPiePqMvRFenybP1CH2MjKdlg= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= diff --git a/internal/exec/describe_dependents.go b/internal/exec/describe_dependents.go index 34618c46f..cdeebb8b9 100644 --- a/internal/exec/describe_dependents.go +++ b/internal/exec/describe_dependents.go @@ -160,7 +160,7 @@ func ExecuteDescribeDependents( return nil, err } - // Skip if the stack component has an empty `settings.dependencies.depends_on` section + // Skip if the stack component has an empty `settings.depends_on` section if reflect.ValueOf(stackComponentSettings).IsZero() || reflect.ValueOf(stackComponentSettings.DependsOn).IsZero() { continue diff --git a/internal/exec/stack_utils.go b/internal/exec/stack_utils.go index aa11b0216..e8d938e2f 100644 --- a/internal/exec/stack_utils.go +++ b/internal/exec/stack_utils.go @@ -93,10 +93,12 @@ func BuildDependentStackNameFromDependsOn( ) (string, error) { var dependentStackName string - if u.SliceContainsString(allStackNames, dependsOn) { - dependentStackName = dependsOn - } else if u.SliceContainsString(componentNamesInCurrentStack, dependsOn) { - dependentStackName = fmt.Sprintf("%s-%s", currentStackName, dependsOn) + dep := strings.Replace(dependsOn, "/", "-", -1) + + if u.SliceContainsString(allStackNames, dep) { + dependentStackName = dep + } else if u.SliceContainsString(componentNamesInCurrentStack, dep) { + dependentStackName = fmt.Sprintf("%s-%s", currentStackName, dep) } else { errorMessage := fmt.Errorf("the component '%[1]s' in the stack '%[2]s' specifies 'depends_on' dependency '%[3]s', "+ "but '%[3]s' is not a stack and not a component in the '%[2]s' stack", diff --git a/pkg/spacelift/spacelift_stack_processor.go b/pkg/spacelift/spacelift_stack_processor.go index 52a689b87..ec6ce36da 100644 --- a/pkg/spacelift/spacelift_stack_processor.go +++ b/pkg/spacelift/spacelift_stack_processor.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/mitchellh/mapstructure" "github.com/pkg/errors" e "github.com/cloudposse/atmos/internal/exec" @@ -63,7 +64,13 @@ func CreateSpaceliftStacks( return nil, err } - return TransformStackConfigToSpaceliftStacks(stacks, stackConfigPathTemplate, cliConfig.Stacks.NamePattern, processImports, rawStackConfigs) + return TransformStackConfigToSpaceliftStacks( + stacks, + stackConfigPathTemplate, + cliConfig.Stacks.NamePattern, + processImports, + rawStackConfigs, + ) } } @@ -129,11 +136,6 @@ func TransformStackConfigToSpaceliftStacks( spaceliftExplicitLabels = i.([]any) } - spaceliftDependsOn := []any{} - if i, ok2 := spaceliftSettings["depends_on"]; ok2 { - spaceliftDependsOn = i.([]any) - } - spaceliftConfig := map[string]any{} spaceliftConfig["enabled"] = spaceliftWorkspaceEnabled @@ -264,9 +266,15 @@ func TransformStackConfigToSpaceliftStacks( terraformComponentNamesInCurrentStack = append(terraformComponentNamesInCurrentStack, strings.Replace(v, "/", "-", -1)) } - for _, v := range spaceliftDependsOn { + // Legacy/deprecated `settings.spacelift.depends_on` + spaceliftDependsOn := []any{} + if i, ok2 := spaceliftSettings["depends_on"]; ok2 { + spaceliftDependsOn = i.([]any) + } + + for _, dep := range spaceliftDependsOn { spaceliftStackNameDependsOn, err := e.BuildDependentStackNameFromDependsOn( - v.(string), + dep.(string), allStackNames, contextPrefix, terraformComponentNamesInCurrentStack, @@ -278,6 +286,57 @@ func TransformStackConfigToSpaceliftStacks( labels = append(labels, fmt.Sprintf("depends-on:%s", spaceliftStackNameDependsOn)) } + // Recommended `settings.depends_on` + var stackComponentSettingsDependsOn schema.Settings + err = mapstructure.Decode(componentSettings, &stackComponentSettingsDependsOn) + if err != nil { + return nil, err + } + + for _, stackComponentSettingsDependsOnContext := range stackComponentSettingsDependsOn.DependsOn { + if stackComponentSettingsDependsOnContext.Namespace == "" { + stackComponentSettingsDependsOnContext.Namespace = context.Namespace + } + if stackComponentSettingsDependsOnContext.Tenant == "" { + stackComponentSettingsDependsOnContext.Tenant = context.Tenant + } + if stackComponentSettingsDependsOnContext.Environment == "" { + stackComponentSettingsDependsOnContext.Environment = context.Environment + } + if stackComponentSettingsDependsOnContext.Stage == "" { + stackComponentSettingsDependsOnContext.Stage = context.Stage + } + + var contextPrefixDependsOn string + + if stackNamePattern != "" { + contextPrefixDependsOn, err = cfg.GetContextPrefix( + stackName, + stackComponentSettingsDependsOnContext, + stackNamePattern, + stackName, + ) + if err != nil { + return nil, err + } + } else { + contextPrefixDependsOn = strings.Replace(stackName, "/", "-", -1) + } + + spaceliftStackNameDependsOn, err := e.BuildDependentStackNameFromDependsOn( + stackComponentSettingsDependsOnContext.Component, + allStackNames, + contextPrefixDependsOn, + terraformComponentNamesInCurrentStack, + component) + if err != nil { + u.LogError(err) + return nil, err + } + labels = append(labels, fmt.Sprintf("depends-on:%s", spaceliftStackNameDependsOn)) + } + + // Add `component` and `folder` labels labels = append(labels, fmt.Sprintf("folder:component/%s", component)) labels = append(labels, fmt.Sprintf("folder:%s", strings.Replace(contextPrefix, "-", "/", -1))) diff --git a/pkg/spacelift/spacelift_stack_processor_test.go b/pkg/spacelift/spacelift_stack_processor_test.go index 23938c7d5..a0b304990 100644 --- a/pkg/spacelift/spacelift_stack_processor_test.go +++ b/pkg/spacelift/spacelift_stack_processor_test.go @@ -1,10 +1,10 @@ package spacelift import ( - "gopkg.in/yaml.v2" "testing" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" ) func TestSpaceliftStackProcessor(t *testing.T) { @@ -91,6 +91,12 @@ func TestSpaceliftStackProcessor(t *testing.T) { newTenant1Ue2Test1TestTestComponentOverrideComponent2InfrastructureStackName := newTenant1Ue2Test1TestTestComponentOverrideComponent2["stack"].(string) assert.Equal(t, "tenant1-ue2-test-1", newTenant1Ue2Test1TestTestComponentOverrideComponent2InfrastructureStackName) + // Test `settings.depends_on` + tenant1Ue2ProdTopLevelComponent1 := spaceliftStacks["tenant1-ue2-prod-top-level-component1"].(map[string]any) + tenant1Ue2ProdTopLevelComponent1Labels := tenant1Ue2ProdTopLevelComponent1["labels"].([]string) + assert.Equal(t, "depends-on:tenant1-ue2-prod-test-test-component-override", tenant1Ue2ProdTopLevelComponent1Labels[35]) + assert.Equal(t, "depends-on:tenant1-ue2-dev-test-test-component", tenant1Ue2ProdTopLevelComponent1Labels[36]) + yamlSpaceliftStacks, err := yaml.Marshal(spaceliftStacks) assert.Nil(t, err) t.Log(string(yamlSpaceliftStacks)) diff --git a/pkg/stack/stack_processor.go b/pkg/stack/stack_processor.go index 5973ae215..ef962c823 100644 --- a/pkg/stack/stack_processor.go +++ b/pkg/stack/stack_processor.go @@ -7,6 +7,8 @@ import ( "sort" "strings" "sync" + "text/template" + "text/template/parse" "github.com/pkg/errors" "gopkg.in/yaml.v2" @@ -211,18 +213,42 @@ func ProcessYAMLConfigFile( if err != nil || len(importMatches) == 0 { // Retry (b/c we are using `doublestar` library and it sometimes has issues reading many files in a Docker container) // TODO: review `doublestar` library + importMatches, err = u.GetGlobMatches(impWithExtPath) - if err != nil { - errorMessage := fmt.Sprintf("no matches found for the import '%s' in the file '%s'\nError: %s", - imp, - relativeFilePath, - err) - return nil, nil, nil, errors.New(errorMessage) - } else if importMatches == nil { - errorMessage := fmt.Sprintf("invalid import in the file '%s'\nNo matches found for the import '%s'", - relativeFilePath, - imp) - return nil, nil, nil, errors.New(errorMessage) + if err != nil || len(importMatches) == 0 { + // The import was not found -> check if the import is a Go template; if not, return the error + t, err2 := template.New(imp).Parse(imp) + if err2 != nil { + return nil, nil, nil, err2 + } + + isGoTemplate := false + + // Iterate over all nodes in the template and check if any of them is of type `NodeAction` (field evaluation) + for _, node := range t.Root.Nodes { + if node.Type() == parse.NodeAction { + isGoTemplate = true + break + } + } + + // If the import is not a Go template, return the error + if !isGoTemplate { + if err != nil { + errorMessage := fmt.Sprintf("no matches found for the import '%s' in the file '%s'\nError: %s", + imp, + relativeFilePath, + err, + ) + return nil, nil, nil, errors.New(errorMessage) + } else if importMatches == nil { + errorMessage := fmt.Sprintf("no matches found for the import '%s' in the file '%s'", + imp, + relativeFilePath, + ) + return nil, nil, nil, errors.New(errorMessage) + } + } } } diff --git a/website/docs/core-concepts/stacks/imports.md b/website/docs/core-concepts/stacks/imports.md index ae791f6cd..7f8151432 100644 --- a/website/docs/core-concepts/stacks/imports.md +++ b/website/docs/core-concepts/stacks/imports.md @@ -51,8 +51,7 @@ Use [mixins](/core-concepts/stacks/mixins) for reusable snippets of configuratio ## Imports Schema -The `import` section supports the following two formats (note that only one format is supported in a stack config file, but different stack -configs can use one format or the other): +The `import` section supports the following two formats: - a list of paths to the imported files, for example: @@ -82,10 +81,35 @@ where: - `context` - an optional freeform map of context variables that are applied as template variables to the imported file (if the imported file is a [Go template](https://pkg.go.dev/text/template)) +
+ +:::note + +In a stack config file, you can use only one of the supported formats. Using the two formats at the same time in a single stack config file is not +currently supported. If you are importing a template file and providing a `context` to it, you have to use the same schema with the `path` to the +files to import all other configs even if they don't use templates and don't require a `context`. + +::: + +For example: + +```yaml + import: + - path: catalog/terraform/eks_cluster_tmpl + context: + flavor: "blue" + enabled: true + service_1_name: "blue-service-1" + service_2_name: "blue-service-2" + - path: catalog/terraform/test-component + - path: catalog/terraform/vpc + - path: catalog/helmfile/echo-server +``` + ## `Go` Templates in Imports -Atmos supports all the functionality of the [Go templates](https://pkg.go.dev/text/template) in imported stack configurations, including -[functions](https://pkg.go.dev/text/template#hdr-Functions) and [Sprig functions](http://masterminds.github.io/sprig/). +Atmos supports all the functionality of [Go templates](https://pkg.go.dev/text/template) in imported stack configurations, including +[functions](https://pkg.go.dev/text/template#hdr-Functions) and [Sprig functions](http://masterminds.github.io/sprig/). Stack configurations can be templatized and then reused with different settings provided via the import `context` section. @@ -93,7 +117,7 @@ For example, we can define the following configuration for EKS Atmos components ```yaml title=stacks/catalog/terraform/eks_cluster_tmpl.yaml # Imports can also be parameterized using `Go` templates -import: [] +import: [ ] components: terraform: @@ -293,7 +317,7 @@ atmos terraform apply eks-blue/cluster -s tenant1-uw1-test-1 atmos terraform apply eks-green/cluster -s tenant1-uw1-test-1 ``` -All the parameterized variables get their values from the all the hierarchical `context` settings: +All the parameterized variables get their values from the hierarchical `context` settings: ```yaml title="atmos describe component eks-blue/cluster -s tenant1-uw1-test-1" vars: @@ -310,6 +334,48 @@ vars: tenant: tenant1 ``` +
+ +## Advanced Examples of Templates in Atmos Configurations + +Atmos supports all the functionality of [Go templates](https://pkg.go.dev/text/template), including [functions](https://pkg.go.dev/text/template#hdr-Functions) and [Sprig functions](http://masterminds.github.io/sprig/). +The Sprig library provides over 70 template functions for `Go's` template language. + +The following example shows how to dynamically include a variable in the Atmos component configuration by using the `hasKey` Sprig function. +The hasKey function returns `true` if the given dictionary contains the given key. + +```yaml +components: + terraform: + eks/iam-role/{{ .app_name }}/{{ .service_environment }}: + metadata: + component: eks/iam-role + settings: + spacelift: + workspace_enabled: true + vars: + enabled: {{ .enabled }} + tags: + Service: {{ .app_name }} + service_account_name: {{ .app_name }} + service_account_namespace: {{ .service_account_namespace }} + {{ if hasKey . "iam_managed_policy_arns" }} + iam_managed_policy_arns: + {{ range $i, $iam_managed_policy_arn := .iam_managed_policy_arns }} + - '{{ $iam_managed_policy_arn }}' + {{ end }} + {{- end }} + {{ if hasKey . "iam_source_policy_documents" }} + iam_source_policy_documents: + {{ range $i, $iam_source_policy_document := .iam_source_policy_documents }} + - '{{ $iam_source_policy_document }}' + {{ end }} + {{- end }} +``` + +The `iam_managed_policy_arns` and `iam_source_policy_documents` variables will be included in the component configuration only if the +provided `context` object has the `iam_managed_policy_arns` and `iam_source_policy_documents` fields. + ## Summary Using imports with context (and hierarchical imports with context) with parameterized config files will help you make the configurations diff --git a/website/docs/integrations/spacelift.md b/website/docs/integrations/spacelift.md index cd53e42cc..2d74d23a5 100644 --- a/website/docs/integrations/spacelift.md +++ b/website/docs/integrations/spacelift.md @@ -8,16 +8,17 @@ Atmos natively supports [Spacelift](https://spacelift.io). This is accomplished the [`cloudposse/terraform-spacelift-cloud-infrastructure-automation`](https://github.com/cloudposse/terraform-spacelift-cloud-infrastructure-automation) terraform module that reads the YAML Stack configurations and produces the Spacelift resources. -Cloud Posse provides two terraform components that implement Spacelift support. +Cloud Posse provides two terraform components for Spacelift support: -- [Terraform Component](/core-concepts/components/) for provising - a [Spacelift Worker Pool](https://github.com/cloudposse/terraform-aws-components/tree/master/modules/spacelift-worker-pool) -- [Terraform Component](/core-concepts/components/) for provisioning - the [Spacelift Stacks](https://github.com/cloudposse/terraform-aws-components/tree/master/modules/spacelift) +- [Terraform Component](/core-concepts/components/) for provisioning a + [Spacelift Worker Pool](https://github.com/cloudposse/terraform-aws-components/tree/main/modules/spacelift/worker-pool) + +- [Terraform Component](/core-concepts/components/) for + provisioning [Spacelift Stacks](https://github.com/cloudposse/terraform-aws-components/tree/main/modules/spacelift/admin-stack) ## Stack Configuration -The Atmos Spacelift Terraform Component supports some `spacelift` specific settings. +Atmos components support the following `spacelift` specific settings: ```yaml components: @@ -25,13 +26,13 @@ components: example: settings: spacelift: - # enable the stack in spacelift - workspace_enabled: true + # enable the stack in Spacelift + workspace_enabled: true administrative: true # auto deploy this stack - autodeploy: true + autodeploy: true # commands to run before init before_init: [] @@ -41,7 +42,7 @@ components: description: Example component - # whether or not to auto destroy resources if the stack is deleted + # whether to auto destroy resources if the stack is deleted stack_destructor_enabled: false worker_pool_name: null @@ -50,9 +51,82 @@ components: policies_enabled: [] # set explicitly below - administrative_trigger_policy_enabled: false + administrative_trigger_policy_enabled: false # policies to enable policies_by_id_enabled: - - trigger-administrative-policy + - trigger-administrative-policy +``` + +
+ +## Spacelift Stack Dependencies + +Atmos supports [Spacelift Stack Dependencies](https://docs.spacelift.io/concepts/stack/stack-dependencies) in component configurations. + +You can define component dependencies by using the `settings.depends_on` section. The section used to define all the Atmos components (in +the same or different stacks) that the current component depends on. + +The `settings.depends_on` section is a map of objects. The map keys are just the descriptions of dependencies and can be strings or numbers. +Provide meaningful descriptions or numbering so that people can understand what the dependencies are about. + +Each object in the `settings.depends_on` section has the following schema: + +- `component` (required) - an Atmos component that the current component depends on +- `namespace` (optional) - the `namespace` where the Atmos component is provisioned +- `tenant` (optional) - the `tenant` where the Atmos component is provisioned +- `environment` (optional) - the `environment` where the Atmos component is provisioned +- `stage` (optional) - the `stage` where the Atmos component is provisioned + +
+ +The `component` attribute is required. The rest are the context variables and are used to define Atmos stacks other than the current stack. +For example, you can specify: + +- `namespace` if the `component` is from a different Organization +- `tenant` if the `component` is from a different Organizational Unit +- `environment` if the `component` is from a different region +- `stage` if the `component` is from a different account +- `tenant`, `environment` and `stage` if the component is from a different Atmos stack (e.g. `tenant1-ue2-dev`) + +
+ +In the following example, we specify that the `top-level-component1` component depends on the following: + +- The `test/test-component-override` component in the same Atmos stack +- The `test/test-component` component in Atmos stacks in the `dev` stage +- The `my-component` component from the `tenant1-ue2-staging` Atmos stack + +```yaml +components: + terraform: + top-level-component1: + settings: + depends_on: + 1: + # If the `context` (namespace, tenant, environment, stage) is not provided, + # the `component` is from the same Atmos stack as this component + component: "test/test-component-override" + 2: + # This component (in any stage) depends on `test/test-component` + # from the `dev` stage (in any `environment` and any `tenant`) + component: "test/test-component" + stage: "dev" + 3: + # This component depends on `my-component` + # from the `tenant1-ue2-staging` Atmos stack + component: "my-component" + tenant: "tenant1" + environment: "ue2" + stage: "staging" + vars: + enabled: true ``` + +
+ +:::tip + +Refer to [`atmos describe dependents` CLI command](/cli/commands/describe/dependents) for more information. + +:::