From b7f8e30d4d2d12bdbd537274921628ef42d8eae5 Mon Sep 17 00:00:00 2001 From: Andriy Knysh Date: Sat, 17 Aug 2024 17:07:12 -0400 Subject: [PATCH] Improve logging for the template function `atmos.Component`. Generate backend config and provider override files in `atmos.Component` function (#674) * updates * updates * updates * updates * updates * updates * updates --- examples/quick-start-advanced/Dockerfile | 2 +- go.mod | 4 +- go.sum | 10 +-- internal/exec/template_funcs_component.go | 67 ++++++++++++++++++- website/docs/integrations/atlantis.mdx | 2 +- .../github-actions/setup-atmos.mdx | 2 +- .../docs/quick-start/advanced/provision.md | 4 +- 7 files changed, 78 insertions(+), 13 deletions(-) diff --git a/examples/quick-start-advanced/Dockerfile b/examples/quick-start-advanced/Dockerfile index f43c8e7f8..cc81ab67f 100644 --- a/examples/quick-start-advanced/Dockerfile +++ b/examples/quick-start-advanced/Dockerfile @@ -6,7 +6,7 @@ ARG GEODESIC_OS=debian # https://atmos.tools/ # https://github.com/cloudposse/atmos # https://github.com/cloudposse/atmos/releases -ARG ATMOS_VERSION=1.86.1 +ARG ATMOS_VERSION=1.86.2 # Terraform: https://github.com/hashicorp/terraform/releases ARG TF_VERSION=1.9.4 diff --git a/go.mod b/go.mod index b22dd951a..ad2d695f1 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/zclconf/go-cty v1.15.0 gopkg.in/yaml.v2 v2.4.0 - mvdan.cc/sh/v3 v3.8.0 + mvdan.cc/sh/v3 v3.9.0 ) require ( @@ -237,7 +237,7 @@ require ( golang.org/x/oauth2 v0.20.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.24.0 // indirect - golang.org/x/term v0.22.0 // indirect + golang.org/x/term v0.23.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect diff --git a/go.sum b/go.sum index 49a2837fb..10b7335e5 100644 --- a/go.sum +++ b/go.sum @@ -552,6 +552,8 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -1521,8 +1523,8 @@ golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1906,8 +1908,8 @@ k8s.io/client-go v0.26.2 h1:s1WkVujHX3kTp4Zn4yGNFK+dlDXy1bAAkIl+cFAiuYI= k8s.io/client-go v0.26.2/go.mod h1:u5EjOuSyBa09yqqyY7m3abZeovO/7D/WehVVlZ2qcqU= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -mvdan.cc/sh/v3 v3.8.0 h1:ZxuJipLZwr/HLbASonmXtcvvC9HXY9d2lXZHnKGjFc8= -mvdan.cc/sh/v3 v3.8.0/go.mod h1:w04623xkgBVo7/IUK89E0g8hBykgEpN0vgOj3RJr6MY= +mvdan.cc/sh/v3 v3.9.0 h1:it14fyjCdQUk4jf/aYxLO3FG8jFarR9GzMCtnlvvD7c= +mvdan.cc/sh/v3 v3.9.0/go.mod h1:cdBk8bgoiBI7lSZqK5JhUuq7OB64VQ7fgm85xelw3Nk= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= oras.land/oras-go/v2 v2.3.1 h1:lUC6q8RkeRReANEERLfH86iwGn55lbSWP20egdFHVec= oras.land/oras-go/v2 v2.3.1/go.mod h1:5AQXVEu1X/FKp1F9DMOb5ZItZBOa0y5dha0yCm4NR9c= diff --git a/internal/exec/template_funcs_component.go b/internal/exec/template_funcs_component.go index 0648e1f4d..111568216 100644 --- a/internal/exec/template_funcs_component.go +++ b/internal/exec/template_funcs_component.go @@ -3,6 +3,7 @@ package exec import ( "context" "fmt" + "path" "sync" "github.com/hashicorp/terraform-exec/tfexec" @@ -24,7 +25,6 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st // If the result for the component in the stack already exists in the cache, return it existingSections, found := componentFuncSyncMap.Load(stackSlug) if found && existingSections != nil { - if cliConfig.Logs.Level == u.LogLevelTrace { u.LogTrace(cliConfig, fmt.Sprintf("Found the result of the template function 'atmos.Component(%s, %s)' in the cache", component, stack)) @@ -72,6 +72,57 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'component_info.component_path' section", component, stack) } + // Auto-generate backend file + if cliConfig.Components.Terraform.AutoGenerateBackendFile { + backendFileName := path.Join(componentPath, "backend.tf.json") + + u.LogTrace(cliConfig, "\nWriting the backend config to file:") + u.LogTrace(cliConfig, backendFileName) + + backendTypeSection, ok := sections["backend_type"].(string) + if !ok { + return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'backend_type' section", component, stack) + } + + backendSection, ok := sections["backend"].(map[any]any) + if !ok { + return nil, fmt.Errorf("the component '%s' in the stack '%s' has an invalid 'backend' section", component, stack) + } + + componentBackendConfig, err := generateComponentBackendConfig(backendTypeSection, backendSection, terraformWorkspace) + if err != nil { + return nil, err + } + + err = u.WriteToFileAsJSON(backendFileName, componentBackendConfig, 0644) + if err != nil { + return nil, err + } + + u.LogTrace(cliConfig, "\nWrote the backend config to file:") + u.LogTrace(cliConfig, backendFileName) + } + + // Generate `providers_override.tf.json` file if the `providers` section is configured + providersSection, ok := sections["providers"].(map[any]any) + + if ok && len(providersSection) > 0 { + providerOverrideFileName := path.Join(componentPath, "providers_override.tf.json") + + u.LogTrace(cliConfig, "\nWriting the provider overrides to file:") + u.LogTrace(cliConfig, providerOverrideFileName) + + var providerOverrides = generateComponentProviderOverrides(providersSection) + err = u.WriteToFileAsJSON(providerOverrideFileName, providerOverrides, 0644) + if err != nil { + return nil, err + } + + u.LogTrace(cliConfig, "\nWrote the provider overrides to file:") + u.LogTrace(cliConfig, providerOverrideFileName) + } + + // Initialize Terraform/OpenTofu tf, err := tfexec.NewTerraform(componentPath, executable) if err != nil { return nil, err @@ -79,30 +130,42 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st ctx := context.Background() + // 'terraform init' + u.LogTrace(cliConfig, fmt.Sprintf("\nExecuting 'terraform init %s -s %s'", component, stack)) err = tf.Init(ctx, tfexec.Upgrade(false)) if err != nil { return nil, err } + u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform init %s -s %s'", component, stack)) + // Terraform workspace + u.LogTrace(cliConfig, fmt.Sprintf("\nExecuting 'terraform workspace new %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack)) err = tf.WorkspaceNew(ctx, terraformWorkspace) if err != nil { + u.LogTrace(cliConfig, fmt.Sprintf("\nWorkspace exists. Executing 'terraform workspace select %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack)) err = tf.WorkspaceSelect(ctx, terraformWorkspace) if err != nil { return nil, err } + u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform workspace select %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack)) + } else { + u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform workspace new %s' for component '%s' in stack '%s'", terraformWorkspace, component, stack)) } + // Terraform output + u.LogTrace(cliConfig, fmt.Sprintf("\nExecuting 'terraform output %s -s %s'", component, stack)) outputMeta, err := tf.Output(ctx) if err != nil { return nil, err } + u.LogTrace(cliConfig, fmt.Sprintf("\nExecuted 'terraform output %s -s %s'", component, stack)) if cliConfig.Logs.Level == u.LogLevelTrace { y, err2 := u.ConvertToYAML(outputMeta) if err2 != nil { u.LogError(err2) } else { - u.LogTrace(cliConfig, fmt.Sprintf("\nResult of 'atmos terraform output %s -s %s' before processing it:\n%s\n", component, stack, y)) + u.LogTrace(cliConfig, fmt.Sprintf("\nResult of 'terraform output %s -s %s' before processing it:\n%s\n", component, stack, y)) } } diff --git a/website/docs/integrations/atlantis.mdx b/website/docs/integrations/atlantis.mdx index 229413a11..18797f5d6 100644 --- a/website/docs/integrations/atlantis.mdx +++ b/website/docs/integrations/atlantis.mdx @@ -673,7 +673,7 @@ on: branches: [ main ] env: - ATMOS_VERSION: 1.86.1 + ATMOS_VERSION: 1.86.2 ATMOS_CLI_CONFIG_PATH: ./ jobs: diff --git a/website/docs/integrations/github-actions/setup-atmos.mdx b/website/docs/integrations/github-actions/setup-atmos.mdx index 75054e106..ecef22edb 100644 --- a/website/docs/integrations/github-actions/setup-atmos.mdx +++ b/website/docs/integrations/github-actions/setup-atmos.mdx @@ -33,6 +33,6 @@ jobs: uses: cloudposse/github-action-setup-atmos with: # Make sure to pin to the latest version of atmos - atmos_version: 1.86.1 + atmos_version: 1.86.2 ``` diff --git a/website/docs/quick-start/advanced/provision.md b/website/docs/quick-start/advanced/provision.md index 4cd559b2b..3a2f60486 100644 --- a/website/docs/quick-start/advanced/provision.md +++ b/website/docs/quick-start/advanced/provision.md @@ -5,7 +5,7 @@ sidebar_label: Provision --- Having configured the Terraform components, the Atmos components catalog, all the mixins and defaults, and the Atmos top-level stacks, we can now -provision the components into the stacks. +provision the components in the stacks. The `vpc` Atmos components use the remote state from the `vpc-flow-logs-bucket` components, therefore the `vpc-flow-logs-bucket` components must be provisioned first. @@ -69,7 +69,7 @@ Let's consider what Atmos does when executing the command `atmos terraform apply context variables specified in the command (`-s` flag) with the context variables defined in the stack configurations, finally finding the matching stack -- Atmos finds the component `vpc` in the stack, processing all the inline configs and all the imports +- Atmos finds the component `vpc` in the stack, processing all the inline configs and all the configs from the imports - Atmos deep-merges all the catalog imports for the `vpc` component and then deep-merges all the variables for the component defined in all sections (global `vars`, terraform `vars`, base components `vars`, component `vars`), producing the final variables for the `vpc` component in