From 846ced74f85860e84561e2718bf136312d125518 Mon Sep 17 00:00:00 2001 From: Andriy Knysh Date: Mon, 8 Nov 2021 13:05:00 -0500 Subject: [PATCH] Fix `terraform import` command. Add `terraform clean` command. Update spacelift processor (#73) * Use varfile in `terraform import` * Add `terraform clean` command * Add `terraform clean` command * Update spacelift processor * Update spacelift processor * Update spacelift processor * Update spacelift processor * Update spacelift processor * Update spacelift processor * Update spacelift processor * Update spacelift processor * Update spacelift processor * Update spacelift processor * Update spacelift processor * Update spacelift processor * Update spacelift processor * `terraform clean` uses `TF_DATA_DIR` ENV var * `terraform clean` uses `TF_DATA_DIR` ENV var --- internal/exec/helmfile.go | 6 +- internal/exec/terraform.go | 64 +++-- internal/exec/utils.go | 118 ++------ pkg/config/config.go | 88 +++++- pkg/config/utils.go | 153 ++++++++++- pkg/spacelift/atmos.yaml | 50 ++++ pkg/spacelift/spacelift_stack_processor.go | 258 ++++++++++++++++-- .../spacelift_stack_processor_test.go | 57 +++- 8 files changed, 650 insertions(+), 144 deletions(-) create mode 100644 pkg/spacelift/atmos.yaml diff --git a/internal/exec/helmfile.go b/internal/exec/helmfile.go index 9e984b5b2..48c9d24a6 100644 --- a/internal/exec/helmfile.go +++ b/internal/exec/helmfile.go @@ -69,15 +69,15 @@ func ExecuteHelmfile(cmd *cobra.Command, args []string) error { info.SubCommand = "sync" } - context := getContextFromVars(info.ComponentVarsSection) + context := c.GetContextFromVars(info.ComponentVarsSection) // Prepare AWS profile - helmAwsProfile := replaceContextTokens(context, c.Config.Components.Helmfile.HelmAwsProfilePattern) + helmAwsProfile := c.ReplaceContextTokens(context, c.Config.Components.Helmfile.HelmAwsProfilePattern) color.Cyan(fmt.Sprintf("\nUsing AWS_PROFILE=%s\n\n", helmAwsProfile)) // Download kubeconfig by running `aws eks update-kubeconfig` kubeconfigPath := fmt.Sprintf("%s/%s-kubecfg", c.Config.Components.Helmfile.KubeconfigPath, info.ContextPrefix) - clusterName := replaceContextTokens(context, c.Config.Components.Helmfile.ClusterNamePattern) + clusterName := c.ReplaceContextTokens(context, c.Config.Components.Helmfile.ClusterNamePattern) color.Cyan(fmt.Sprintf("Downloading kubeconfig from the cluster '%s' and saving it to %s\n\n", clusterName, kubeconfigPath)) err = execCommand("aws", diff --git a/internal/exec/terraform.go b/internal/exec/terraform.go index e85d66fe0..e8fcc8d07 100644 --- a/internal/exec/terraform.go +++ b/internal/exec/terraform.go @@ -49,6 +49,42 @@ func ExecuteTerraform(cmd *cobra.Command, args []string) error { )) } + varFile := fmt.Sprintf("%s-%s.terraform.tfvars.json", info.ContextPrefix, info.Component) + planFile := fmt.Sprintf("%s-%s.planfile", info.ContextPrefix, info.Component) + + if info.SubCommand == "clean" { + fmt.Println("Deleting '.terraform' folder") + _ = os.RemoveAll(path.Join(componentPath, ".terraform")) + + fmt.Println("Deleting '.terraform.lock.hcl' file") + _ = os.Remove(path.Join(componentPath, ".terraform.lock.hcl")) + + fmt.Println(fmt.Sprintf("Deleting terraform varfile: %s", varFile)) + _ = os.Remove(path.Join(componentPath, varFile)) + + fmt.Println(fmt.Sprintf("Deleting terraform planfile: %s", planFile)) + _ = os.Remove(path.Join(componentPath, planFile)) + + tfDataDir := os.Getenv("TF_DATA_DIR") + if len(tfDataDir) > 0 && tfDataDir != "." && tfDataDir != "/" && tfDataDir != "./" { + color.Cyan("Found ENV var TF_DATA_DIR=%s", tfDataDir) + var userAnswer string + fmt.Println(fmt.Sprintf("Do you want to delete the folder '%s'? (only 'yes' will be accepted to approve)", tfDataDir)) + fmt.Print("Enter a value: ") + count, err := fmt.Scanln(&userAnswer) + if count > 0 && err != nil { + return err + } + if userAnswer == "yes" { + fmt.Println(fmt.Sprintf("Deleting folder '%s'", tfDataDir)) + _ = os.RemoveAll(tfDataDir) + } + } + + fmt.Println() + return nil + } + // Write variables to a file var varFileName, varFileNameFromArg string @@ -66,19 +102,17 @@ func ExecuteTerraform(cmd *cobra.Command, args []string) error { varFileName = varFileNameFromArg } else { if len(info.ComponentFolderPrefix) == 0 { - varFileName = fmt.Sprintf("%s/%s/%s-%s.terraform.tfvars.json", + varFileName = path.Join( c.Config.Components.Terraform.BasePath, finalComponent, - info.ContextPrefix, - info.Component, + varFile, ) } else { - varFileName = fmt.Sprintf("%s/%s/%s/%s-%s.terraform.tfvars.json", + varFileName = path.Join( c.Config.Components.Terraform.BasePath, info.ComponentFolderPrefix, finalComponent, - info.ContextPrefix, - info.Component, + varFile, ) } } @@ -170,9 +204,6 @@ func ExecuteTerraform(cmd *cobra.Command, args []string) error { } fmt.Println(fmt.Sprintf(fmt.Sprintf("Working dir: %s", workingDir))) - planFile := fmt.Sprintf("%s-%s.planfile", info.ContextPrefix, info.Component) - varFile := fmt.Sprintf("%s-%s.terraform.tfvars.json", info.ContextPrefix, info.Component) - var workspaceName string if len(info.BaseComponent) > 0 { workspaceName = fmt.Sprintf("%s-%s", info.ContextPrefix, info.Component) @@ -189,6 +220,9 @@ func ExecuteTerraform(cmd *cobra.Command, args []string) error { case "destroy": allArgsAndFlags = append(allArgsAndFlags, []string{"-var-file", varFile}...) break + case "import": + allArgsAndFlags = append(allArgsAndFlags, []string{"-var-file", varFile}...) + break case "apply": if info.UseTerraformPlan == true { allArgsAndFlags = append(allArgsAndFlags, []string{planFile}...) @@ -201,16 +235,12 @@ func ExecuteTerraform(cmd *cobra.Command, args []string) error { allArgsAndFlags = append(allArgsAndFlags, info.AdditionalArgsAndFlags...) // Run `terraform workspace` - if info.SubCommand != "clean" { - err = execCommand(info.Command, []string{"workspace", "select", workspaceName}, componentPath, nil) + err = execCommand(info.Command, []string{"workspace", "select", workspaceName}, componentPath, nil) + if err != nil { + err = execCommand(info.Command, []string{"workspace", "new", workspaceName}, componentPath, nil) if err != nil { - err = execCommand(info.Command, []string{"workspace", "new", workspaceName}, componentPath, nil) - if err != nil { - return err - } + return err } - } else { - return errors.New("terraform clean not implemented") } // Check if the terraform command requires a user interaction, diff --git a/internal/exec/utils.go b/internal/exec/utils.go index 28db6bcca..3183b6557 100644 --- a/internal/exec/utils.go +++ b/internal/exec/utils.go @@ -3,7 +3,7 @@ package exec import ( "errors" "fmt" - "github.com/cloudposse/atmos/pkg/config" + c "github.com/cloudposse/atmos/pkg/config" g "github.com/cloudposse/atmos/pkg/globals" s "github.com/cloudposse/atmos/pkg/stack" "github.com/cloudposse/atmos/pkg/utils" @@ -91,8 +91,8 @@ func findComponentConfig( } // processConfigAndStacks processes CLI config and stacks -func processConfigAndStacks(componentType string, cmd *cobra.Command, args []string) (config.ConfigAndStacksInfo, error) { - var configAndStacksInfo config.ConfigAndStacksInfo +func processConfigAndStacks(componentType string, cmd *cobra.Command, args []string) (c.ConfigAndStacksInfo, error) { + var configAndStacksInfo c.ConfigAndStacksInfo if len(args) < 1 { return configAndStacksInfo, errors.New("invalid number of arguments") @@ -134,20 +134,20 @@ func processConfigAndStacks(componentType string, cmd *cobra.Command, args []str } // Process and merge CLI configurations - err = config.InitConfig() + err = c.InitConfig() if err != nil { return configAndStacksInfo, err } - err = config.ProcessConfig(configAndStacksInfo) + err = c.ProcessConfig(configAndStacksInfo) if err != nil { return configAndStacksInfo, err } // Process stack config file(s) _, stacksMap, err := s.ProcessYAMLConfigFiles( - config.ProcessedConfig.StacksBaseAbsolutePath, - config.ProcessedConfig.StackConfigFilesAbsolutePaths, + c.ProcessedConfig.StacksBaseAbsolutePath, + c.ProcessedConfig.StackConfigFilesAbsolutePaths, false, true) @@ -159,27 +159,27 @@ func processConfigAndStacks(componentType string, cmd *cobra.Command, args []str if g.LogVerbose { fmt.Println() var msg string - if config.ProcessedConfig.StackType == "Directory" { + if c.ProcessedConfig.StackType == "Directory" { msg = "Found the config file for the provided stack:" } else { msg = "Found config files:" } color.Cyan(msg) - err = utils.PrintAsYAML(config.ProcessedConfig.StackConfigFilesRelativePaths) + err = utils.PrintAsYAML(c.ProcessedConfig.StackConfigFilesRelativePaths) if err != nil { return configAndStacksInfo, err } } - if len(config.Config.Stacks.NamePattern) < 1 { + if len(c.Config.Stacks.NamePattern) < 1 { return configAndStacksInfo, errors.New("stack name pattern must be provided in 'stacks.name_pattern' config or 'ATMOS_STACKS_NAME_PATTERN' ENV variable") } - stackNamePatternParts := strings.Split(config.Config.Stacks.NamePattern, "-") + stackNamePatternParts := strings.Split(c.Config.Stacks.NamePattern, "-") // Check and process stacks - if config.ProcessedConfig.StackType == "Directory" { + if c.ProcessedConfig.StackType == "Directory" { _, configAndStacksInfo.ComponentVarsSection, configAndStacksInfo.ComponentBackendSection, configAndStacksInfo.ComponentBackendType, @@ -198,7 +198,7 @@ func processConfigAndStacks(componentType string, cmd *cobra.Command, args []str return configAndStacksInfo, errors.New(fmt.Sprintf("Stack '%s' does not match the stack name pattern '%s'", configAndStacksInfo.Stack, - config.Config.Stacks.NamePattern)) + c.Config.Stacks.NamePattern)) } var tenant string @@ -269,7 +269,7 @@ func processConfigAndStacks(componentType string, cmd *cobra.Command, args []str "Are the component and stack names correct? Did you forget an import?", configAndStacksInfo.ComponentFromArg, configAndStacksInfo.Stack, - config.Config.Stacks.NamePattern, + c.Config.Stacks.NamePattern, )) } } @@ -310,59 +310,18 @@ func processConfigAndStacks(componentType string, cmd *cobra.Command, args []str } // Process context - configAndStacksInfo.Context = getContextFromVars(configAndStacksInfo.ComponentVarsSection) - contextPrefix := "" - - for _, part := range stackNamePatternParts { - if part == "{tenant}" { - if len(configAndStacksInfo.Context.Tenant) == 0 { - return configAndStacksInfo, - errors.New(fmt.Sprintf("The stack name pattern '%s' specifies 'tenant`, but the stack %s does not have a tenant defined", - config.Config.Stacks.NamePattern, - configAndStacksInfo.Stack, - )) - } - if len(contextPrefix) == 0 { - contextPrefix = configAndStacksInfo.Context.Tenant - } else { - contextPrefix = contextPrefix + "-" + configAndStacksInfo.Context.Tenant - } - } else if part == "{environment}" { - if len(configAndStacksInfo.Context.Environment) == 0 { - return configAndStacksInfo, - errors.New(fmt.Sprintf("The stack name pattern '%s' specifies 'environment`, but the stack %s does not have an environment defined", - config.Config.Stacks.NamePattern, - configAndStacksInfo.Stack, - )) - } - if len(contextPrefix) == 0 { - contextPrefix = configAndStacksInfo.Context.Environment - } else { - contextPrefix = contextPrefix + "-" + configAndStacksInfo.Context.Environment - } - } else if part == "{stage}" { - if len(configAndStacksInfo.Context.Stage) == 0 { - return configAndStacksInfo, - errors.New(fmt.Sprintf("The stack name pattern '%s' specifies 'stage`, but the stack %s does not have a stage defined", - config.Config.Stacks.NamePattern, - configAndStacksInfo.Stack, - )) - } - if len(contextPrefix) == 0 { - contextPrefix = configAndStacksInfo.Context.Stage - } else { - contextPrefix = contextPrefix + "-" + configAndStacksInfo.Context.Stage - } - } + configAndStacksInfo.Context = c.GetContextFromVars(configAndStacksInfo.ComponentVarsSection) + configAndStacksInfo.ContextPrefix, err = c.GetContextPrefix(configAndStacksInfo.Stack, configAndStacksInfo.Context, c.Config.Stacks.NamePattern) + if err != nil { + return configAndStacksInfo, err } - configAndStacksInfo.ContextPrefix = contextPrefix return configAndStacksInfo, nil } // processArgsAndFlags removes common args and flags from the provided list of arguments/flags -func processArgsAndFlags(inputArgsAndFlags []string) (config.ArgsAndFlagsInfo, error) { - var info config.ArgsAndFlagsInfo +func processArgsAndFlags(inputArgsAndFlags []string) (c.ArgsAndFlagsInfo, error) { + var info c.ArgsAndFlagsInfo var additionalArgsAndFlags []string var globalOptions []string @@ -515,43 +474,6 @@ func execCommand(command string, args []string, dir string, env []string) error return cmd.Run() } -func getContextFromVars(vars map[interface{}]interface{}) config.Context { - var context config.Context - - if namespace, ok := vars["namespace"].(string); ok { - context.Namespace = namespace - } - - if tenant, ok := vars["tenant"].(string); ok { - context.Tenant = tenant - } - - if environment, ok := vars["environment"].(string); ok { - context.Environment = environment - } - - if stage, ok := vars["stage"].(string); ok { - context.Stage = stage - } - - if region, ok := vars["region"].(string); ok { - context.Region = region - } - - return context -} - -func replaceContextTokens(context config.Context, pattern string) string { - return strings.Replace( - strings.Replace( - strings.Replace( - strings.Replace(pattern, - "{namespace}", context.Namespace, 1), - "{environment}", context.Environment, 1), - "{tenant}", context.Tenant, 1), - "{stage}", context.Stage, 1) -} - func generateComponentBackendConfig(backendType string, backendConfig map[interface{}]interface{}) map[string]interface{} { return map[string]interface{}{ "terraform": map[string]interface{}{ diff --git a/pkg/config/config.go b/pkg/config/config.go index 39b47e49b..e4e8a68f1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -5,7 +5,7 @@ import ( "encoding/json" "fmt" g "github.com/cloudposse/atmos/pkg/globals" - "github.com/cloudposse/atmos/pkg/utils" + u "github.com/cloudposse/atmos/pkg/utils" "github.com/fatih/color" "github.com/mitchellh/go-homedir" "github.com/pkg/errors" @@ -209,14 +209,14 @@ func ProcessConfig(configAndStacksInfo ConfigAndStacksInfo) error { ProcessedConfig.StacksBaseAbsolutePath = stacksBaseAbsPath // Convert the included stack paths to absolute paths - includeStackAbsPaths, err := utils.JoinAbsolutePathWithPaths(stacksBaseAbsPath, Config.Stacks.IncludedPaths) + includeStackAbsPaths, err := u.JoinAbsolutePathWithPaths(stacksBaseAbsPath, Config.Stacks.IncludedPaths) if err != nil { return err } ProcessedConfig.IncludeStackAbsolutePaths = includeStackAbsPaths // Convert the excluded stack paths to absolute paths - excludeStackAbsPaths, err := utils.JoinAbsolutePathWithPaths(stacksBaseAbsPath, Config.Stacks.ExcludedPaths) + excludeStackAbsPaths, err := u.JoinAbsolutePathWithPaths(stacksBaseAbsPath, Config.Stacks.ExcludedPaths) if err != nil { return err } @@ -237,7 +237,7 @@ func ProcessConfig(configAndStacksInfo ConfigAndStacksInfo) error { ProcessedConfig.HelmfileDirAbsolutePath = helmfileDirAbsPath // If the specified stack name is a logical name, find all stack config files in the provided paths - stackConfigFilesAbsolutePaths, stackConfigFilesRelativePaths, stackIsPhysicalPath, err := findAllStackConfigsInPaths( + stackConfigFilesAbsolutePaths, stackConfigFilesRelativePaths, stackIsPhysicalPath, err := findAllStackConfigsInPathsForStack( configAndStacksInfo.Stack, includeStackAbsPaths, excludeStackAbsPaths, @@ -294,7 +294,7 @@ func ProcessConfig(configAndStacksInfo ConfigAndStacksInfo) error { if g.LogVerbose { color.Cyan("\nFinal CLI configuration:") - err = utils.PrintAsYAML(Config) + err = u.PrintAsYAML(Config) if err != nil { return err } @@ -303,11 +303,87 @@ func ProcessConfig(configAndStacksInfo ConfigAndStacksInfo) error { return nil } +// ProcessConfigForSpacelift processes config for Spacelift +func ProcessConfigForSpacelift() error { + // Process ENV vars + err := processEnvVars() + if err != nil { + return err + } + + // Check config + err = checkConfig() + if err != nil { + return err + } + + // Convert stacks base path to absolute path + stacksBaseAbsPath, err := filepath.Abs(Config.Stacks.BasePath) + if err != nil { + return err + } + ProcessedConfig.StacksBaseAbsolutePath = stacksBaseAbsPath + + // Convert the included stack paths to absolute paths + includeStackAbsPaths, err := u.JoinAbsolutePathWithPaths(stacksBaseAbsPath, Config.Stacks.IncludedPaths) + if err != nil { + return err + } + ProcessedConfig.IncludeStackAbsolutePaths = includeStackAbsPaths + + // Convert the excluded stack paths to absolute paths + excludeStackAbsPaths, err := u.JoinAbsolutePathWithPaths(stacksBaseAbsPath, Config.Stacks.ExcludedPaths) + if err != nil { + return err + } + ProcessedConfig.ExcludeStackAbsolutePaths = excludeStackAbsPaths + + // Convert terraform dir to absolute path + terraformDirAbsPath, err := filepath.Abs(Config.Components.Terraform.BasePath) + if err != nil { + return err + } + ProcessedConfig.TerraformDirAbsolutePath = terraformDirAbsPath + + // Convert helmfile dir to absolute path + helmfileDirAbsPath, err := filepath.Abs(Config.Components.Helmfile.BasePath) + if err != nil { + return err + } + ProcessedConfig.HelmfileDirAbsolutePath = helmfileDirAbsPath + + // If the specified stack name is a logical name, find all stack config files in the provided paths + stackConfigFilesAbsolutePaths, stackConfigFilesRelativePaths, err := findAllStackConfigsInPaths( + includeStackAbsPaths, + excludeStackAbsPaths, + ) + + if err != nil { + return err + } + + if len(stackConfigFilesAbsolutePaths) < 1 { + j, err := yaml.Marshal(includeStackAbsPaths) + if err != nil { + return err + } + errorMessage := fmt.Sprintf("\nNo stack config files found in the provided "+ + "paths:\n%s\n\nCheck if 'stacks.base_path', 'stacks.included_paths' and 'stacks.excluded_paths' are correctly set in CLI config "+ + "files or ENV vars.", j) + return errors.New(errorMessage) + } + + ProcessedConfig.StackConfigFilesAbsolutePaths = stackConfigFilesAbsolutePaths + ProcessedConfig.StackConfigFilesRelativePaths = stackConfigFilesRelativePaths + + return nil +} + // https://github.com/NCAR/go-figure // https://github.com/spf13/viper/issues/181 // https://medium.com/@bnprashanth256/reading-configuration-files-and-environment-variables-in-go-golang-c2607f912b63 func processConfigFile(path string, v *viper.Viper) error { - if !utils.FileExists(path) { + if !u.FileExists(path) { if g.LogVerbose { fmt.Println(fmt.Sprintf("No config found in %s", path)) } diff --git a/pkg/config/utils.go b/pkg/config/utils.go index d69507321..f63e837d8 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -2,6 +2,7 @@ package config import ( "errors" + "fmt" "github.com/bmatcuk/doublestar/v4" g "github.com/cloudposse/atmos/pkg/globals" s "github.com/cloudposse/atmos/pkg/stack" @@ -13,8 +14,8 @@ import ( "strings" ) -// findAllStackConfigsInPaths finds all stack config files in the paths specified by globs -func findAllStackConfigsInPaths( +// findAllStackConfigsInPathsForStack finds all stack config files in the paths specified by globs for the provided stack +func findAllStackConfigsInPathsForStack( stack string, includeStackPaths []string, excludeStackPaths []string, @@ -87,6 +88,59 @@ func findAllStackConfigsInPaths( return absolutePaths, relativePaths, false, nil } +// findAllStackConfigsInPaths finds all stack config files in the paths specified by globs +func findAllStackConfigsInPaths( + includeStackPaths []string, + excludeStackPaths []string, +) ([]string, []string, error) { + + var absolutePaths []string + var relativePaths []string + + for _, p := range includeStackPaths { + pathWithExt := p + + ext := filepath.Ext(p) + if ext == "" { + ext = g.DefaultStackConfigFileExtension + pathWithExt = p + ext + } + + // Find all matches in the glob + matches, err := s.GetGlobMatches(pathWithExt) + if err != nil { + return nil, nil, err + } + + // Exclude files that match any of the excludePaths + if matches != nil && len(matches) > 0 { + for _, matchedFileAbsolutePath := range matches { + matchedFileRelativePath := u.TrimBasePathFromPath(ProcessedConfig.StacksBaseAbsolutePath+"/", matchedFileAbsolutePath) + include := true + + for _, excludePath := range excludeStackPaths { + excludeMatch, err := doublestar.PathMatch(excludePath, matchedFileAbsolutePath) + if err != nil { + color.Red("%s", err) + include = false + continue + } else if excludeMatch { + include = false + continue + } + } + + if include == true { + absolutePaths = append(absolutePaths, matchedFileAbsolutePath) + relativePaths = append(relativePaths, matchedFileRelativePath) + } + } + } + } + + return absolutePaths, relativePaths, nil +} + func processEnvVars() error { stacksBasePath := os.Getenv("ATMOS_STACKS_BASE_PATH") if len(stacksBasePath) > 0 { @@ -200,3 +254,98 @@ func processLogsConfig() error { } return nil } + +// GetContextFromVars creates a context object from the provided variables +func GetContextFromVars(vars map[interface{}]interface{}) Context { + var context Context + + if namespace, ok := vars["namespace"].(string); ok { + context.Namespace = namespace + } + + if tenant, ok := vars["tenant"].(string); ok { + context.Tenant = tenant + } + + if environment, ok := vars["environment"].(string); ok { + context.Environment = environment + } + + if stage, ok := vars["stage"].(string); ok { + context.Stage = stage + } + + if region, ok := vars["region"].(string); ok { + context.Region = region + } + + return context +} + +// GetContextPrefix calculates context prefix +func GetContextPrefix(stack string, context Context, stackNamePattern string) (string, error) { + if len(stackNamePattern) == 0 { + return "", + errors.New(fmt.Sprintf("Stack name pattern must be provided")) + } + + contextPrefix := "" + stackNamePatternParts := strings.Split(stackNamePattern, "-") + + for _, part := range stackNamePatternParts { + if part == "{tenant}" { + if len(context.Tenant) == 0 { + return "", + errors.New(fmt.Sprintf("The stack name pattern '%s' specifies 'tenant`, but the stack %s does not have a tenant defined", + stackNamePattern, + stack, + )) + } + if len(contextPrefix) == 0 { + contextPrefix = context.Tenant + } else { + contextPrefix = contextPrefix + "-" + context.Tenant + } + } else if part == "{environment}" { + if len(context.Environment) == 0 { + return "", + errors.New(fmt.Sprintf("The stack name pattern '%s' specifies 'environment`, but the stack %s does not have an environment defined", + stackNamePattern, + stack, + )) + } + if len(contextPrefix) == 0 { + contextPrefix = context.Environment + } else { + contextPrefix = contextPrefix + "-" + context.Environment + } + } else if part == "{stage}" { + if len(context.Stage) == 0 { + return "", + errors.New(fmt.Sprintf("The stack name pattern '%s' specifies 'stage`, but the stack %s does not have a stage defined", + Config.Stacks.NamePattern, + stack, + )) + } + if len(contextPrefix) == 0 { + contextPrefix = context.Stage + } else { + contextPrefix = contextPrefix + "-" + context.Stage + } + } + } + + return contextPrefix, nil +} + +// ReplaceContextTokens replaces tokens in the context pattern +func ReplaceContextTokens(context Context, pattern string) string { + return strings.Replace( + strings.Replace( + strings.Replace( + strings.Replace(pattern, + "{namespace}", context.Namespace, 1), + "{environment}", context.Environment, 1), + "{tenant}", context.Tenant, 1), + "{stage}", context.Stage, 1) +} diff --git a/pkg/spacelift/atmos.yaml b/pkg/spacelift/atmos.yaml new file mode 100644 index 000000000..3a4cf34d9 --- /dev/null +++ b/pkg/spacelift/atmos.yaml @@ -0,0 +1,50 @@ +# CLI config is loaded from the following locations (from lowest to highest priority): +# system dir (`/usr/local/etc/atmos` on Linux, `%LOCALAPPDATA%/atmos` on Windows) +# home dir (~/.atmos) +# current directory +# ENV vars +# Command-line arguments +# +# It supports POSIX-style Globs for file names/paths (double-star `**` is supported) +# https://en.wikipedia.org/wiki/Glob_(programming) + +components: + terraform: + # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_BASE_PATH` ENV var, or `--terraform-dir` command-line argument + # Supports both absolute and relative paths + base_path: "../../examples/complete/components/terraform" + # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_APPLY_AUTO_APPROVE` ENV var + apply_auto_approve: false + # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_DEPLOY_RUN_INIT` ENV var, or `--deploy-run-init` command-line argument + deploy_run_init: true + # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_AUTO_GENERATE_BACKEND_FILE` ENV var, or `--auto-generate-backend-file` command-line argument + auto_generate_backend_file: false + helmfile: + # Can also be set using `ATMOS_COMPONENTS_HELMFILE_BASE_PATH` ENV var, or `--helmfile-dir` command-line argument + # Supports both absolute and relative paths + base_path: "../../examples/complete/components/helmfile" + # Can also be set using `ATMOS_COMPONENTS_HELMFILE_KUBECONFIG_PATH` ENV var + kubeconfig_path: "/dev/shm" + # Can also be set using `ATMOS_COMPONENTS_HELMFILE_HELM_AWS_PROFILE_PATTERN` ENV var + helm_aws_profile_pattern: "{namespace}-{tenant}-gbl-{stage}-helm" + # Can also be set using `ATMOS_COMPONENTS_HELMFILE_CLUSTER_NAME_PATTERN` ENV var + cluster_name_pattern: "{namespace}-{tenant}-{environment}-{stage}-eks-cluster" + +stacks: + # Can also be set using `ATMOS_STACKS_BASE_PATH` ENV var, or `--config-dir` and `--stacks-dir` command-line arguments + # Supports both absolute and relative paths + base_path: "../../examples/complete/stacks" + # Can also be set using `ATMOS_STACKS_INCLUDED_PATHS` ENV var (comma-separated values string) + included_paths: + - "**/*" + # Can also be set using `ATMOS_STACKS_EXCLUDED_PATHS` ENV var (comma-separated values string) + excluded_paths: + - "globals/**/*" + - "catalog/**/*" + - "**/*globals*" + # Can also be set using `ATMOS_STACKS_NAME_PATTERN` ENV var + name_pattern: "{tenant}-{environment}-{stage}" + +logs: + verbose: false + colors: true diff --git a/pkg/spacelift/spacelift_stack_processor.go b/pkg/spacelift/spacelift_stack_processor.go index a42cf066e..4653babb7 100644 --- a/pkg/spacelift/spacelift_stack_processor.go +++ b/pkg/spacelift/spacelift_stack_processor.go @@ -2,8 +2,9 @@ package spacelift import ( "fmt" + c "github.com/cloudposse/atmos/pkg/config" s "github.com/cloudposse/atmos/pkg/stack" - "github.com/cloudposse/atmos/pkg/utils" + u "github.com/cloudposse/atmos/pkg/utils" "github.com/pkg/errors" "strings" ) @@ -17,15 +18,34 @@ func CreateSpaceliftStacks( processComponentDeps bool, processImports bool, stackConfigPathTemplate string) (map[string]interface{}, error) { - var _, mapResult, err = s.ProcessYAMLConfigFiles(basePath, filePaths, processStackDeps, processComponentDeps) - if err != nil { - return nil, err + + if filePaths != nil && len(filePaths) > 0 { + _, stacks, err := s.ProcessYAMLConfigFiles(basePath, filePaths, processStackDeps, processComponentDeps) + if err != nil { + return nil, err + } + + return LegacyTransformStackConfigToSpaceliftStacks(stacks, stackConfigPathTemplate, processImports) + } else { + err := c.InitConfig() + if err != nil { + return nil, err + } + err = c.ProcessConfigForSpacelift() + if err != nil { + return nil, err + } + _, stacks, err := s.ProcessYAMLConfigFiles(c.ProcessedConfig.StacksBaseAbsolutePath, c.ProcessedConfig.StackConfigFilesAbsolutePaths, processStackDeps, processComponentDeps) + if err != nil { + return nil, err + } + + return TransformStackConfigToSpaceliftStacks(stacks, stackConfigPathTemplate, c.Config.Stacks.NamePattern, processImports) } - return TransformStackConfigToSpaceliftStacks(mapResult, stackConfigPathTemplate, processImports) } -// TransformStackConfigToSpaceliftStacks takes a map of stack configs and transforms it to a map of Spacelift stacks -func TransformStackConfigToSpaceliftStacks( +// LegacyTransformStackConfigToSpaceliftStacks takes a map of stack configs and transforms it to a map of Spacelift stacks +func LegacyTransformStackConfigToSpaceliftStacks( stacks map[string]interface{}, stackConfigPathTemplate string, processImports bool) (map[string]interface{}, error) { @@ -65,7 +85,7 @@ func TransformStackConfigToSpaceliftStacks( if terraformComponents, ok := componentsSection["terraform"]; ok { terraformComponentsMap := terraformComponents.(map[string]interface{}) - terraformComponentNamesInCurrentStack := utils.StringKeysFromMap(terraformComponentsMap) + terraformComponentNamesInCurrentStack := u.StringKeysFromMap(terraformComponentsMap) for component, v := range terraformComponentsMap { componentMap := v.(map[string]interface{}) @@ -162,7 +182,7 @@ func TransformStackConfigToSpaceliftStacks( spaceliftConfig["workspace"] = strings.Replace(workspace, "/", "-", -1) // labels - labels := []string{} + var labels []string for _, v := range imports { labels = append(labels, fmt.Sprintf("import:"+stackConfigPathTemplate, v)) } @@ -190,13 +210,14 @@ func TransformStackConfigToSpaceliftStacks( labels = append(labels, fmt.Sprintf("folder:component/%s", component)) - stackFolder := stackName - if !strings.Contains(stackName, "/") { - stackFolder = strings.Replace(stackName, "-", "/", -1) + // Split on the first `-` and get the two parts: environment and stage + stackNameParts := strings.SplitN(stackName, "-", 2) + stackNamePartsLen := len(stackNameParts) + if stackNamePartsLen == 2 { + labels = append(labels, fmt.Sprintf("folder:%s/%s", stackNameParts[0], stackNameParts[1])) } - labels = append(labels, fmt.Sprintf("folder:%s", stackFolder)) - spaceliftConfig["labels"] = utils.UniqueStrings(labels) + spaceliftConfig["labels"] = u.UniqueStrings(labels) // Add Spacelift stack config to the final map spaceliftStackName := strings.Replace(fmt.Sprintf("%s-%s", stackName, component), "/", "-", -1) @@ -209,6 +230,211 @@ func TransformStackConfigToSpaceliftStacks( return res, nil } +// TransformStackConfigToSpaceliftStacks takes a map of stack configs and transforms it to a map of Spacelift stacks +func TransformStackConfigToSpaceliftStacks( + stacks map[string]interface{}, + stackConfigPathTemplate string, + stackNamePattern string, + processImports bool) (map[string]interface{}, error) { + + res := map[string]interface{}{} + + var allStackNames []string + for stackName, stackConfig := range stacks { + config := stackConfig.(map[interface{}]interface{}) + + if i, ok := config["components"]; ok { + componentsSection := i.(map[string]interface{}) + + if terraformComponents, ok := componentsSection["terraform"]; ok { + terraformComponentsMap := terraformComponents.(map[string]interface{}) + + for component, v := range terraformComponentsMap { + componentMap := v.(map[string]interface{}) + componentVars := map[interface{}]interface{}{} + if i, ok2 := componentMap["vars"]; ok2 { + componentVars = i.(map[interface{}]interface{}) + } + context := c.GetContextFromVars(componentVars) + contextPrefix, err := c.GetContextPrefix(stackName, context, stackNamePattern) + if err != nil { + return nil, err + } + + spaceliftStackName := strings.Replace(fmt.Sprintf("%s-%s", contextPrefix, component), "/", "-", -1) + allStackNames = append(allStackNames, spaceliftStackName) + } + } + } + } + + for stackName, stackConfig := range stacks { + config := stackConfig.(map[interface{}]interface{}) + var imports []string + + if processImports == true { + if i, ok := config["imports"]; ok { + imports = i.([]string) + } + } + + if i, ok := config["components"]; ok { + componentsSection := i.(map[string]interface{}) + + if terraformComponents, ok := componentsSection["terraform"]; ok { + terraformComponentsMap := terraformComponents.(map[string]interface{}) + + for component, v := range terraformComponentsMap { + componentMap := v.(map[string]interface{}) + + componentSettings := map[interface{}]interface{}{} + if i, ok2 := componentMap["settings"]; ok2 { + componentSettings = i.(map[interface{}]interface{}) + } + + spaceliftSettings := map[interface{}]interface{}{} + spaceliftWorkspaceEnabled := false + + if i, ok2 := componentSettings["spacelift"]; ok2 { + spaceliftSettings = i.(map[interface{}]interface{}) + + if i3, ok3 := spaceliftSettings["workspace_enabled"]; ok3 { + spaceliftWorkspaceEnabled = i3.(bool) + } + } + + // If Spacelift workspace is disabled, don't include it, continue to the next component + if spaceliftWorkspaceEnabled == false { + continue + } + + spaceliftExplicitLabels := []interface{}{} + if i, ok2 := spaceliftSettings["labels"]; ok2 { + spaceliftExplicitLabels = i.([]interface{}) + } + + spaceliftDependsOn := []interface{}{} + if i, ok2 := spaceliftSettings["depends_on"]; ok2 { + spaceliftDependsOn = i.([]interface{}) + } + + spaceliftConfig := map[string]interface{}{} + spaceliftConfig["enabled"] = spaceliftWorkspaceEnabled + + componentVars := map[interface{}]interface{}{} + if i, ok2 := componentMap["vars"]; ok2 { + componentVars = i.(map[interface{}]interface{}) + } + + componentEnv := map[interface{}]interface{}{} + if i, ok2 := componentMap["env"]; ok2 { + componentEnv = i.(map[interface{}]interface{}) + } + + componentDeps := []string{} + if i, ok2 := componentMap["deps"]; ok2 { + componentDeps = i.([]string) + } + + componentStacks := []string{} + if i, ok2 := componentMap["stacks"]; ok2 { + componentStacks = i.([]string) + } + + context := c.GetContextFromVars(componentVars) + contextPrefix, err := c.GetContextPrefix(stackName, context, stackNamePattern) + if err != nil { + return nil, err + } + + spaceliftStackName := strings.Replace(fmt.Sprintf("%s-%s", contextPrefix, component), "/", "-", -1) + + spaceliftConfig["component"] = component + spaceliftConfig["stack"] = spaceliftStackName + spaceliftConfig["imports"] = imports + spaceliftConfig["vars"] = componentVars + spaceliftConfig["settings"] = componentSettings + spaceliftConfig["env"] = componentEnv + spaceliftConfig["deps"] = componentDeps + spaceliftConfig["stacks"] = componentStacks + + baseComponentName := "" + if baseComponent, baseComponentExist := componentMap["component"]; baseComponentExist { + baseComponentName = baseComponent.(string) + } + spaceliftConfig["base_component"] = baseComponentName + + // backend + backendTypeName := "" + if backendType, backendTypeExist := componentMap["backend_type"]; backendTypeExist { + backendTypeName = backendType.(string) + } + spaceliftConfig["backend_type"] = backendTypeName + + componentBackend := map[interface{}]interface{}{} + if i, ok2 := componentMap["backend"]; ok2 { + componentBackend = i.(map[interface{}]interface{}) + } + spaceliftConfig["backend"] = componentBackend + + // workspace + var workspace string + if backendTypeName == "s3" && baseComponentName == "" { + workspace = contextPrefix + } else { + workspace = fmt.Sprintf("%s-%s", contextPrefix, component) + } + spaceliftConfig["workspace"] = strings.Replace(workspace, "/", "-", -1) + + // labels + labels := []string{} + for _, v := range imports { + labels = append(labels, fmt.Sprintf("import:"+stackConfigPathTemplate, v)) + } + for _, v := range componentStacks { + labels = append(labels, fmt.Sprintf("stack:"+stackConfigPathTemplate, v)) + } + for _, v := range componentDeps { + labels = append(labels, fmt.Sprintf("deps:"+stackConfigPathTemplate, v)) + } + for _, v := range spaceliftExplicitLabels { + labels = append(labels, v.(string)) + } + + var terraformComponentNamesInCurrentStack []string + + for v := range terraformComponentsMap { + terraformComponentNamesInCurrentStack = append(terraformComponentNamesInCurrentStack, strings.Replace(v, "/", "-", -1)) + } + + for _, v := range spaceliftDependsOn { + spaceliftStackNameDependsOn, err := buildSpaceliftDependsOnStackName( + v.(string), + allStackNames, + contextPrefix, + terraformComponentNamesInCurrentStack, + component) + if err != nil { + return nil, err + } + labels = append(labels, fmt.Sprintf("depends-on:%s", spaceliftStackNameDependsOn)) + } + + labels = append(labels, fmt.Sprintf("folder:component/%s", component)) + labels = append(labels, fmt.Sprintf("folder:%s", strings.Replace(contextPrefix, "-", "/", -1))) + + spaceliftConfig["labels"] = u.UniqueStrings(labels) + + // Add Spacelift stack config to the final map + res[spaceliftStackName] = spaceliftConfig + } + } + } + } + + return res, nil +} + func buildSpaceliftDependsOnStackName( dependsOn string, allStackNames []string, @@ -218,9 +444,9 @@ func buildSpaceliftDependsOnStackName( ) (string, error) { var spaceliftStackName string - if utils.SliceContainsString(allStackNames, dependsOn) { + if u.SliceContainsString(allStackNames, dependsOn) { spaceliftStackName = dependsOn - } else if utils.SliceContainsString(componentNamesInCurrentStack, dependsOn) { + } else if u.SliceContainsString(componentNamesInCurrentStack, dependsOn) { spaceliftStackName = fmt.Sprintf("%s-%s", currentStackName, dependsOn) } else { errorMessage := errors.New(fmt.Sprintf("Component '%[1]s' in stack '%[2]s' specifies 'depends_on' dependency '%[3]s', "+ diff --git a/pkg/spacelift/spacelift_stack_processor_test.go b/pkg/spacelift/spacelift_stack_processor_test.go index 2aa324e30..deac8ab22 100644 --- a/pkg/spacelift/spacelift_stack_processor_test.go +++ b/pkg/spacelift/spacelift_stack_processor_test.go @@ -8,6 +8,59 @@ import ( ) func TestSpaceliftStackProcessor(t *testing.T) { + processStackDeps := true + processComponentDeps := true + processImports := true + stackConfigPathTemplate := "stacks/%s.yaml" + + var spaceliftStacks, err = CreateSpaceliftStacks("", nil, processStackDeps, processComponentDeps, processImports, stackConfigPathTemplate) + assert.Nil(t, err) + assert.Equal(t, 24, len(spaceliftStacks)) + + tenant1Ue2DevInfraVpcStack := spaceliftStacks["tenant1-ue2-dev-infra-vpc"].(map[string]interface{}) + tenant1Ue2DevInfraVpcStackStackName := tenant1Ue2DevInfraVpcStack["stack"].(string) + tenant1Ue2DevInfraVpcStackBackend := tenant1Ue2DevInfraVpcStack["backend"].(map[interface{}]interface{}) + tenant1Ue2DevInfraVpcStackBackendWorkspaceKeyPrefix := tenant1Ue2DevInfraVpcStackBackend["workspace_key_prefix"].(string) + assert.Equal(t, "tenant1-ue2-dev-infra-vpc", tenant1Ue2DevInfraVpcStackStackName) + assert.Equal(t, "infra-vpc", tenant1Ue2DevInfraVpcStackBackendWorkspaceKeyPrefix) + + tenant1Ue2DevTestTestComponentOverrideComponent := spaceliftStacks["tenant1-ue2-dev-test-test-component-override"].(map[string]interface{}) + tenant1Ue2DevTestTestComponentOverrideComponentStackName := tenant1Ue2DevTestTestComponentOverrideComponent["stack"].(string) + tenant1Ue2DevTestTestComponentOverrideComponentBackend := tenant1Ue2DevTestTestComponentOverrideComponent["backend"].(map[interface{}]interface{}) + tenant1Ue2DevTestTestComponentOverrideComponentBaseComponent := tenant1Ue2DevTestTestComponentOverrideComponent["base_component"].(string) + tenant1Ue2DevTestTestComponentOverrideComponentBackendWorkspaceKeyPrefix := tenant1Ue2DevTestTestComponentOverrideComponentBackend["workspace_key_prefix"].(string) + tenant1Ue2DevTestTestComponentOverrideComponentDeps := tenant1Ue2DevTestTestComponentOverrideComponent["deps"].([]string) + tenant1Ue2DevTestTestComponentOverrideComponentLabels := tenant1Ue2DevTestTestComponentOverrideComponent["labels"].([]string) + assert.Equal(t, "tenant1-ue2-dev-test-test-component-override", tenant1Ue2DevTestTestComponentOverrideComponentStackName) + assert.Equal(t, "test-test-component", tenant1Ue2DevTestTestComponentOverrideComponentBackendWorkspaceKeyPrefix) + assert.Equal(t, "test/test-component", tenant1Ue2DevTestTestComponentOverrideComponentBaseComponent) + assert.Equal(t, 11, len(tenant1Ue2DevTestTestComponentOverrideComponentDeps)) + assert.Equal(t, "catalog/terraform/services/service-1", tenant1Ue2DevTestTestComponentOverrideComponentDeps[0]) + assert.Equal(t, "catalog/terraform/services/service-1-override", tenant1Ue2DevTestTestComponentOverrideComponentDeps[1]) + assert.Equal(t, "catalog/terraform/services/service-2", tenant1Ue2DevTestTestComponentOverrideComponentDeps[2]) + assert.Equal(t, "catalog/terraform/services/service-2-override", tenant1Ue2DevTestTestComponentOverrideComponentDeps[3]) + assert.Equal(t, "catalog/terraform/tenant1-ue2-dev", tenant1Ue2DevTestTestComponentOverrideComponentDeps[4]) + assert.Equal(t, "catalog/terraform/test-component", tenant1Ue2DevTestTestComponentOverrideComponentDeps[5]) + assert.Equal(t, "catalog/terraform/test-component-override", tenant1Ue2DevTestTestComponentOverrideComponentDeps[6]) + assert.Equal(t, "globals/globals", tenant1Ue2DevTestTestComponentOverrideComponentDeps[7]) + assert.Equal(t, "globals/tenant1-globals", tenant1Ue2DevTestTestComponentOverrideComponentDeps[8]) + assert.Equal(t, "globals/ue2-globals", tenant1Ue2DevTestTestComponentOverrideComponentDeps[9]) + assert.Equal(t, "tenant1/ue2/dev", tenant1Ue2DevTestTestComponentOverrideComponentDeps[10]) + assert.Equal(t, 32, len(tenant1Ue2DevTestTestComponentOverrideComponentLabels)) + assert.Equal(t, "deps:stacks/catalog/terraform/test-component-override.yaml", tenant1Ue2DevTestTestComponentOverrideComponentLabels[25]) + assert.Equal(t, "deps:stacks/globals/globals.yaml", tenant1Ue2DevTestTestComponentOverrideComponentLabels[26]) + assert.Equal(t, "deps:stacks/globals/tenant1-globals.yaml", tenant1Ue2DevTestTestComponentOverrideComponentLabels[27]) + assert.Equal(t, "deps:stacks/globals/ue2-globals.yaml", tenant1Ue2DevTestTestComponentOverrideComponentLabels[28]) + assert.Equal(t, "deps:stacks/tenant1/ue2/dev.yaml", tenant1Ue2DevTestTestComponentOverrideComponentLabels[29]) + assert.Equal(t, "folder:component/test/test-component-override", tenant1Ue2DevTestTestComponentOverrideComponentLabels[30]) + assert.Equal(t, "folder:tenant1/ue2/dev", tenant1Ue2DevTestTestComponentOverrideComponentLabels[31]) + + yamlSpaceliftStacks, err := yaml.Marshal(spaceliftStacks) + assert.Nil(t, err) + t.Log(string(yamlSpaceliftStacks)) +} + +func TestLegacySpaceliftStackProcessor(t *testing.T) { basePath := "../../examples/complete/stacks" filePaths := []string{ @@ -26,6 +79,7 @@ func TestSpaceliftStackProcessor(t *testing.T) { var spaceliftStacks, err = CreateSpaceliftStacks(basePath, filePaths, processStackDeps, processComponentDeps, processImports, stackConfigPathTemplate) assert.Nil(t, err) + assert.Equal(t, 24, len(spaceliftStacks)) tenant1Ue2DevInfraVpcStack := spaceliftStacks["tenant1-ue2-dev-infra-vpc"].(map[string]interface{}) tenant1Ue2DevInfraVpcStackBackend := tenant1Ue2DevInfraVpcStack["backend"].(map[interface{}]interface{}) @@ -52,14 +106,13 @@ func TestSpaceliftStackProcessor(t *testing.T) { assert.Equal(t, "globals/tenant1-globals", tenant1Ue2DevTestTestComponentOverrideComponentDeps[8]) assert.Equal(t, "globals/ue2-globals", tenant1Ue2DevTestTestComponentOverrideComponentDeps[9]) assert.Equal(t, "tenant1/ue2/dev", tenant1Ue2DevTestTestComponentOverrideComponentDeps[10]) - assert.Equal(t, 32, len(tenant1Ue2DevTestTestComponentOverrideComponentLabels)) + assert.Equal(t, 31, len(tenant1Ue2DevTestTestComponentOverrideComponentLabels)) assert.Equal(t, "deps:stacks/catalog/terraform/test-component-override.yaml", tenant1Ue2DevTestTestComponentOverrideComponentLabels[25]) assert.Equal(t, "deps:stacks/globals/globals.yaml", tenant1Ue2DevTestTestComponentOverrideComponentLabels[26]) assert.Equal(t, "deps:stacks/globals/tenant1-globals.yaml", tenant1Ue2DevTestTestComponentOverrideComponentLabels[27]) assert.Equal(t, "deps:stacks/globals/ue2-globals.yaml", tenant1Ue2DevTestTestComponentOverrideComponentLabels[28]) assert.Equal(t, "deps:stacks/tenant1/ue2/dev.yaml", tenant1Ue2DevTestTestComponentOverrideComponentLabels[29]) assert.Equal(t, "folder:component/test/test-component-override", tenant1Ue2DevTestTestComponentOverrideComponentLabels[30]) - assert.Equal(t, "folder:tenant1/ue2/dev", tenant1Ue2DevTestTestComponentOverrideComponentLabels[31]) yamlSpaceliftStacks, err := yaml.Marshal(spaceliftStacks) assert.Nil(t, err)