Improvements and new features (#66)

* Put additional args and flags at the end of `atmos terraform` commands

* Add `remote_state_backend`

* Add `remote_state_backend`

* Add `remote_state_backend`

* Add `remote_state_backend`

* Add `remote_state_backend`

* Add `AutoGenerateBackendFile`

* Add `AutoGenerateBackendFile`

* Add `AutoGenerateBackendFile`

atmos.yaml

@@ -17,6 +17,8 @@ components:
     apply_auto_approve: false
     # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_DEPLOY_RUN_INIT` ENV var, or `--deploy-run-init` command-line argument
     deploy_run_init: true
+    # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_AUTO_GENERATE_BACKEND_FILE` ENV var, or `--auto-generate-backend-file` command-line argument
+    auto_generate_backend_file: false
   helmfile:
     # Can also be set using `ATMOS_COMPONENTS_HELMFILE_BASE_PATH` ENV var, or `--helmfile-dir` command-line argument
     # Supports both absolute and relative paths

examples/complete/atmos.yaml

@@ -17,6 +17,8 @@ components:
     apply_auto_approve: false
     # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_DEPLOY_RUN_INIT` ENV var, or `--deploy-run-init` command-line argument
     deploy_run_init: true
+    # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_AUTO_GENERATE_BACKEND_FILE` ENV var, or `--auto-generate-backend-file` command-line argument
+    auto_generate_backend_file: false
   helmfile:
     # Can also be set using `ATMOS_COMPONENTS_HELMFILE_BASE_PATH` ENV var, or `--helmfile-dir` command-line argument
     # Supports both absolute and relative paths

examples/complete/rootfs/usr/local/etc/atmos/atmos.yaml

@@ -17,6 +17,8 @@ components:
     apply_auto_approve: false
     # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_DEPLOY_RUN_INIT` ENV var, or `--deploy-run-init` command-line argument
     deploy_run_init: true
+    # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_AUTO_GENERATE_BACKEND_FILE` ENV var, or `--auto-generate-backend-file` command-line argument
+    auto_generate_backend_file: false
   helmfile:
     # Can also be set using `ATMOS_COMPONENTS_HELMFILE_BASE_PATH` ENV var, or `--helmfile-dir` command-line argument
     # Supports both absolute and relative paths

examples/complete/stacks/catalog/terraform/test-component-override.yaml

@@ -16,3 +16,11 @@ components:
       component: "test/test-component"
       # Other variables can be overridden here
       vars: {}
+      # Override remote state backend for this component
+      remote_state_backend_type: static # s3, remote, vault, static, etc.
+      remote_state_backend:
+        static:
+          val1: true
+          val2: "2"
+          val3: 3
+          val4: null

examples/complete/stacks/catalog/terraform/test-component.yaml

@@ -12,6 +12,9 @@ components:
       backend:
         s3:
           workspace_key_prefix: test-test-component
+      remote_state_backend:
+        s3:
+          workspace_key_prefix: test-test-component
       settings:
         spacelift:
           workspace_enabled: true

examples/complete/stacks/catalog/terraform/top-level-component1.yaml

@@ -12,6 +12,9 @@ components:
       backend:
         s3:
           workspace_key_prefix: top-level-component1
+      remote_state_backend:
+        s3:
+          workspace_key_prefix: top-level-component1
       settings:
         spacelift:
           workspace_enabled: true

examples/complete/stacks/catalog/terraform/vpc.yaml

@@ -4,6 +4,9 @@ components:
       backend:
         s3:
           workspace_key_prefix: infra-vpc
+      remote_state_backend:
+        s3:
+          workspace_key_prefix: infra-vpc
       settings:
         spacelift:
           workspace_enabled: true

examples/complete/stacks/globals/globals.yaml

@@ -3,19 +3,30 @@ vars:
 
 terraform:
   vars: {}
-  backend_type: s3 # s3, remote, vault, etc.
+
+  backend_type: s3 # s3, remote, vault, static, etc.
   backend:
     s3:
       encrypt: true
       bucket: "eg-ue2-root-tfstate"
       key: "terraform.tfstate"
       dynamodb_table: "eg-ue2-root-tfstate-lock"
-      profile: "eg-gbl-root-terraform"
       acl: "bucket-owner-full-control"
       region: "us-east-2"
     remote:
     vault:
 
+  remote_state_backend_type: s3 # s3, remote, vault, static, etc.
+  remote_state_backend:
+    s3:
+      encrypt: true
+      bucket: "eg-ue2-root-tfstate"
+      key: "terraform.tfstate"
+      dynamodb_table: "eg-ue2-root-tfstate-lock"
+      profile: "eg-gbl-root-terraform"
+      acl: "bucket-owner-full-control"
+      region: "us-east-2"
+
 helmfile:
   vars: {}
 

internal/config/config.go

@@ -24,9 +24,10 @@ var (
 	defaultConfig = Configuration{
 		Components: Components{
 			Terraform: Terraform{
-				BasePath:         "./components/terraform",
-				ApplyAutoApprove: false,
-				DeployRunInit:    true,
+				BasePath:                "./components/terraform",
+				ApplyAutoApprove:        false,
+				DeployRunInit:           true,
+				AutoGenerateBackendFile: false,
 			},
 			Helmfile: Helmfile{
 				BasePath:              "./components/helmfile",
@@ -77,7 +78,8 @@ func InitConfig() error {
 
 	if g.LogVerbose {
 		color.Cyan("\nProcessing and merging configurations in the following order:\n")
-		fmt.Println("system dir, home dir, current dir, ENV vars, command-line arguments\n")
+		fmt.Println("system dir, home dir, current dir, ENV vars, command-line arguments")
+		fmt.Println()
 	}
 
 	v := viper.New()
@@ -184,6 +186,14 @@ func ProcessConfig(configAndStacksInfo ConfigAndStacksInfo) error {
 		Config.Components.Terraform.DeployRunInit = deployRunInitBool
 		color.Cyan(fmt.Sprintf("Using command line argument '%s=%s'", g.DeployRunInitFlag, configAndStacksInfo.DeployRunInit))
 	}
+	if len(configAndStacksInfo.AutoGenerateBackendFile) > 0 {
+		autoGenerateBackendFileBool, err := strconv.ParseBool(configAndStacksInfo.AutoGenerateBackendFile)
+		if err != nil {
+			return err
+		}
+		Config.Components.Terraform.AutoGenerateBackendFile = autoGenerateBackendFileBool
+		color.Cyan(fmt.Sprintf("Using command line argument '%s=%s'", g.AutoGenerateBackendFileFlag, configAndStacksInfo.AutoGenerateBackendFile))
+	}
 
 	// Check config
 	err = checkConfig()

internal/config/schema.go

@@ -1,9 +1,10 @@
 package config
 
 type Terraform struct {
-	BasePath         string `yaml:"base_path" json:"base_path" mapstructure:"base_path"`
-	ApplyAutoApprove bool   `yaml:"apply_auto_approve" json:"apply_auto_approve" mapstructure:"apply_auto_approve"`
-	DeployRunInit    bool   `yaml:"deploy_run_init" json:"deploy_run_init" mapstructure:"deploy_run_init"`
+	BasePath                string `yaml:"base_path" json:"base_path" mapstructure:"base_path"`
+	ApplyAutoApprove        bool   `yaml:"apply_auto_approve" json:"apply_auto_approve" mapstructure:"apply_auto_approve"`
+	DeployRunInit           bool   `yaml:"deploy_run_init" json:"deploy_run_init" mapstructure:"deploy_run_init"`
+	AutoGenerateBackendFile bool   `yaml:"auto_generate_backend_file" json:"auto_generate_backend_file" mapstructure:"auto_generate_backend_file"`
 }
 
 type Helmfile struct {
@@ -56,15 +57,16 @@ type Context struct {
 }
 
 type ArgsAndFlagsInfo struct {
-	AdditionalArgsAndFlags []string
-	SubCommand             string
-	ComponentFromArg       string
-	GlobalOptions          []string
-	TerraformDir           string
-	HelmfileDir            string
-	ConfigDir              string
-	StacksDir              string
-	DeployRunInit          string
+	AdditionalArgsAndFlags  []string
+	SubCommand              string
+	ComponentFromArg        string
+	GlobalOptions           []string
+	TerraformDir            string
+	HelmfileDir             string
+	ConfigDir               string
+	StacksDir               string
+	DeployRunInit           string
+	AutoGenerateBackendFile string
 }
 
 type ConfigAndStacksInfo struct {
@@ -79,6 +81,7 @@ type ConfigAndStacksInfo struct {
 	SubCommand              string
 	ComponentVarsSection    map[interface{}]interface{}
 	ComponentBackendSection map[interface{}]interface{}
+	ComponentBackendType    string
 	AdditionalArgsAndFlags  []string
 	GlobalOptions           []string
 	TerraformDir            string
@@ -88,4 +91,5 @@ type ConfigAndStacksInfo struct {
 	Context                 Context
 	ContextPrefix           string
 	DeployRunInit           string
+	AutoGenerateBackendFile string
 }

internal/config/utils.go

@@ -137,6 +137,16 @@ func processEnvVars() error {
 		Config.Components.Terraform.DeployRunInit = deployRunInitBool
 	}
 
+	componentsTerraformAutoGenerateBackendFile := os.Getenv("ATMOS_COMPONENTS_TERRAFORM_AUTO_GENERATE_BACKEND_FILE")
+	if len(componentsTerraformAutoGenerateBackendFile) > 0 {
+		color.Cyan("Found ENV var ATMOS_COMPONENTS_TERRAFORM_AUTO_GENERATE_BACKEND_FILE=%s", componentsTerraformAutoGenerateBackendFile)
+		componentsTerraformAutoGenerateBackendFileBool, err := strconv.ParseBool(componentsTerraformAutoGenerateBackendFile)
+		if err != nil {
+			return err
+		}
+		Config.Components.Terraform.AutoGenerateBackendFile = componentsTerraformAutoGenerateBackendFileBool
+	}
+
 	componentsHelmfileBasePath := os.Getenv("ATMOS_COMPONENTS_HELMFILE_BASE_PATH")
 	if len(componentsHelmfileBasePath) > 0 {
 		color.Cyan("Found ENV var ATMOS_COMPONENTS_HELMFILE_BASE_PATH=%s", componentsHelmfileBasePath)

internal/exec/describe.go

@@ -71,9 +71,15 @@ func ExecuteDescribeComponent(cmd *cobra.Command, args []string) error {
 
 	// Check and process stacks
 	if c.ProcessedConfig.StackType == "Directory" {
-		componentSection, componentVarsSection, _, err = findComponentConfig(stack, stacksMap, "terraform", component)
+		componentSection,
+			componentVarsSection,
+			_, _, _, _,
+			err = findComponentConfig(stack, stacksMap, "terraform", component)
 		if err != nil {
-			componentSection, componentVarsSection, _, err = findComponentConfig(stack, stacksMap, "helmfile", component)
+			componentSection,
+				componentVarsSection,
+				_, _, _, _,
+				err = findComponentConfig(stack, stacksMap, "helmfile", component)
 			if err != nil {
 				return err
 			}
@@ -108,9 +114,15 @@ func ExecuteDescribeComponent(cmd *cobra.Command, args []string) error {
 		}
 
 		for stackName := range stacksMap {
-			componentSection, componentVarsSection, _, err = findComponentConfig(stackName, stacksMap, "terraform", component)
+			componentSection,
+				componentVarsSection,
+				_, _, _, _,
+				err = findComponentConfig(stackName, stacksMap, "terraform", component)
 			if err != nil {
-				componentSection, componentVarsSection, _, err = findComponentConfig(stackName, stacksMap, "helmfile", component)
+				componentSection,
+					componentVarsSection,
+					_, _, _, _,
+					err = findComponentConfig(stackName, stacksMap, "helmfile", component)
 				if err != nil {
 					continue
 				}

internal/exec/terraform.go

@@ -96,6 +96,31 @@ func ExecuteTerraform(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
+	// Auto generate backend file
+	var backendFileName string
+	if c.Config.Components.Terraform.AutoGenerateBackendFile == true {
+		fmt.Println()
+		if len(info.ComponentFolderPrefix) == 0 {
+			backendFileName = fmt.Sprintf("%s/%s/backend.tf.json",
+				c.Config.Components.Terraform.BasePath,
+				finalComponent,
+			)
+		} else {
+			backendFileName = fmt.Sprintf("%s/%s/%s/backend.tf.json",
+				c.Config.Components.Terraform.BasePath,
+				info.ComponentFolderPrefix,
+				finalComponent,
+			)
+		}
+		color.Cyan("Writing backend config to file:")
+		fmt.Println(backendFileName)
+		var componentBackendConfig = generateComponentBackendConfig(info.ComponentBackendType, info.ComponentBackendSection)
+		err = utils.WriteToFileAsJSON(backendFileName, componentBackendConfig, 0644)
+		if err != nil {
+			return err
+		}
+	}
+
 	// Run `terraform init`
 	runTerraformInit := true
 	if info.SubCommand == "init" ||
@@ -154,7 +179,7 @@ func ExecuteTerraform(cmd *cobra.Command, args []string) error {
 		workspaceName = info.ContextPrefix
 	}
 
-	allArgsAndFlags := append([]string{info.SubCommand}, info.AdditionalArgsAndFlags...)
+	allArgsAndFlags := []string{info.SubCommand}
 
 	switch info.SubCommand {
 	case "plan":
@@ -168,6 +193,8 @@ func ExecuteTerraform(cmd *cobra.Command, args []string) error {
 		break
 	}
 
+	allArgsAndFlags = append(allArgsAndFlags, info.AdditionalArgsAndFlags...)
+
 	// Run `terraform workspace`
 	err = execCommand(info.Command, []string{"workspace", "select", workspaceName}, componentPath, nil)
 	if err != nil {

internal/exec/terraform_generate.go

@@ -69,10 +69,16 @@ func ExecuteTerraformGenerateBackend(cmd *cobra.Command, args []string) error {
 	var componentSection map[string]interface{}
 	var componentVarsSection map[interface{}]interface{}
 	var componentBackendSection map[interface{}]interface{}
+	var componentBackendType string
 
 	// Check and process stacks
 	if c.ProcessedConfig.StackType == "Directory" {
-		componentSection, componentVarsSection, componentBackendSection, err = findComponentConfig(stack, stacksMap, "terraform", component)
+		componentSection,
+			componentVarsSection,
+			componentBackendSection,
+			componentBackendType,
+			_, _,
+			err = findComponentConfig(stack, stacksMap, "terraform", component)
 		if err != nil {
 			return err
 		}
@@ -104,7 +110,12 @@ func ExecuteTerraformGenerateBackend(cmd *cobra.Command, args []string) error {
 		}
 
 		for stackName := range stacksMap {
-			componentSection, componentVarsSection, componentBackendSection, err = findComponentConfig(stackName, stacksMap, "terraform", component)
+			componentSection,
+				componentVarsSection,
+				componentBackendSection,
+				componentBackendType,
+				_, _,
+				err = findComponentConfig(stackName, stacksMap, "terraform", component)
 			if err != nil {
 				continue
 			}
@@ -152,17 +163,15 @@ func ExecuteTerraformGenerateBackend(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	if componentBackendType == "" {
+		return errors.New(fmt.Sprintf("\n'backend_type' is missing for the '%s' component.\n", component))
+	}
+
 	if componentBackendSection == nil {
 		return errors.New(fmt.Sprintf("\nCould not find 'backend' config for the '%s' component.\n", component))
 	}
 
-	var componentBackendConfig = map[string]interface{}{
-		"terraform": map[string]interface{}{
-			"backend": map[string]interface{}{
-				"s3": componentBackendSection,
-			},
-		},
-	}
+	var componentBackendConfig = generateComponentBackendConfig(componentBackendType, componentBackendSection)
 
 	color.Cyan("\nComponent backend config:\n\n")
 	err = utils.PrintAsJSON(componentBackendConfig)