Update remote state backend. Add acceptance tests (#76)

* Update `remote_state_backend`

* Update `remote_state_backend`

* Update `remote_state_backend`

* Update `remote_state_backend`

* Update `remote_state_backend`

* Update `remote_state_backend`

* Update `remote_state_backend`

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
       - name: 'Setup Go'
         uses: actions/setup-go@v2
         with:
-          go-version: '1.17'
+          go-version: '1.16'
 
       # Print Go version
       - run: go version

.github/workflows/test.yml

@@ -0,0 +1,66 @@
+name: Tests
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths-ignore:
+      - "README.md"
+
+  release:
+    types: [created]
+    paths-ignore:
+      - "README.md"
+
+  workflow_dispatch:
+
+jobs:
+  # ensure the code builds...
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Set up Go
+        uses: actions/[email protected]
+        with:
+          go-version: "1.16"
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/[email protected]
+
+      - name: Get dependencies
+        run: |
+          make deps
+
+      - name: Build
+        run: |
+          make build
+
+      - name: Version
+        run: |
+          make version
+
+  # run acceptance tests
+  test:
+    name: Test
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Set up Go
+        uses: actions/[email protected]
+        with:
+          go-version: "1.16"
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/[email protected]
+
+      - name: Get dependencies
+        run: |
+          make deps
+
+      - name: Acceptance tests
+        timeout-minutes: 10
+        run: |
+          make testacc

Makefile

@@ -1,8 +1,9 @@
+TEST?=$$(go list ./... | grep -v 'vendor')
 SHELL := /bin/bash
-GOOS=darwin
-#GOOS=linux
+#GOOS=darwin
+GOOS=linux
 GOARCH=amd64
-VERSION=test2
+VERSION=test3
 
 # List of targets the `readme` target should call before generating the readme
 export README_DEPS ?= docs/targets.md
@@ -13,10 +14,21 @@ export README_DEPS ?= docs/targets.md
 lint:
 	$(SELF) terraform/install terraform/get-modules terraform/get-plugins terraform/lint terraform/validate
 
-build:
+get:
+	go get
+
+build: get
 	env GOOS=${GOOS} GOARCH=${GOARCH} go build -o build/atmos -v -ldflags "-X 'github.com/cloudposse/atmos/cmd.Version=${VERSION}'"
 
+version: build
+	chmod +x ./build/atmos
+	./build/atmos version
+
 deps:
 	go mod download
 
-.PHONY: lint build deps
+# Run acceptance tests
+testacc: get
+	go test $(TEST) -v $(TESTARGS) -timeout 2m
+
+.PHONY: lint get build deps version testacc

examples/complete/rootfs/usr/local/etc/atmos/atmos.yaml

@@ -12,7 +12,7 @@ components:
   terraform:
     # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_BASE_PATH` ENV var, or `--terraform-dir` command-line argument
     # Supports both absolute and relative paths
-    base_path: "./components/terraform"
+    base_path: "/components/terraform"
     # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_APPLY_AUTO_APPROVE` ENV var
     apply_auto_approve: false
     # Can also be set using `ATMOS_COMPONENTS_TERRAFORM_DEPLOY_RUN_INIT` ENV var, or `--deploy-run-init` command-line argument
@@ -22,7 +22,7 @@ components:
   helmfile:
     # Can also be set using `ATMOS_COMPONENTS_HELMFILE_BASE_PATH` ENV var, or `--helmfile-dir` command-line argument
     # Supports both absolute and relative paths
-    base_path: "./components/helmfile"
+    base_path: "/components/helmfile"
     # Can also be set using `ATMOS_COMPONENTS_HELMFILE_KUBECONFIG_PATH` ENV var
     kubeconfig_path: "/dev/shm"
     # Can also be set using `ATMOS_COMPONENTS_HELMFILE_HELM_AWS_PROFILE_PATTERN` ENV var
@@ -33,7 +33,7 @@ components:
 stacks:
   # Can also be set using `ATMOS_STACKS_BASE_PATH` ENV var, or `--config-dir` and `--stacks-dir` command-line arguments
   # Supports both absolute and relative paths
-  base_path: "./stacks"
+  base_path: "/stacks"
   # Can also be set using `ATMOS_STACKS_INCLUDED_PATHS` ENV var (comma-separated values string)
   included_paths:
     - "**/*"

examples/complete/stacks/catalog/terraform/test-component.yaml

@@ -12,9 +12,6 @@ components:
       backend:
         s3:
           workspace_key_prefix: test-test-component
-      remote_state_backend:
-        s3:
-          workspace_key_prefix: test-test-component
       settings:
         spacelift:
           workspace_enabled: true

examples/complete/stacks/catalog/terraform/top-level-component1.yaml

@@ -12,9 +12,6 @@ components:
       backend:
         s3:
           workspace_key_prefix: top-level-component1
-      remote_state_backend:
-        s3:
-          workspace_key_prefix: top-level-component1
       settings:
         spacelift:
           workspace_enabled: true

examples/complete/stacks/catalog/terraform/vpc.yaml

@@ -4,9 +4,6 @@ components:
       backend:
         s3:
           workspace_key_prefix: infra-vpc
-      remote_state_backend:
-        s3:
-          workspace_key_prefix: infra-vpc
       settings:
         spacelift:
           workspace_enabled: true

examples/complete/stacks/globals/globals.yaml

@@ -13,19 +13,13 @@ terraform:
       dynamodb_table: "eg-ue2-root-tfstate-lock"
       acl: "bucket-owner-full-control"
       region: "us-east-2"
+      role_arn: null
     remote:
     vault:
 
-  remote_state_backend_type: s3 # s3, remote, vault, static, etc.
   remote_state_backend:
     s3:
-      encrypt: true
-      bucket: "eg-ue2-root-tfstate"
-      key: "terraform.tfstate"
-      dynamodb_table: "eg-ue2-root-tfstate-lock"
-      profile: "eg-gbl-root-terraform"
-      acl: "bucket-owner-full-control"
-      region: "us-east-2"
+      role_arn: "arn:aws:iam::123456789012:role/eg-gbl-root-terraform"
 
 helmfile:
   vars: {}

pkg/stack/stack_processor.go

@@ -462,7 +462,9 @@ func ProcessConfig(
 					finalComponentBackendType = componentBackendType
 				}
 
-				finalComponentBackendSection, err := m.Merge([]map[interface{}]interface{}{globalBackendSection, baseComponentBackendSection, componentBackendSection})
+				finalComponentBackendSection, err := m.Merge([]map[interface{}]interface{}{globalBackendSection,
+					baseComponentBackendSection,
+					componentBackendSection})
 				if err != nil {
 					return nil, err
 				}
@@ -473,21 +475,34 @@ func ProcessConfig(
 				}
 
 				// Final remote state backend
-				finalComponentRemoteStateBackendType := globalRemoteStateBackendType
+				finalComponentRemoteStateBackendType := finalComponentBackendType
+				if len(globalRemoteStateBackendType) > 0 {
+					finalComponentRemoteStateBackendType = globalRemoteStateBackendType
+				}
 				if len(baseComponentRemoteStateBackendType) > 0 {
 					finalComponentRemoteStateBackendType = baseComponentRemoteStateBackendType
 				}
 				if len(componentRemoteStateBackendType) > 0 {
 					finalComponentRemoteStateBackendType = componentRemoteStateBackendType
 				}
 
-				finalComponentRemoteStateBackendSection, err := m.Merge([]map[interface{}]interface{}{globalRemoteStateBackendSection, baseComponentRemoteStateBackendSection, componentRemoteStateBackendSection})
+				finalComponentRemoteStateBackendSection, err := m.Merge([]map[interface{}]interface{}{globalRemoteStateBackendSection,
+					baseComponentRemoteStateBackendSection,
+					componentRemoteStateBackendSection})
+				if err != nil {
+					return nil, err
+				}
+
+				// Merge `backend` and `remote_state_backend` sections
+				// This will allow keeping `remote_state_backend` section DRY
+				finalComponentRemoteStateBackendSectionMerged, err := m.Merge([]map[interface{}]interface{}{finalComponentBackendSection,
+					finalComponentRemoteStateBackendSection})
 				if err != nil {
 					return nil, err
 				}
 
 				finalComponentRemoteStateBackend := map[interface{}]interface{}{}
-				if i, ok2 := finalComponentRemoteStateBackendSection[finalComponentRemoteStateBackendType]; ok2 {
+				if i, ok2 := finalComponentRemoteStateBackendSectionMerged[finalComponentRemoteStateBackendType]; ok2 {
 					finalComponentRemoteStateBackend = i.(map[interface{}]interface{})
 				}
 

pkg/stack/stack_processor_test.go

@@ -61,15 +61,57 @@ func TestStackProcessor(t *testing.T) {
 
 	infraVpcComponent := terraformComponents["infra/vpc"].(map[interface{}]interface{})
 	infraVpcComponentBackend := infraVpcComponent["backend"].(map[interface{}]interface{})
-	infraVpcComponentBackendWorkspaceKeyPrefix := infraVpcComponentBackend["workspace_key_prefix"].(string)
+	infraVpcComponentBackendType := infraVpcComponent["backend_type"]
+	infraVpcComponentRemoteSateBackend := infraVpcComponent["remote_state_backend"].(map[interface{}]interface{})
+	infraVpcComponentRemoteSateBackendType := infraVpcComponent["remote_state_backend_type"]
+	infraVpcComponentBackendWorkspaceKeyPrefix := infraVpcComponentBackend["workspace_key_prefix"]
+	infraVpcComponentRemoteStateBackendWorkspaceKeyPrefix := infraVpcComponentRemoteSateBackend["workspace_key_prefix"]
 	assert.Equal(t, "infra-vpc", infraVpcComponentBackendWorkspaceKeyPrefix)
+	assert.Equal(t, "infra-vpc", infraVpcComponentRemoteStateBackendWorkspaceKeyPrefix)
+	assert.Equal(t, "s3", infraVpcComponentBackendType)
+	assert.Equal(t, "s3", infraVpcComponentRemoteSateBackendType)
+
+	testTestComponent := terraformComponents["test/test-component"].(map[interface{}]interface{})
+	testTestComponentBackend := testTestComponent["backend"].(map[interface{}]interface{})
+	testTestComponentBackendType := testTestComponent["backend_type"]
+	testTestComponentBackendBucket := testTestComponentBackend["bucket"]
+	testTestComponentBackendWorkspaceKeyPrefix := testTestComponentBackend["workspace_key_prefix"]
+	testTestComponentBackendRoleArn := testTestComponentBackend["role_arn"]
+	testTestComponentRemoteStateBackend := testTestComponent["remote_state_backend"].(map[interface{}]interface{})
+	testTestComponentRemoteStateBackendType := testTestComponent["remote_state_backend_type"]
+	testTestComponentRemoteStateBackendBucket := testTestComponentRemoteStateBackend["bucket"]
+	testTestComponentRemoteStateBackendWorkspaceKeyPrefix := testTestComponentRemoteStateBackend["workspace_key_prefix"]
+	testTestComponentRemoteStateBackendRoleArn := testTestComponentRemoteStateBackend["role_arn"]
+	assert.Equal(t, "eg-ue2-root-tfstate", testTestComponentBackendBucket)
+	assert.Equal(t, "eg-ue2-root-tfstate", testTestComponentRemoteStateBackendBucket)
+	assert.Equal(t, "test-test-component", testTestComponentBackendWorkspaceKeyPrefix)
+	assert.Equal(t, "test-test-component", testTestComponentRemoteStateBackendWorkspaceKeyPrefix)
+	assert.Equal(t, "s3", testTestComponentBackendType)
+	assert.Equal(t, "s3", testTestComponentRemoteStateBackendType)
+	assert.Equal(t, nil, testTestComponentBackendRoleArn)
+	assert.Equal(t, "arn:aws:iam::123456789012:role/eg-gbl-root-terraform", testTestComponentRemoteStateBackendRoleArn)
 
 	testTestComponentOverrideComponent := terraformComponents["test/test-component-override"].(map[interface{}]interface{})
 	testTestComponentOverrideComponentBackend := testTestComponentOverrideComponent["backend"].(map[interface{}]interface{})
-	testTestComponentOverrideComponentBaseComponent := testTestComponentOverrideComponent["component"].(string)
-	testTestComponentOverrideComponentBackendWorkspaceKeyPrefix := testTestComponentOverrideComponentBackend["workspace_key_prefix"].(string)
+	testTestComponentOverrideComponentBackendType := testTestComponentOverrideComponent["backend_type"]
+	testTestComponentOverrideComponentBackendWorkspaceKeyPrefix := testTestComponentOverrideComponentBackend["workspace_key_prefix"]
+	testTestComponentOverrideComponentBackendBucket := testTestComponentOverrideComponentBackend["bucket"]
+	testTestComponentOverrideComponentRemoteStateBackend := testTestComponentOverrideComponent["remote_state_backend"].(map[interface{}]interface{})
+	testTestComponentOverrideComponentRemoteStateBackendVal1 := testTestComponentOverrideComponentRemoteStateBackend["val1"].(bool)
+	testTestComponentOverrideComponentRemoteStateBackendVal2 := testTestComponentOverrideComponentRemoteStateBackend["val2"]
+	testTestComponentOverrideComponentRemoteStateBackendVal3 := testTestComponentOverrideComponentRemoteStateBackend["val3"].(int)
+	testTestComponentOverrideComponentRemoteStateBackendVal4 := testTestComponentOverrideComponentRemoteStateBackend["val4"]
+	testTestComponentOverrideComponentRemoteStateBackendType := testTestComponentOverrideComponent["remote_state_backend_type"]
+	testTestComponentOverrideComponentBaseComponent := testTestComponentOverrideComponent["component"]
 	assert.Equal(t, "test-test-component", testTestComponentOverrideComponentBackendWorkspaceKeyPrefix)
+	assert.Equal(t, "eg-ue2-root-tfstate", testTestComponentOverrideComponentBackendBucket)
 	assert.Equal(t, "test/test-component", testTestComponentOverrideComponentBaseComponent)
+	assert.Equal(t, "s3", testTestComponentOverrideComponentBackendType)
+	assert.Equal(t, "static", testTestComponentOverrideComponentRemoteStateBackendType)
+	assert.Equal(t, true, testTestComponentOverrideComponentRemoteStateBackendVal1)
+	assert.Equal(t, "2", testTestComponentOverrideComponentRemoteStateBackendVal2)
+	assert.Equal(t, 3, testTestComponentOverrideComponentRemoteStateBackendVal3)
+	assert.Equal(t, nil, testTestComponentOverrideComponentRemoteStateBackendVal4)
 
 	yamlConfig, err := yaml.Marshal(mapConfig1)
 	assert.Nil(t, err)