feat: bump AWS provider to 5.0

Terraform AWS provider 5.0 introduces some schema incompatabilities for the aws_kinesis_firehose resource. We address them in this comit and bump the minimum required version accordingly. For users upgrading from older providers, this change should be a no-op. Introduce tests for the EKS example module, since that is the most susceptible of breakage.
observeinc · Feb 1, 2024 · 3cc3da2 · 3cc3da2
1 parent c7aeef9
commit 3cc3da2
Show file tree

Hide file tree

Showing 27 changed files with 180 additions and 84 deletions.
diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml
@@ -13,9 +13,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
       - name: Build matrix
         id: matrix
         run: |
@@ -37,12 +37,12 @@ jobs:
       OBSERVE_CUSTOMER: 0
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
       - name: Terraform min/max versions
         id: minMax
-        uses: clowdhaus/terraform-min-max@v1.0.2
+        uses: clowdhaus/terraform-min-max@v1.2.6
         with:
           directory: ${{ matrix.directory }}
       - name: Install Terraform v${{ steps.minMax.outputs.minVersion }}
@@ -69,7 +69,7 @@ jobs:
         uses: actions/checkout@v2
       - name: Terraform min/max versions
         id: minMax
-        uses: clowdhaus/terraform-min-max@v1.0.2
+        uses: clowdhaus/terraform-min-max@v1.2.6
     outputs:
       minVersion: ${{ steps.minMax.outputs.minVersion }}
       maxVersion: ${{ steps.minMax.outputs.maxVersion }}
@@ -95,8 +95,11 @@ jobs:
       - name: Install pre-commit dependencies
         run: |
           pip install pre-commit
-          curl -Lo ./terraform-docs.tar.gz https://github.com/terraform-docs/terraform-docs/releases/download/v0.15.0/terraform-docs-v0.15.0-$(uname)-amd64.tar.gz && tar -xzf terraform-docs.tar.gz terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/
-          curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/
+          curl -Lo ./terraform-docs.tar.gz https://github.com/terraform-docs/terraform-docs/releases/download/v0.16.0/terraform-docs-v0.16.0-$(uname)-amd64.tar.gz && tar -xzf terraform-docs.tar.gz terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/
+      - uses: terraform-linters/setup-tflint@v3
+        name: Setup TFLint
+        with:
+          tflint_version: v0.45.0
       - name: Execute pre-commit
         # Run all pre-commit checks on max version supported
         if: ${{ matrix.version ==  needs.getBaseVersion.outputs.maxVersion }}

diff --git a/.github/workflows/tests-integration.yaml b/.github/workflows/tests-integration.yaml
@@ -0,0 +1,99 @@
+name: Run IAC Integration Tests
+
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+  workflow_call:
+  schedule:
+    - cron:  '0 0 * * 2' # Tuesday at 00:00 UTC
+
+jobs:
+  permission_check:
+    runs-on: ubuntu-latest
+    outputs:
+      can-write: ${{ steps.check.outputs.can-write }}
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+    steps:
+    - id: check
+      run: |
+        # If the AWS_ACCESS_KEY_ID secret is MIA we can't run tests
+        if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then
+            echo "can-write=false" >> $GITHUB_OUTPUT
+        else
+            echo "can-write=true" >> $GITHUB_OUTPUT
+        fi
+
+  prepare_matrix:
+    needs: [permission_check]
+    if: needs.permission_check.outputs.can-write == 'true'
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.find_hcl_files.outputs.matrix }}
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Setup the test matrix
+      id: find_hcl_files
+      run: |
+        echo "matrix=$( find . -type d -name tests -print | sed 's:/[^/]*$::' | jq  -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+
+    - uses: actions/checkout@v4
+
+    - name: DCE Provision
+      uses: observeinc/[email protected]
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        budget-amount: ${{ vars.BUDGET_AMOUNT }}
+        budget-currency: 'USD'
+        expiry: '30m'
+        email: '[email protected]'
+
+    - name: Setup tmate session
+      uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
+      with:
+        limit-access-to-actor: true
+
+  test-integration:
+    runs-on: ubuntu-latest
+    needs: [permission_check, prepare_matrix]
+    if: needs.permission_check.outputs.can-write == 'true'
+    strategy:
+      matrix:
+        testfile: ${{fromJson(needs.prepare_matrix.outputs.matrix)}}
+    steps:
+    - name: DCE Use
+      id: dce_setup
+      uses: observeinc/[email protected]
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+    - name: checkout
+      uses: actions/checkout@v4
+
+    - name: Integration test for ${{ matrix.testfile }}
+      run: DIR=${{ matrix.testfile }} make test-dir
+      env:
+        AWS_REGION: us-west-2
+
+  cleanup:
+    needs: [permission_check, test-integration]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+    - name: DCE Cleanup
+      if: needs.permission_check.outputs.can-write == 'true'
+      uses: observeinc/[email protected]
+      with:
+        action-type: 'decommission'
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/README.md b/README.md
@@ -122,14 +122,14 @@ This repository contains examples of how to solve for concrete usecases:
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.1.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.75, <5.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.0 |
 | <a name="requirement_random"></a> [random](#requirement\_random) | >= 3.0.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.75, <5.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.0 |
 | <a name="provider_random"></a> [random](#provider\_random) | >= 3.0.0 |
 
 ## Modules
@@ -178,8 +178,6 @@ No modules.
 | <a name="input_observe_token"></a> [observe\_token](#input\_observe\_token) | Observe Token | `string` | n/a | yes |
 | <a name="input_observe_url"></a> [observe\_url](#input\_observe\_url) | Observe URL. Deprecated. | `string` | `""` | no |
 | <a name="input_s3_delivery_bucket"></a> [s3\_delivery\_bucket](#input\_s3\_delivery\_bucket) | S3 bucket to be used as backup for message delivery | <pre>object({<br>    arn = string<br>  })</pre> | `null` | no |
-| <a name="input_s3_delivery_buffer_interval"></a> [s3\_delivery\_buffer\_interval](#input\_s3\_delivery\_buffer\_interval) | Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. | `number` | `300` | no |
-| <a name="input_s3_delivery_buffer_size"></a> [s3\_delivery\_buffer\_size](#input\_s3\_delivery\_buffer\_size) | Buffer incoming data to the specified size, in MiBs, before delivering it to the destination. | `number` | `5` | no |
 | <a name="input_s3_delivery_cloudwatch_log_stream_name"></a> [s3\_delivery\_cloudwatch\_log\_stream\_name](#input\_s3\_delivery\_cloudwatch\_log\_stream\_name) | Log stream name for S3 delivery logs. If empty, log stream will be disabled | `string` | `"S3Delivery"` | no |
 | <a name="input_s3_delivery_compression_format"></a> [s3\_delivery\_compression\_format](#input\_s3\_delivery\_compression\_format) | The compression format. If no value is specified, the default is UNCOMPRESSED. | `string` | `"UNCOMPRESSED"` | no |
 | <a name="input_s3_delivery_prefix"></a> [s3\_delivery\_prefix](#input\_s3\_delivery\_prefix) | The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered Amazon S3 files | `string` | `null` | no |

diff --git a/examples/cross-account/README.md b/examples/cross-account/README.md
@@ -59,14 +59,14 @@ Note that this will create AWS resources - once you are done, run `terraform des
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.1.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 2.68 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.0 |
 | <a name="requirement_random"></a> [random](#requirement\_random) | >= 3.0.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 2.68 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.0 |
 
 ## Modules
 

diff --git a/examples/cross-account/versions.tf b/examples/cross-account/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 2.68"
+      version = ">= 5.0"
     }
     random = {
       source  = "hashicorp/random"

diff --git a/examples/eks/README.md b/examples/eks/README.md
@@ -22,29 +22,31 @@ Note that this will create AWS resources - once you are done, run `terraform des
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.1.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.20.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.0 |
 | <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.0.1 |
 | <a name="requirement_random"></a> [random](#requirement\_random) | 3.1.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.20.0 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | 3.1.0 |
 
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_eks"></a> [eks](#module\_eks) | terraform-aws-modules/eks/aws | 18.3.1 |
+| <a name="module_eks"></a> [eks](#module\_eks) | terraform-aws-modules/eks/aws | 18.31.2 |
 | <a name="module_observe_kinesis_firehose"></a> [observe\_kinesis\_firehose](#module\_observe\_kinesis\_firehose) | ../../modules/eks | n/a |
-| <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | 3.2.0 |
+| <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 |
 
 ## Resources
 
 | Name | Type |
 |------|------|
 | [aws_subnet.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource |
+| [random_pet.run](https://registry.terraform.io/providers/hashicorp/random/3.1.0/docs/resources/pet) | resource |
 | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source |
 | [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source |
 | [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
@@ -53,8 +55,8 @@ Note that this will create AWS resources - once you are done, run `terraform des
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | EKS Cluster Name | `string` | `"observe-eks-demo"` | no |
-| <a name="input_cluster_version"></a> [cluster\_version](#input\_cluster\_version) | EKS Cluster Version | `string` | `"1.21"` | no |
+| <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | EKS Cluster Name | `string` | `null` | no |
+| <a name="input_cluster_version"></a> [cluster\_version](#input\_cluster\_version) | EKS Cluster Version | `string` | `"1.27"` | no |
 | <a name="input_observe_collection_endpoint"></a> [observe\_collection\_endpoint](#input\_observe\_collection\_endpoint) | Observe Collection Endpoint, e.g https://123456789012.collect.observeinc.com | `string` | n/a | yes |
 | <a name="input_observe_token"></a> [observe\_token](#input\_observe\_token) | Observe token | `string` | n/a | yes |
 

diff --git a/examples/eks/eks.tf b/examples/eks/eks.tf
@@ -1,7 +1,7 @@
 module "eks" {
   source          = "terraform-aws-modules/eks/aws"
-  version         = "18.3.1"
-  cluster_name    = var.cluster_name
+  version         = "18.31.2"
+  cluster_name    = local.cluster_name
   cluster_version = var.cluster_version
   subnet_ids      = module.vpc.private_subnets
 

diff --git a/examples/eks/main.tf b/examples/eks/main.tf
@@ -1,3 +1,7 @@
+locals {
+  cluster_name = var.cluster_name != null ? var.cluster_name : random_pet.run.id
+}
+
 data "aws_eks_cluster" "cluster" {
   name = module.eks.cluster_id
 }
@@ -6,6 +10,8 @@ data "aws_eks_cluster_auth" "cluster" {
   name = module.eks.cluster_id
 }
 
+resource "random_pet" "run" {}
+
 provider "kubernetes" {
   host                   = data.aws_eks_cluster.cluster.endpoint
   token                  = data.aws_eks_cluster_auth.cluster.token

diff --git a/examples/eks/tests/eks.tftest.hcl b/examples/eks/tests/eks.tftest.hcl
@@ -0,0 +1,6 @@
+run "setup" {
+  variables {
+    observe_collection_endpoint = "https://101.collect.observeinc.com"
+    observe_token               = "dsfake:hoho"
+  }
+}
diff --git a/examples/eks/variables.tf b/examples/eks/variables.tf
@@ -1,15 +1,14 @@
 variable "cluster_name" {
   description = "EKS Cluster Name"
   type        = string
-  nullable    = false
-  default     = "observe-eks-demo"
+  default     = null
 }
 
 variable "cluster_version" {
   description = "EKS Cluster Version"
   type        = string
   nullable    = false
-  default     = "1.21"
+  default     = "1.27"
 }
 
 variable "observe_collection_endpoint" {

diff --git a/examples/eks/versions.tf b/examples/eks/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.20.0"
+      version = ">= 5.0"
     }
 
     random = {

diff --git a/examples/eks/vpc.tf b/examples/eks/vpc.tf
@@ -2,9 +2,9 @@ data "aws_availability_zones" "available" {}
 
 module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
-  version = "3.2.0"
+  version = "~> 5.0"
 
-  name                 = var.cluster_name
+  name                 = local.cluster_name
   cidr                 = "10.0.0.0/16"
   azs                  = data.aws_availability_zones.available.names
   private_subnets      = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
@@ -14,17 +14,17 @@ module "vpc" {
   enable_dns_hostnames = true
 
   tags = {
-    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
+    "kubernetes.io/cluster/${local.cluster_name}" = "shared"
   }
 
   public_subnet_tags = {
-    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
-    "kubernetes.io/role/elb"                    = "1"
+    "kubernetes.io/cluster/${local.cluster_name}" = "shared"
+    "kubernetes.io/role/elb"                      = "1"
   }
 
   private_subnet_tags = {
-    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
-    "kubernetes.io/role/internal-elb"           = "1"
+    "kubernetes.io/cluster/${local.cluster_name}" = "shared"
+    "kubernetes.io/role/internal-elb"             = "1"
   }
 }
 

diff --git a/examples/eventbridge/README.md b/examples/eventbridge/README.md
@@ -24,14 +24,14 @@ Note that this will create AWS resources - once you are done, run `terraform des
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.1.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 2.68 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.0 |
 | <a name="requirement_random"></a> [random](#requirement\_random) | >= 3.0.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 2.68 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.0 |
 | <a name="provider_random"></a> [random](#provider\_random) | >= 3.0.0 |
 
 ## Modules

diff --git a/examples/eventbridge/versions.tf b/examples/eventbridge/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 2.68"
+      version = ">= 5.0"
     }
     random = {
       source  = "hashicorp/random"

diff --git a/examples/kinesis/README.md b/examples/kinesis/README.md
@@ -24,14 +24,14 @@ Note that this will create AWS resources - once you are done, run `terraform des
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.1.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 2.68 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.0 |
 | <a name="requirement_random"></a> [random](#requirement\_random) | >= 3.0.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 2.68 |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.0 |
 | <a name="provider_random"></a> [random](#provider\_random) | >= 3.0.0 |
 
 ## Modules

diff --git a/examples/kinesis/versions.tf b/examples/kinesis/versions.tf
@@ -4,7 +4,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 2.68"
+      version = ">= 5.0"
     }
     random = {
       source  = "hashicorp/random"