From de574b48cda2b0a7997a6298d33775f0c32cd657 Mon Sep 17 00:00:00 2001 From: jamesrwarren Date: Thu, 17 Oct 2024 13:47:06 +0100 Subject: [PATCH] DDLS-372 move to OIDC in github actions --- .github/workflows/_build.yml | 124 ++++++++++++++++ .github/workflows/_deploy.yml | 86 +++++++++++ .github/workflows/build_containers.yml | 139 ------------------ .github/workflows/terraform.yml | 138 ----------------- .github/workflows/workflow_path_to_live.yml | 74 ++++++++++ .../workflows/workflow_pull_request_path.yml | 82 +++++++++++ terraform/versions.tf | 16 +- 7 files changed, 374 insertions(+), 285 deletions(-) create mode 100644 .github/workflows/_build.yml create mode 100644 .github/workflows/_deploy.yml delete mode 100644 .github/workflows/build_containers.yml delete mode 100644 .github/workflows/terraform.yml create mode 100644 .github/workflows/workflow_path_to_live.yml create mode 100644 .github/workflows/workflow_pull_request_path.yml diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml new file mode 100644 index 0000000..345a77b --- /dev/null +++ b/.github/workflows/_build.yml @@ -0,0 +1,124 @@ +name: "[Docker] Build and Push Images" + +defaults: + run: + shell: bash + +on: + workflow_call: + inputs: + branch_name: + description: "Name of the branch doing the build" + required: true + type: string + tag: + description: "Tag for docker image" + required: false + default: "latest" + type: string + push_to_ecr: + description: "Whether to push to ECR" + required: false + default: true + type: boolean + +jobs: + docker_build_scan_push: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - svc_name: "opg-incident-response" + docker_file: "Dockerfile.response" + - svc_name: "incident-response/nginx" + docker_file: "Dockerfile.nginx" + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v3 + + - name: set up docker buildx + uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 + + - name: export dates + id: cache-dates + run: | + echo "cache_date_today=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT + echo "cache_date_yesterday=$(date -d 'yesterday' +'%Y%m%d')" >> $GITHUB_OUTPUT + + - name: cache docker layers + id: cache-docker + uses: actions/cache@8469c94c6a180dfb41a1bd7e1b46ac557ea124f1 # pin@v3.0.8 + with: + path: /tmp/.buildx-cache + key: ${{ matrix.svc_name }}-${{ inputs.branch_name }}-${{ steps.cache-dates.outputs.cache_date_today }}-${{ github.sha }} + restore-keys: | + ${{ matrix.svc_name }}-${{ inputs.branch_name }}-${{ steps.cache-dates.outputs.cache_date_today }} + ${{ matrix.svc_name }}-main-${{ steps.cache-dates.outputs.cache_date_today }} + ${{ matrix.svc_name }}-${{ inputs.branch_name }}-${{ steps.cache-dates.outputs.cache_date_yesterday }} + ${{ matrix.svc_name }}-main-${{ steps.cache-dates.outputs.cache_date_yesterday }} + + - name: install aws cli + uses: unfor19/install-aws-cli-action@46282f151073130d90347412d9c4ef0640177f22 # pin@v1.0.3 + + - name: configure OIDC AWS credentials for ECR push + if: ${{ github.actor != 'dependabot[bot]' }} + uses: aws-actions/configure-aws-credentials@2cefa29f8797029f898b1baeff3e21a144128687 # pin@v1.7.0 + with: + role-to-assume: arn:aws:iam::311462405659:role/incident-response-gh-actions-ecr-push + role-session-name: github-actions-ecr-push + role-duration-seconds: 900 + aws-region: eu-west-1 + + - name: build docker image + env: + IMAGE_NAME: ${{ matrix.svc_name }} + DOCKERFILE: ${{ matrix.docker_file }} + BRANCH_NAME: ${{ inputs.branch_name }} + run: | + if [ "${BRANCH_NAME}" == "main" ]; then + docker buildx build \ + -f ${DOCKERFILE} \ + --cache-to=type=local,dest=/tmp/.buildx-cache-new \ + --tag ${IMAGE_NAME}:latest \ + --output type=docker \ + . + else + docker buildx build \ + -f ${DOCKERFILE} \ + --cache-from=type=local,src=/tmp/.buildx-cache \ + --cache-to=type=local,dest=/tmp/.buildx-cache-new \ + --tag ${IMAGE_NAME}:latest \ + --output type=docker \ + . + fi + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: ecr login + id: login_ecr + uses: aws-actions/amazon-ecr-login@d5dd46d537c86e506335323688c342319bedcfe1 # pin@v1.5.1 + with: + registries: 311462405659 + + - name: show build tag + env: + IMAGE_TAG: ${{ inputs.tag }} + run: echo "Tag to use - ${IMAGE_TAG}" + + - name: tag and push container + if: ${{ github.actor != 'dependabot[bot]' }} + env: + ECR_REGISTRY: ${{ steps.login_ecr.outputs.registry }} + IMAGE_TAG: ${{ inputs.tag }} + IMAGE_NAME: ${{ matrix.svc_name }} + BRANCH_NAME: ${{ inputs.branch_name }} + run: | + docker tag $IMAGE_NAME:latest $ECR_REGISTRY/$IMAGE_NAME:$IMAGE_TAG + if [[ "${BRANCH_NAME}" == "main" ]]; then + docker tag $IMAGE_NAME:latest $ECR_REGISTRY/$IMAGE_NAME:latest + docker tag $IMAGE_NAME:latest $ECR_REGISTRY/$IMAGE_NAME:main-$IMAGE_TAG + fi + docker push --all-tags $ECR_REGISTRY/$IMAGE_NAME diff --git a/.github/workflows/_deploy.yml b/.github/workflows/_deploy.yml new file mode 100644 index 0000000..66a6c44 --- /dev/null +++ b/.github/workflows/_deploy.yml @@ -0,0 +1,86 @@ +on: + workflow_call: + inputs: + workspace: + description: "Terraform workspace" + required: true + type: string + image_tag: + description: "Image tag to use" + required: false + type: string + default: "" + apply: + description: "Whether to apply terraform" + required: false + type: boolean + default: false + account_name: + required: false + type: string + default: development + description: "Account to get credentials for" + +jobs: + terraform_workflow: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # pin@v3 + + - uses: unfor19/install-aws-cli-action@46282f151073130d90347412d9c4ef0640177f22 # pin@v1.0.3 + + - id: terraform_version + name: get terraform version + uses: ministryofjustice/opg-github-actions/.github/actions/terraform-version@v3.0.8 + with: + terraform_directory: terraform + + - uses: hashicorp/setup-terraform@1b93182764c8332e7679b2393cb307cbe7baf9dc # pin@v2.0.0 + with: + terraform_version: ${{ steps.terraform_version.outputs.version }} + terraform_wrapper: false + + - uses: terraform-linters/setup-tflint@v4 + name: Setup TFLint + with: + tflint_version: v0.50.1 + + - name: configure OIDC AWS credentials for terraform + uses: aws-actions/configure-aws-credentials@2cefa29f8797029f898b1baeff3e21a144128687 # pin@v1.7.0 + with: + role-to-assume: "arn:aws:iam::631181914621:role/oidc-incident-response-${{ inputs.account_name }}" + role-session-name: github-actions-terraform-incident-response + role-duration-seconds: 7400 + aws-region: eu-west-1 + + - name: terraform format + run: terraform fmt --check --recursive + + - name: TF Lint + run: tflint --recursive + + - name: terraform init for environment + run: terraform init -input=false + working-directory: terraform + + - name: terraform plan + env: + TF_WORKSPACE: ${{ inputs.workspace }} + TF_VAR_app_tag: ${{ inputs.image_tag }} + run: | + terraform workspace show + terraform plan -input=false -auto-approve --lock-timeout=300s --parallelism=200 --out=${TF_WORKSPACE}.plan > ${TF_WORKSPACE}.log + + - name: output plan + run: cat ${TF_WORKSPACE}.log + + - name: output concise plan + run: cat ${TF_WORKSPACE}.log | grep '\.' | grep '#' || true + + - name: terraform apply + if: inputs.apply + env: + TF_WORKSPACE: ${{ inputs.workspace }} + TF_VAR_app_tag: ${{ inputs.image_tag }} + CI: true + run: terraform apply -input=false -auto-approve -parallelism=200 -lock-timeout=300s ${{ env.TF_WORKSPACE }}.plan diff --git a/.github/workflows/build_containers.yml b/.github/workflows/build_containers.yml deleted file mode 100644 index 285d3a7..0000000 --- a/.github/workflows/build_containers.yml +++ /dev/null @@ -1,139 +0,0 @@ -name: Build - -on: - pull_request: - branches: - - main - paths-ignore: - - 'terraform/*' - push: - branches: - - main - paths-ignore: - - 'terraform/*' - -defaults: - run: - shell: bash - -jobs: - build-containers: - name: 'Build container only' - if: ${{ github.actor == 'dependabot[bot]' }} - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Build - run: | - docker build -f Dockerfile.response . - docker build -f Dockerfile.nginx . - - - build: - name: 'Build & Push Containers' - if: ${{ github.actor != 'dependabot[bot]' }} - runs-on: ubuntu-latest - outputs: - build_tag: ${{ steps.create_tag.outputs.new_tag }} - - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - role-to-assume: arn:aws:iam::311462405659:role/incident-response-ci - role-duration-seconds: 3600 - role-session-name: GitHubActions - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - with: - registries: 311462405659 - - - name: Check out code - uses: actions/checkout@v4 - - - name: Extract branch name - run: | - if [ "${{ github.head_ref }}" == "" ]; then - echo BRANCH_NAME=main >> $GITHUB_ENV - else - echo BRANCH_NAME=$(echo ${{ github.head_ref }} | sed 's/\//-/g') >> $GITHUB_ENV - fi - id: extract_branch - - - name: Bump version and push tag - uses: anothrNick/github-tag-action@1.61.0 - id: create_tag - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - INITIAL_VERSION: 1.0.0 - DEFAULT_BUMP: minor - PRERELEASE: true - PRERELEASE_SUFFIX: ${{ env.BRANCH_NAME }} - RELEASE_BRANCHES: main - WITH_V: true - - - name: Build & Push - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - RESPONSE_ECR_REPOSITORY: opg-incident-response - NGINX_ECR_REPOSITORY: incident-response/nginx - IMAGE_TAG: ${{ steps.create_tag.outputs.new_tag }} - if: github.ref != 'refs/heads/main' - run: | - docker build -f Dockerfile.response -t $ECR_REGISTRY/$RESPONSE_ECR_REPOSITORY:$IMAGE_TAG . - docker push $ECR_REGISTRY/$RESPONSE_ECR_REPOSITORY:$IMAGE_TAG - docker build -f Dockerfile.nginx -t $ECR_REGISTRY/$NGINX_ECR_REPOSITORY:$IMAGE_TAG . - docker push $ECR_REGISTRY/$NGINX_ECR_REPOSITORY:$IMAGE_TAG - - - name: Build & Push - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - RESPONSE_ECR_REPOSITORY: opg-incident-response - NGINX_ECR_REPOSITORY: incident-response/nginx - IMAGE_TAG: ${{ steps.create_tag.outputs.new_tag }} - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - run: | - docker build -f Dockerfile.response -t $ECR_REGISTRY/$RESPONSE_ECR_REPOSITORY:$IMAGE_TAG . - docker push $ECR_REGISTRY/$RESPONSE_ECR_REPOSITORY:$IMAGE_TAG - docker build -f Dockerfile.nginx -t $ECR_REGISTRY/$NGINX_ECR_REPOSITORY:$IMAGE_TAG . - docker push $ECR_REGISTRY/$NGINX_ECR_REPOSITORY:$IMAGE_TAG - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ steps.create_tag.outputs.new_tag }} - release_name: ${{ steps.create_tag.outputs.new_tag }} - draft: false - prerelease: false - - push_to_parameter_store: - name: Store Tag in Parameter Store - needs: build - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - role-to-assume: arn:aws:iam::997462338508:role/incident-response-ci - role-duration-seconds: 3600 - role-session-name: GitHubActions - - name: Install AWS CLI - id: install-aws-cli - uses: unfor19/install-aws-cli-action@v1 - - name: Push Tag to Parameter Store - run: | - aws ssm put-parameter --name "incident-response-production-tag" --type "String" --value "${{ needs.build.outputs.build_tag }}" --overwrite --region=eu-west-1 diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml deleted file mode 100644 index 88b94ef..0000000 --- a/.github/workflows/terraform.yml +++ /dev/null @@ -1,138 +0,0 @@ -name: Terraform Lint, Plan, Apply - -on: - pull_request: - branches: - - main - paths: - - 'terraform/*' - push: - branches: - - main - workflow_dispatch: - -defaults: - run: - shell: bash - working-directory: terraform - -jobs: - pull-tag: - name: Pull latest tag from parameter store. - runs-on: ubuntu-latest - outputs: - latest-tag: ${{ steps.output_tag.outputs.tag }} - steps: - - uses: actions/checkout@v4 - - name: Wait for build - uses: lewagon/wait-on-check-action@v1.3.1 - with: - ref: main - repo-token: ${{ secrets.GITHUB_TOKEN }} - wait-interval: 20 - running-workflow-name: 'Pull latest tag from parameter store.' - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - role-to-assume: arn:aws:iam::997462338508:role/incident-response-ci - role-duration-seconds: 3600 - role-session-name: GitHubActions - - name: Install AWS CLI - id: install-aws-cli - uses: unfor19/install-aws-cli-action@v1 - - name: Pull Tag from Parameter Store - run: | - echo 'TAG_NAME='$(aws ssm get-parameter --region "eu-west-1" --name "incident-response-production-tag" --query Parameter.Value) >> $GITHUB_ENV - - name: Output Tag - id: output_tag - run: echo "::set-output name=tag::${{ env.TAG_NAME }}" - - lint-and-validate: - name: Terraform Lint & Validate - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: hashicorp/setup-terraform@v1 - with: - terraform_version: 1.8.1 - terraform_wrapper: false - - uses: terraform-linters/setup-tflint@v4 - name: Setup TFLint - with: - tflint_version: v0.50.1 - - - name: Configure AWS Credentials For Terraform - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - role-session-name: GitHubActionsTerraform - - - name: Terraform Format - run: terraform fmt --check --recursive - - - name: TF Lint - run: tflint --recursive - - - name: Terraform Init - run: terraform init - - - name: Terraform Validate - run: terraform validate - - - plan-and-apply: - name: Plan ${{ matrix.environment }} - runs-on: ubuntu-latest - needs: - - lint-and-validate - - pull-tag - env: - TF_VAR_app_tag: ${{ needs.pull-tag.outputs.latest-tag }} - strategy: - max-parallel: 1 - matrix: - include: - - environment: "Development" - workspace_environment: "development" - - - environment: "Production" - workspace_environment: "production" - - steps: - - uses: actions/checkout@v4 - - uses: hashicorp/setup-terraform@v1 - with: - terraform_version: 1.8.1 - terraform_wrapper: false - - - name: Configure AWS Credentials For Terraform - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - role-session-name: GitHubActionsTerraform - - - name: Setup - run: echo TF_WORKSPACE=${{ matrix.workspace_environment }} >> $GITHUB_ENV - - - name: Init - run: terraform init - - - name: Plan - run: terraform plan --lock-timeout=300s --parallelism=200 --out=${{ env.TF_WORKSPACE }}.plan > ${{ env.TF_WORKSPACE }}.log - - - name: Output Plan - run: cat ${{ env.TF_WORKSPACE }}.log - - - name: Output ConcisePlan - run: cat ${{ env.TF_WORKSPACE }}.log | grep '\.' | grep '#' || true - - - name: Apply ${{ matrix.environment }} - if: github.ref == 'refs/heads/main' - run: terraform apply -parallelism=200 -lock-timeout=300s ${{ env.TF_WORKSPACE }}.plan diff --git a/.github/workflows/workflow_path_to_live.yml b/.github/workflows/workflow_path_to_live.yml new file mode 100644 index 0000000..1ac4192 --- /dev/null +++ b/.github/workflows/workflow_path_to_live.yml @@ -0,0 +1,74 @@ +name: "[Workflow] Path to Live" + +concurrency: + group: pull-request-workflow + +on: + push: + branches: + - main # match main + +permissions: + id-token: write + contents: write + security-events: write + pull-requests: read + actions: none + checks: none + deployments: none + issues: none + packages: none + repository-projects: none + statuses: none + +defaults: + run: + shell: bash + +jobs: + workflow_variables: + runs-on: ubuntu-latest + name: output workflow variables + outputs: + parsed_branch: main + build_identifier: main + version_tag: ${{ steps.semver_tag.outputs.created_tag }} + steps: + - uses: actions/checkout@9a9194f87191a7e9055e3e9b95b8cfb13023bb08 # pin@v3 + + - name: generate semver tag for release + id: semver_tag + uses: ministryofjustice/opg-github-actions/.github/actions/semver-tag@v3.1.0 + with: + prerelease: false + default_bump: "minor" + + build_scan_push_containers: + name: Docker Build, Scan and Push + uses: ./.github/workflows/_build.yml + needs: [workflow_variables] + with: + tag: ${{ needs.workflow_variables.outputs.version_tag }} + branch_name: ${{ needs.workflow_variables.outputs.parsed_branch }} + + deploy_to_development_environment: + if: ${{ github.actor != 'dependabot[bot]' }} + name: Deploy to Development Environment + needs: [ + workflow_variables, + build_scan_push_containers + ] + uses: ./.github/workflows/_deploy.yml + with: + workspace: development + image_tag: ${{ needs.workflow_variables.outputs.version_tag }} + account_name: development + apply: true + + end_of_pr_workflow: + name: End of PR Workflow + runs-on: ubuntu-latest + needs: [workflow_variables,deploy_to_development_environment] + steps: + - name: End of PR Workflow + run: echo "${{ needs.workflow_variables.outputs.version_tag }} tested, built and deployed to Production" diff --git a/.github/workflows/workflow_pull_request_path.yml b/.github/workflows/workflow_pull_request_path.yml new file mode 100644 index 0000000..8c99f1c --- /dev/null +++ b/.github/workflows/workflow_pull_request_path.yml @@ -0,0 +1,82 @@ +name: "[Workflow] Pull request to branch" + +concurrency: + group: path-to-live-workflow + +on: + pull_request: + branches: + - main + +permissions: + id-token: write + contents: write + security-events: write + pull-requests: read + actions: none + checks: none + deployments: none + issues: none + packages: none + repository-projects: none + statuses: none + +defaults: + run: + shell: bash + +jobs: + workflow_variables: + runs-on: ubuntu-latest + name: output workflow variables + outputs: + parsed_branch: ${{ steps.branch_name.outputs.safe }} + build_identifier: ${{ steps.branch_name.outputs.safe }}${{ github.event.pull_request.number }} + version_tag: ${{ steps.semver_tag.outputs.created_tag }} + steps: + - uses: actions/checkout@9a9194f87191a7e9055e3e9b95b8cfb13023bb08 # pin@v3 + + - name: generate safe branch name + id: branch_name + uses: ministryofjustice/opg-github-actions/.github/actions/branch-name@v3.1.0 + + - name: output build identifier + run: | + echo "Build Identifier: ${{ steps.branch_name.outputs.safe }}${{ github.event.pull_request.number }}" >> $GITHUB_STEP_SUMMARY + + - name: generate semver tag for pull request + id: semver_tag + uses: ministryofjustice/opg-github-actions/.github/actions/semver-tag@v3.1.0 + with: + prerelease: true + default_bump: "minor" + + build_scan_push_containers: + name: Docker Build, Scan and Push + uses: ./.github/workflows/_build.yml + needs: [workflow_variables] + with: + tag: ${{ needs.workflow_variables.outputs.version_tag }} + branch_name: ${{ needs.workflow_variables.outputs.parsed_branch }} + + deploy_to_development_environment: + if: ${{ github.actor != 'dependabot[bot]' }} + name: Deploy to Development Environment + needs: [ + workflow_variables, + build_scan_push_containers + ] + uses: ./.github/workflows/_deploy.yml + with: + workspace: ${{ needs.workflow_variables.outputs.build_identifier }} + image_tag: ${{ needs.workflow_variables.outputs.version_tag }} + account_name: development + apply: true + + end_of_pr_workflow: + name: End of PR Workflow + runs-on: ubuntu-latest + needs: [workflow_variables,deploy_to_development_environment] + steps: + - name: End of PR Workflow + run: echo "${{ needs.workflow_variables.outputs.version_tag }} tested, built and deployed to Development" diff --git a/terraform/versions.tf b/terraform/versions.tf index aff3dc4..9f739b2 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -1,4 +1,12 @@ terraform { + required_version = "1.9.7" + required_providers { + aws = { + source = "hashicorp/aws" + version = "5.42.0" + } + } + backend "s3" { bucket = "opg.terraform.state" key = "opg-incident-response/terraform.tfstate" @@ -7,14 +15,6 @@ terraform { role_arn = "arn:aws:iam::311462405659:role/incident-response-ci" dynamodb_table = "remote_lock" } - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 5.42.0" - } - } - required_version = ">= 1.8.1" } provider "aws" {