Merge pull request #580 from ministryofjustice/csr/make-known-good-am… #51
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: hmpps | |
| # This pipeline works across multiple terragrunt projects within a team | |
| # directory with locking (turnstyle) and PR comment functionality. | |
| # | |
| # The init step figures out which projects to plan/apply | |
| # The plan/apply steps run across each project as a matrix | |
| # | |
| # It allow both plan/apply via manual dispatch. | |
| # For CI/CD, it runs a plan when PR raised and apply when merged. | |
| # | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| projects: | |
| description: 'One or more project to plan/apply, space separated' | |
| required: true | |
| default: '' | |
| action: | |
| description: 'Set to plan or apply' | |
| required: true | |
| default: 'plan' | |
| pull_request: | |
| types: | |
| - opened | |
| - edited | |
| - synchronize | |
| - reopened | |
| branches: | |
| - main | |
| paths: | |
| - teams/hmpps/** | |
| - modules/imagebuilder/** | |
| - .github/workflows/hmpps.yml | |
| - ansible/** | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - teams/hmpps/** | |
| - modules/imagebuilder/** | |
| - .github/workflows/hmpps.yml | |
| - ansible/** | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| contents: read # This is required for actions/checkout | |
| pull-requests: write # For posting comments to PR | |
| env: | |
| AWS_REGION: "eu-west-2" | |
| ENVIRONMENT_MANAGEMENT: ${{ secrets.MODERNISATION_PLATFORM_ENVIRONMENTS }} | |
| # set branch to main on pull requests so plan changes are same as push-to-main | |
| TF_IN_AUTOMATION: true | |
| TF_VAR_BRANCH_NAME: ${{ github.event_name == 'pull_request' && 'main' || (github.head_ref || github.ref_name) }} | |
| TF_VAR_GH_ACTOR_NAME: ${{ github.actor}} | |
| TF_ENV: production | |
| TEAM_DIR: teams/hmpps | |
| PROJECT_DIR_DEPTH: 3 | |
| DEFAULT_TERRAFORM_VERSION: 1.3.3 | |
| TERRAGRUNT_VERSION: v0.36.1 | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| init: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| action: ${{ steps.parseinput.outputs.action }} | |
| projects: ${{ steps.parseinput.outputs.projects }} | |
| pr_number: ${{ steps.parseinput.outputs.pr_number }} | |
| matrix: ${{ steps.setupproject.outputs.matrix }} | |
| steps: | |
| - name: Checkout the code | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 | |
| - name: Set Account Number | |
| run: echo "ACCOUNT_NUMBER=$(jq -r -e '.modernisation_platform_account_id' <<< $ENVIRONMENT_MANAGEMENT)" >> $GITHUB_ENV | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 | |
| with: | |
| role-to-assume: "arn:aws:iam::${{ env.ACCOUNT_NUMBER }}:role/github-actions" | |
| role-session-name: githubactionsrolesession | |
| aws-region: ${{ env.AWS_REGION }} | |
| # get PR number: for posting PR comments and figuring out changed files | |
| - name: Get PR number on push | |
| id: get_pr_number | |
| if: ${{ github.event_name == 'push' }} | |
| run: | | |
| pr_number=$(curl -sS \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ | |
| https://api.github.com/repos/${{ github.repository }}/commits/${{ github.sha }}/pulls \ | |
| | jq -r '.[0].number') | |
| if [[ $pr_number == "null" ]]; then | |
| echo "Could not find PR number for commit=${{ github.sha }}" | |
| exit 1 | |
| fi | |
| echo "pr=${pr_number}" >> $GITHUB_OUTPUT | |
| # for workflow dispatch, check valid project specified | |
| - name: Validate workflow dispatch | |
| id: projects_workflow_dispatch | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| echo "Validate [projects=${{ github.event.inputs.projects }}] [action=${{ github.event.inputs.action }}]" | |
| set +o pipefail | |
| projects=${{ github.event.inputs.projects }} | |
| if [[ ${projects} == "all" ]]; then | |
| projects=$(find "${TEAM_DIR}" -name terragrunt.hcl | grep -v '/.terra' | sed -r 's|/[^/]+$||' | cut -d/ -f${PROJECT_DIR_DEPTH}- | sort -u) | |
| else | |
| for project in ${{ github.event.inputs.projects }}; do | |
| if [[ ! -e "${TEAM_DIR}/${project}/terragrunt.hcl" ]]; then | |
| echo "Project not found ${TEAM_DIR}/${project}/terragrunt.hcl" >&2 | |
| exit 1 | |
| fi | |
| done | |
| fi | |
| action=${{ github.event.inputs.action }} | |
| if [[ $action != "plan" && $action != "apply" && $action != "driftcheck" ]]; then | |
| echo "Unexpected value for $action [$action], must be plan/apply/driftcheck" >&2 | |
| exit 1 | |
| fi | |
| echo "action=[$action]; list of projects" [${projects}] | |
| echo "projects=${projects}" >> $GITHUB_OUTPUT | |
| echo "action=${action}" >> $GITHUB_OUTPUT | |
| - name: Get list of committed files | |
| id: files | |
| if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} | |
| run: | | |
| if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then | |
| pr_number="${{ github.event.pull_request.number }}" | |
| elif [[ "${GITHUB_EVENT_NAME}" == "push" ]]; then | |
| pr_number="${{ steps.get_pr_number.outputs.pr }}" | |
| else | |
| echo "Unexpected github event name ${GITHUB_EVENT_NAME}" | |
| exit 1 | |
| fi | |
| echo "pr_number=${pr_number}" >> $GITHUB_OUTPUT | |
| pr_files=$(curl -sS \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ | |
| https://api.github.com/repos/${{ github.repository }}/pulls/${pr_number}/files \ | |
| | jq .[].filename \ | |
| | sed 's/"//g' \ | |
| | tr '\n' ' ') | |
| echo "file_list=${pr_files}" >> $GITHUB_OUTPUT | |
| - name: Detect pull/push projects | |
| id: projects_pull_push | |
| if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} | |
| run: | | |
| updated_files="${{ steps.files.outputs.file_list }}" | |
| echo "Updated files: ${updated_files[@]}" | |
| set +o pipefail | |
| allprojects=$(find ${TEAM_DIR} -name terragrunt.hcl | grep -v '/.terra' | grep -v "${TEAM_DIR}/terragrunt.hcl" | sed -r "s;${TEAM_DIR}/([^/]+)/.+;\1;" | sort -u) | |
| echo "all projects: ${allprojects[@]}" | |
| changed=$(echo "${{ steps.files.outputs.file_list }}" | tr "[:space:]" "\n" | grep -v "${TEAM_DIR}/components/" | sed -r 's|/[^/]+$||' | grep "${TEAM_DIR}/" | sed -r "s;${TEAM_DIR}/([^/]+);\1;g" | sort -u) | |
| workflow_path=.github/workflows/${GITHUB_WORKFLOW}.yml | |
| common_files=( $(ls -p ${TEAM_DIR} | grep -v /) ) # base dir files | |
| common_files=( "${common_files[@]/#/${TEAM_DIR}\/}" ) # prefix with base dir | |
| common_files+=("${workflow_path}") # add workflow to common files | |
| common_files+=("modules/imagebuilder/.+") | |
| echo "common files: ${common_files[@]}" | |
| projects=$(comm -12 <(echo "${changed}") <(echo "${allprojects}")) | |
| for file in ${common_files[@]}; do | |
| if [[ " ${updated_files[*]} " =~ " ${file} " ]]; then | |
| projects=${allprojects[@]} | |
| fi | |
| done | |
| echo List of projects [ ${projects} ] | |
| echo projects=${projects} >> $GITHUB_OUTPUT | |
| - name: Parse inputs | |
| id: parseinput | |
| run: | | |
| echo "Parsing input parameters event=${GITHUB_EVENT_NAME}" | |
| action="plan" | |
| pr_number="" | |
| if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then | |
| projects="${{ steps.projects_workflow_dispatch.outputs.projects }}" | |
| action="${{ steps.projects_workflow_dispatch.outputs.action }}" | |
| elif [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then | |
| projects="${{ steps.projects_pull_push.outputs.projects }}" | |
| pr_number="${{ github.event.pull_request.number }}" | |
| elif [[ "${GITHUB_EVENT_NAME}" == "push" ]]; then | |
| projects="${{ steps.projects_pull_push.outputs.projects }}" | |
| pr_number="${{ steps.get_pr_number.outputs.pr }}" | |
| action="apply" | |
| else | |
| echo "Unsupported event ${GITHUB_EVENT_NAME}" | |
| exit 1 | |
| fi | |
| echo "Set action=${action} pr_number=${pr_number} project=${projects}" | |
| echo "projects=${projects}" >> $GITHUB_OUTPUT | |
| echo "action=${action}" >> $GITHUB_OUTPUT | |
| echo "pr_number=${pr_number}" >> $GITHUB_OUTPUT | |
| - name: Setup project matrix | |
| id: setupproject | |
| env: | |
| projects: ${{ steps.parseinput.outputs.projects }} | |
| run: | | |
| echo "Setup ansible actions for projects [${projects}]" | |
| echo -n "matrix={\"include\":[" >> $GITHUB_OUTPUT | |
| delimiter="" | |
| for project in ${projects}; do | |
| echo -n "${delimiter}" >> $GITHUB_OUTPUT | |
| echo -n '{"project":"'${project}'"}' >> $GITHUB_OUTPUT | |
| delimiter="," | |
| done | |
| echo "]}" >> $GITHUB_OUTPUT | |
| - name: Hide previous PR comments | |
| if: ${{ github.event_name == 'pull_request' }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| COMMENT_BODY_CONTAINS: "**`${{ env.TEAM_DIR }}/" | |
| PR_NUMBER: "${{ github.event.pull_request.number }}" | |
| run: | | |
| cd ${GITHUB_WORKSPACE}/scripts/minimise-comments | |
| go build | |
| ./minimise-comments | |
| plan: | |
| needs: init | |
| if: ${{ needs.init.outputs.projects != '' }} | |
| strategy: | |
| matrix: ${{ fromJson(needs.init.outputs.matrix) }} | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout the code | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 | |
| - name: Set Account Number | |
| run: echo "ACCOUNT_NUMBER=$(jq -r -e '.modernisation_platform_account_id' <<< $ENVIRONMENT_MANAGEMENT)" >> $GITHUB_ENV | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 | |
| with: | |
| role-to-assume: "arn:aws:iam::${{ env.ACCOUNT_NUMBER }}:role/github-actions" | |
| role-session-name: githubactionsrolesession | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get terraform version | |
| id: discover | |
| run: | | |
| echo "Get terraform version for ${{ matrix.project }}" | |
| set +o pipefail | |
| required_version=$(grep ^terraform "${TEAM_DIR}/${{ matrix.project }}"/*.tf -A 20 | grep -w required_version | cut -d\" -f2) | |
| if [[ -z "$required_version" ]]; then | |
| echo "Using default terraform version specified in pipeline $DEFAULT_TERRAFORM_VERSION" >&2 | |
| required_version="=${DEFAULT_TERRAFORM_VERSION}" | |
| fi | |
| if [[ ! "$required_version" =~ =[0-9]+.[0-9]+.[0-9]+ ]]; then | |
| echo "Unexpected terraform required_version format, expect '=x.y.z': $required_version" >&2 | |
| exit 1 | |
| fi | |
| version=$(echo "$required_version" | cut -d= -f2) | |
| echo "version=${version}" >> $GITHUB_OUTPUT | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3 | |
| with: | |
| terraform_version: ${{ steps.discover.outputs.version }} | |
| terraform_wrapper: false | |
| - name: Setup Terragrunt | |
| run: | | |
| wget -O /usr/local/bin/terragrunt https://github.com/gruntwork-io/terragrunt/releases/download/${TERRAGRUNT_VERSION}/terragrunt_linux_amd64 | |
| chmod u+x /usr/local/bin/terragrunt | |
| - name: Init | |
| working-directory: "${{ env.TEAM_DIR }}/${{ matrix.project }}" | |
| run: | | |
| terragrunt init | |
| terragrunt workspace select "core-shared-services-${TF_ENV}" || terragrunt workspace new "core-shared-services-${TF_ENV}" | |
| - name: Plan | |
| id: plan | |
| working-directory: "${{ env.TEAM_DIR }}/${{ matrix.project }}" | |
| run: | | |
| exitcode=0 | |
| chmod +x ${GITHUB_WORKSPACE}/scripts/redact-output.sh | |
| terragrunt plan -no-color -detailed-exitcode | ${GITHUB_WORKSPACE}/scripts/redact-output.sh | tee tfplan.txt || exitcode=$? | |
| echo "terragrunt plan exit code = $exitcode" | |
| echo "exitcode=${exitcode}" >> $GITHUB_OUTPUT | |
| (( exitcode == 1 )) && exit 1 || exit 0 | |
| - name: Create Plan PR message | |
| if: ${{ (github.event_name == 'pull_request' || github.event_name == 'push') && steps.plan.outputs.exitcode == '2' }} | |
| working-directory: "${{ env.TEAM_DIR }}/${{ matrix.project }}" | |
| run: | | |
| comment() { | |
| url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| len=$(cat tfplan.txt | wc -c) | |
| echo '**`${{ env.TEAM_DIR }}/${{ matrix.project }}`** terragrunt plan on `${{ github.event_name }}` event [#${{ github.run_number }}](${url})' | |
| echo | |
| echo '```' | |
| head -c 65476 tfplan.txt | |
| echo | |
| echo '```' | |
| if [[ $len -gt 65476 ]]; then | |
| echo "** Truncated output. See $url for the rest **" | |
| fi | |
| } | |
| echo 'TF_PLAN_OUT<<EOF' >> $GITHUB_ENV | |
| comment >> $GITHUB_ENV | |
| echo 'EOF' >> $GITHUB_ENV | |
| - name: Post Plan to PR | |
| env: | |
| message: "${{ env.TF_PLAN_OUT }}" | |
| pr_number: "${{ needs.init.outputs.pr_number }}" | |
| run: | | |
| escaped_message=$(echo "$message" | jq -Rsa .) | |
| curl -sS -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ | |
| "https://api.github.com/repos/${{ github.repository }}/issues/${pr_number}/comments" \ | |
| -d '{"body":'"${escaped_message}"'}' | |
| apply: | |
| needs: | |
| - init | |
| - plan | |
| if: ${{ needs.init.outputs.projects != '' && needs.init.outputs.action == 'apply' }} | |
| strategy: | |
| matrix: ${{ fromJson(needs.init.outputs.matrix) }} | |
| fail-fast: false | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: core-shared-services | |
| steps: | |
| - name: Checkout the code | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 | |
| - name: Set Account Number | |
| run: echo "ACCOUNT_NUMBER=$(jq -r -e '.modernisation_platform_account_id' <<< $ENVIRONMENT_MANAGEMENT)" >> $GITHUB_ENV | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v4.0.0 | |
| with: | |
| role-to-assume: "arn:aws:iam::${{ env.ACCOUNT_NUMBER }}:role/github-actions" | |
| role-session-name: githubactionsrolesession | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Get terraform version | |
| id: discover | |
| run: | | |
| echo "Get terraform version for ${{ matrix.project }}" | |
| set +o pipefail | |
| required_version=$(grep ^terraform "${TEAM_DIR}/${{ matrix.project }}"/*.tf -A 20 | grep -w required_version | cut -d\" -f2) | |
| if [[ -z "$required_version" ]]; then | |
| echo "Using default terraform version specified in pipeline $DEFAULT_TERRAFORM_VERSION" >&2 | |
| required_version="=${DEFAULT_TERRAFORM_VERSION}" | |
| fi | |
| if [[ ! "$required_version" =~ =[0-9]+.[0-9]+.[0-9]+ ]]; then | |
| echo "Unexpected terraform required_version format, expect '=x.y.z': $required_version" >&2 | |
| exit 1 | |
| fi | |
| version=$(echo "$required_version" | cut -d= -f2) | |
| echo "version=${version}" >> $GITHUB_OUTPUT | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3 | |
| with: | |
| terraform_version: ${{ steps.discover.outputs.version }} | |
| terraform_wrapper: false | |
| - name: Setup Terragrunt | |
| run: | | |
| wget -O /usr/local/bin/terragrunt https://github.com/gruntwork-io/terragrunt/releases/download/${TERRAGRUNT_VERSION}/terragrunt_linux_amd64 | |
| chmod u+x /usr/local/bin/terragrunt | |
| - name: Init | |
| working-directory: "${{ env.TEAM_DIR }}/${{ matrix.project }}" | |
| run: | | |
| terragrunt init | |
| terragrunt workspace select "core-shared-services-${TF_ENV}" || terragrunt workspace new "core-shared-services-${TF_ENV}" | |
| - name: Plan | |
| id: plan | |
| working-directory: "${{ env.TEAM_DIR }}/${{ matrix.project }}" | |
| run: | | |
| exitcode=0 | |
| chmod +x ${GITHUB_WORKSPACE}/scripts/redact-output.sh | |
| terragrunt plan -detailed-exitcode -no-color -out=tf.plan | ${GITHUB_WORKSPACE}/scripts/redact-output.sh || exitcode=$? | |
| echo "terragrunt plan exit code = $exitcode" | |
| echo "exitcode=${exitcode}" >> $GITHUB_OUTPUT | |
| (( exitcode == 1 )) && exit 1 || exit 0 | |
| - name: Apply | |
| if: ${{ steps.plan.outputs.exitcode == '2' }} | |
| working-directory: "${{ env.TEAM_DIR }}/${{ matrix.project }}" | |
| run: | | |
| terragrunt apply tf.plan -no-color | ${GITHUB_WORKSPACE}/scripts/redact-output.sh | tee tfapply.txt | |
| - name: Create Apply PR message | |
| if: ${{ github.event_name == 'push' && steps.plan.outputs.exitcode == '2' }} | |
| working-directory: "${{ env.TEAM_DIR }}/${{ matrix.project }}" | |
| run: | | |
| comment() { | |
| url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| len=$(cat tfapply.txt | wc -c) | |
| echo '**`${{ env.TEAM_DIR }}/${{ matrix.project }}`** terraform apply on `${{ github.event_name }}` event [#${{ github.run_number }}](${url})' | |
| echo | |
| echo '```' | |
| head -c 65476 tfapply.txt | |
| echo | |
| echo '```' | |
| if [[ $len -gt 65476 ]]; then | |
| echo "** Truncated output. See $url for the rest **" | |
| fi | |
| } | |
| echo 'TF_APPLY_OUT<<EOF' >> $GITHUB_ENV | |
| comment >> $GITHUB_ENV | |
| echo 'EOF' >> $GITHUB_ENV | |
| - name: Post Apply to PR | |
| if: ${{ github.event_name == 'push' && steps.plan.outputs.exitcode == '2' }} | |
| env: | |
| message: "${{ env.TF_APPLY_OUT }}" | |
| pr_number: "${{ needs.init.outputs.pr_number }}" | |
| run: | | |
| escaped_message=$(echo "$message" | jq -Rsa .) | |
| curl -sS -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ | |
| "https://api.github.com/repos/${{ github.repository }}/issues/${pr_number}/comments" \ | |
| -d '{"body":'"${escaped_message}"'}' |