From 00d39d0692f487df7664d5af669fd680ced111d6 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 26 Sep 2023 00:19:13 +0000 Subject: [PATCH 01/57] feat: add event trigger for when pre-releases or releases are published github.ref_name covers getting either branch _or tag_ name, so with this commit, a run of this workflow would publish a Docker tag with the git tag name --- .github/workflows/docker-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 70c569a0..674defda 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -7,6 +7,9 @@ on: - dev - test - prod + release: + types: + - published defaults: run: From ade17c6d81a6b0f603b8b7a68d3516bc572cefd4 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 26 Sep 2023 23:04:30 +0000 Subject: [PATCH 02/57] refactor: make pre-releases/releases use the correct deploy environment --- .github/workflows/docker-publish.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 674defda..2c7dd58a 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,8 +18,9 @@ defaults: jobs: deploy: runs-on: ubuntu-latest - environment: ${{ github.ref_name }} - concurrency: ${{ github.ref_name }} + # this expression works like a ternary operation (see https://github.com/actions/runner/issues/409#issuecomment-727565588) + environment: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} + concurrency: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} steps: - name: Checkout From d613425cad470fb4513edfa69d894a97f26d8eb8 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 26 Sep 2023 17:25:01 +0000 Subject: [PATCH 03/57] refactor: move concurrency config to workflow level instead of job level also set 'cancel-in-progress' to true so that pending runs will cancel if a newer run is queued for a given concurrency group --- .github/workflows/docker-publish.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 2c7dd58a..a47b1974 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,12 +15,15 @@ defaults: run: shell: bash +concurrency: + # this expression works like a ternary operation (see https://github.com/actions/runner/issues/409#issuecomment-727565588) + group: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} + cancel-in-progress: true + jobs: deploy: runs-on: ubuntu-latest - # this expression works like a ternary operation (see https://github.com/actions/runner/issues/409#issuecomment-727565588) environment: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} - concurrency: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} steps: - name: Checkout From c93ac27c3324411129ee3e38971131bb69fa9aee Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 28 Sep 2023 14:52:25 +0000 Subject: [PATCH 04/57] feat: make git hash available in static file --- .github/workflows/docker-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 70c569a0..a2391775 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -22,6 +22,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Write commit SHA to file + run: echo "${{ github.sha }}" >> eligibility_server/static/sha.txt + - name: Docker Login to GitHub Container Registry uses: docker/login-action@v3 with: From 340f2b24689b3472a3e546d9f24181884b2726a0 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 28 Sep 2023 15:13:33 +0000 Subject: [PATCH 05/57] feat: pipdeptree to a static file --- .github/workflows/docker-publish.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a2391775..1071e3c3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -22,6 +22,21 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version-file: .github/workflows/.python-version + cache: pip + cache-dependency-path: "**/pyproject.toml" + + - name: Write python packages to file + run: | + python -m venv .venv + source .venv/bin/activate + pip install pipdeptree + pip install -e . + pipdeptree + pipdeptree >> eligibility_server/static/requirements.txt + - name: Write commit SHA to file run: echo "${{ github.sha }}" >> eligibility_server/static/sha.txt From c3c010c2a60a93b87fb31f1d883d5fa6481f4e28 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Fri, 29 Sep 2023 00:03:22 +0000 Subject: [PATCH 06/57] fix: add step to create static directory needed for various static files --- .github/workflows/docker-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 1071e3c3..7179f172 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -28,6 +28,9 @@ jobs: cache: pip cache-dependency-path: "**/pyproject.toml" + - name: Create /static directory + run: mkdir -p eligibility_server/static + - name: Write python packages to file run: | python -m venv .venv From 9218a012bcee07fc8611fb2e08d7add32bdfc709 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Wed, 27 Sep 2023 17:55:18 +0000 Subject: [PATCH 07/57] feat: run tests before deploying if triggered by pre-release/release --- .github/workflows/docker-publish.yml | 6 ++++++ .github/workflows/run-tests.yml | 1 + 2 files changed, 7 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a47b1974..cbc50c20 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -21,7 +21,13 @@ concurrency: cancel-in-progress: true jobs: + test: + uses: ./.github/workflows/run-tests.yml + if: github.event_name == 'release' && (github.event.release.prerelease || !github.event.release.prerelease) + deploy: + needs: test + if: always() && !cancelled() && (github.event_name != 'release' || needs.test.result == 'success') runs-on: ubuntu-latest environment: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 2cc3ee2a..efb9c7b2 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,6 +1,7 @@ name: Run pytests on: + workflow_call: pull_request: branches: ["*"] From 07a2e2e5f7d8c86c7cb0ab06e9e48ce5a5338bdd Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Fri, 29 Sep 2023 19:28:28 +0000 Subject: [PATCH 08/57] refactor: remove unneeded conditions from test if expression the workflow itself is already filtered to only listen to release publishes --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index cbc50c20..47928699 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,7 +23,7 @@ concurrency: jobs: test: uses: ./.github/workflows/run-tests.yml - if: github.event_name == 'release' && (github.event.release.prerelease || !github.event.release.prerelease) + if: github.event_name == 'release' deploy: needs: test From 1ec809954a050c1a3f535843d3a2b784b253a8c3 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Fri, 29 Sep 2023 20:08:35 +0000 Subject: [PATCH 09/57] refactor: extract out deploy-environment name as environment variable --- .github/workflows/docker-publish.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 47928699..d721886d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,9 +15,12 @@ defaults: run: shell: bash -concurrency: +env: # this expression works like a ternary operation (see https://github.com/actions/runner/issues/409#issuecomment-727565588) - group: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} + - DEPLOYMENT_ENVIRONMENT: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} + +concurrency: + group: ${{ env.DEPLOYMENT_ENVIRONMENT }} cancel-in-progress: true jobs: @@ -29,7 +32,7 @@ jobs: needs: test if: always() && !cancelled() && (github.event_name != 'release' || needs.test.result == 'success') runs-on: ubuntu-latest - environment: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} + environment: ${{ env.DEPLOYMENT_ENVIRONMENT }} steps: - name: Checkout From f36ca19927d12906481545023626864d6a227971 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Fri, 29 Sep 2023 20:10:16 +0000 Subject: [PATCH 10/57] feat: create Docker tag with deployment environment name --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d721886d..ba3353cf 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -60,5 +60,6 @@ jobs: file: Dockerfile push: true tags: | + ghcr.io/${{ github.repository }}:${{ env.DEPLOYMENT_ENVIRONMENT }} ghcr.io/${{ github.repository }}:${{ github.ref_name }} ghcr.io/${{ github.repository }}:${{ github.sha }} From 659c5c80f9fc658718c8fc1abb55b2c8c42c83f2 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Fri, 29 Sep 2023 21:24:08 +0000 Subject: [PATCH 11/57] refactor: remove unneeded and redundant usage of always() --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ba3353cf..548a1dc5 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -30,7 +30,7 @@ jobs: deploy: needs: test - if: always() && !cancelled() && (github.event_name != 'release' || needs.test.result == 'success') + if: (!cancelled() && (github.event_name != 'release' || needs.test.result == 'success')) runs-on: ubuntu-latest environment: ${{ env.DEPLOYMENT_ENVIRONMENT }} From 0586d7352cc0bbe111d523ffc8b8112d2544646c Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Fri, 29 Sep 2023 21:57:19 +0000 Subject: [PATCH 12/57] chore: add comment explaining 'deploy' job condition --- .github/workflows/docker-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 548a1dc5..308cbd3b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -30,6 +30,8 @@ jobs: deploy: needs: test + # !cancelled() is needed because if the whole workflow was cancelled, we don't want this job to run. + # (github.event_name != 'release' || needs.test.result == 'success') is needed because if `test` did run, we only want this to run if `test` succeeded. if: (!cancelled() && (github.event_name != 'release' || needs.test.result == 'success')) runs-on: ubuntu-latest environment: ${{ env.DEPLOYMENT_ENVIRONMENT }} From 874473ac04214685eb366effd3f13856387a656c Mon Sep 17 00:00:00 2001 From: Kegan Maher Date: Fri, 29 Sep 2023 15:01:40 -0700 Subject: [PATCH 13/57] feat(frontdoor): allow access to static files dir maintain access to healthcheck endpoint --- terraform/front_door.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/front_door.tf b/terraform/front_door.tf index f5106559..5500daac 100644 --- a/terraform/front_door.tf +++ b/terraform/front_door.tf @@ -74,7 +74,7 @@ resource "azurerm_cdn_frontdoor_firewall_policy" "main" { custom_block_response_body = base64encode("Forbidden") custom_rule { - name = "healthcheck" + name = "public_access" enabled = true type = "MatchRule" priority = 1 @@ -82,8 +82,8 @@ resource "azurerm_cdn_frontdoor_firewall_policy" "main" { match_condition { match_variable = "RequestUri" - operator = "Equal" - match_values = ["https://${azurerm_cdn_frontdoor_endpoint.main.host_name}:443/healthcheck"] + operator = "BeginsWith" + match_values = ["https://${azurerm_cdn_frontdoor_endpoint.main.host_name}:443/healthcheck", "https://${azurerm_cdn_frontdoor_endpoint.main.host_name}:443/static/"] } } From e1b4e08d268da1011311df80a32f1b1275736e7b Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Mon, 2 Oct 2023 16:18:18 +0000 Subject: [PATCH 14/57] fix: 'env' context is not available in the places we need it put the expression back as inline values instead of being a variable --- .github/workflows/docker-publish.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 3a19e5f7..d7c2af0e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -15,12 +15,9 @@ defaults: run: shell: bash -env: - # this expression works like a ternary operation (see https://github.com/actions/runner/issues/409#issuecomment-727565588) - - DEPLOYMENT_ENVIRONMENT: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} - concurrency: - group: ${{ env.DEPLOYMENT_ENVIRONMENT }} + # this expression gives us the name of the deployment environment. It works like a ternary operation (see https://github.com/actions/runner/issues/409#issuecomment-727565588) + group: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} cancel-in-progress: true jobs: @@ -34,7 +31,7 @@ jobs: # (github.event_name != 'release' || needs.test.result == 'success') is needed because if `test` did run, we only want this to run if `test` succeeded. if: (!cancelled() && (github.event_name != 'release' || needs.test.result == 'success')) runs-on: ubuntu-latest - environment: ${{ env.DEPLOYMENT_ENVIRONMENT }} + environment: ${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} steps: - name: Checkout @@ -83,6 +80,6 @@ jobs: file: Dockerfile push: true tags: | - ghcr.io/${{ github.repository }}:${{ env.DEPLOYMENT_ENVIRONMENT }} + ghcr.io/${{ github.repository }}:${{ github.event_name != 'release' && github.ref_name || github.event.release.prerelease && 'test' || 'prod' }} ghcr.io/${{ github.repository }}:${{ github.ref_name }} ghcr.io/${{ github.repository }}:${{ github.sha }} From f530f364e44bf1936f775ae00b546ac3c234fb71 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 20:39:18 +0000 Subject: [PATCH 15/57] fix: custom rule names can only consist of letters and numbers --- terraform/front_door.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/front_door.tf b/terraform/front_door.tf index 5500daac..772c9ee2 100644 --- a/terraform/front_door.tf +++ b/terraform/front_door.tf @@ -74,7 +74,7 @@ resource "azurerm_cdn_frontdoor_firewall_policy" "main" { custom_block_response_body = base64encode("Forbidden") custom_rule { - name = "public_access" + name = "publicaccess" enabled = true type = "MatchRule" priority = 1 From f9ecd71b5f3965fe2d204338d28759a1876c5ccb Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 21:47:11 +0000 Subject: [PATCH 16/57] refactor: move pipeline stages out into template --- terraform/pipeline/azure-pipelines.yml | 149 +------------------------ terraform/pipeline/deploy.yml | 149 +++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 148 deletions(-) create mode 100644 terraform/pipeline/deploy.yml diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/pipeline/azure-pipelines.yml index 16b7aa3c..dc04ced3 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/pipeline/azure-pipelines.yml @@ -25,151 +25,4 @@ pool: vmImage: ubuntu-latest stages: - - stage: TerraformPlan - jobs: - - job: Plan - variables: - - name: OTHER_SOURCE - value: $[variables['System.PullRequest.SourceBranch']] - - name: INDIVIDUAL_SOURCE - value: $[variables['Build.SourceBranchName']] - - name: IS_TAG - value: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/')] - - name: TARGET - value: $[variables['System.PullRequest.TargetBranch']] - steps: - # set the workspace variable at runtime (rather than build time) so that all the necessary variables are available, and we can use Python - # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/set-variables-scripts?view=azure-devops&tabs=bash#about-tasksetvariable - - bash: | - WORKSPACE=$(python terraform/pipeline/workspace.py) - echo "##vso[task.setvariable variable=workspace;isOutput=true]$WORKSPACE" - - TAG_TYPE=$(python terraform/pipeline/tag.py) - echo "##vso[task.setvariable variable=tag_type;isOutput=true]$TAG_TYPE" - name: setvars - displayName: Determine deployment environment - env: - REASON: $(Build.Reason) - # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformInstaller#readme - - task: TerraformInstaller@0 - displayName: Install Terraform - inputs: - terraformVersion: 1.3.1 - # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformTask/TerraformTaskV3#readme - - task: TerraformTaskV3@3 - displayName: Terraform init - inputs: - provider: azurerm - command: init - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: -input=false - # service connection - backendServiceArm: deployer - # needs to match main.tf - backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform - backendAzureRmStorageAccountName: courtesycardsterraform - backendAzureRmContainerName: tfstate - backendAzureRmKey: terraform.tfstate - - task: TerraformTaskV3@3 - displayName: Select environment - inputs: - provider: azurerm - command: custom - customCommand: workspace - commandOptions: select $(setvars.workspace) - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # service connection - environmentServiceNameAzureRM: deployer - - task: TerraformTaskV3@3 - displayName: Terraform plan - inputs: - provider: azurerm - command: plan - # wait for lock to be released, in case being used by another pipeline run - # https://discuss.hashicorp.com/t/terraform-plan-wait-for-lock-to-be-released/6870/2 - commandOptions: -input=false -lock-timeout=5m -out=$(Build.ArtifactStagingDirectory)/tfplan - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # service connection - environmentServiceNameAzureRM: deployer - # need to publish the tfplan to used by next stage if it's going to run - - publish: "$(Build.ArtifactStagingDirectory)" - displayName: "Publish tfplan for use in TerraformApply" - artifact: savedPlan - condition: | - or( - in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), - eq(variables['setvars.tag_type'], 'test'), - eq(variables['setvars.tag_type'], 'prod') - ) - - stage: TerraformApply - dependsOn: TerraformPlan - variables: - - name: workspace - value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.workspace'] ] - - name: tag_type - value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.tag_type'] ] - # only run on dev, test, or prod branches OR if it's a tag for test or prod - condition: | - or( - in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), - eq(variables['tag_type'], 'test'), - eq(variables['tag_type'], 'prod') - ) - jobs: - - deployment: Apply - condition: succeeded() - environment: Approval - variables: - - name: workspace - value: $[ stageDependencies.TerraformPlan.Plan.outputs['setvars.workspace'] ] - - name: tag_type - value: $[ stageDependencies.TerraformPlan.Plan.outputs['setvars.tag_type'] ] - strategy: - runOnce: - deploy: - steps: - - checkout: self - - download: current - displayName: "Download plan file published from TerraformPlan" - artifact: savedPlan - - task: TerraformInstaller@0 - displayName: Install Terraform - inputs: - terraformVersion: 1.3.1 - # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformTask/TerraformTaskV3#readme - - task: TerraformTaskV3@3 - displayName: Terraform init - inputs: - provider: azurerm - command: init - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: -input=false - # service connection - backendServiceArm: deployer - # needs to match main.tf - backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform - backendAzureRmStorageAccountName: courtesycardsterraform - backendAzureRmContainerName: tfstate - backendAzureRmKey: terraform.tfstate - - task: TerraformTaskV3@3 - displayName: Select environment - inputs: - provider: azurerm - command: custom - customCommand: workspace - commandOptions: select $(workspace) - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # service connection - environmentServiceNameAzureRM: deployer - - task: TerraformTaskV3@3 - displayName: Terraform apply - inputs: - provider: azurerm - command: apply - # (ditto the lock comment above) - commandOptions: -input=false -lock-timeout=5m $(Pipeline.Workspace)/savedPlan/tfplan - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # service connection - environmentServiceNameAzureRM: deployer + - template: deploy.yml diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml new file mode 100644 index 00000000..f4b4ece8 --- /dev/null +++ b/terraform/pipeline/deploy.yml @@ -0,0 +1,149 @@ +stages: + - stage: TerraformPlan + jobs: + - job: Plan + variables: + - name: OTHER_SOURCE + value: $[variables['System.PullRequest.SourceBranch']] + - name: INDIVIDUAL_SOURCE + value: $[variables['Build.SourceBranchName']] + - name: IS_TAG + value: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/')] + - name: TARGET + value: $[variables['System.PullRequest.TargetBranch']] + steps: + # set the workspace variable at runtime (rather than build time) so that all the necessary variables are available, and we can use Python + # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/set-variables-scripts?view=azure-devops&tabs=bash#about-tasksetvariable + - bash: | + WORKSPACE=$(python terraform/pipeline/workspace.py) + echo "##vso[task.setvariable variable=workspace;isOutput=true]$WORKSPACE" + + TAG_TYPE=$(python terraform/pipeline/tag.py) + echo "##vso[task.setvariable variable=tag_type;isOutput=true]$TAG_TYPE" + name: setvars + displayName: Determine deployment environment + env: + REASON: $(Build.Reason) + # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformInstaller#readme + - task: TerraformInstaller@0 + displayName: Install Terraform + inputs: + terraformVersion: 1.3.1 + # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformTask/TerraformTaskV3#readme + - task: TerraformTaskV3@3 + displayName: Terraform init + inputs: + provider: azurerm + command: init + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow + commandOptions: -input=false + # service connection + backendServiceArm: deployer + # needs to match main.tf + backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform + backendAzureRmStorageAccountName: courtesycardsterraform + backendAzureRmContainerName: tfstate + backendAzureRmKey: terraform.tfstate + - task: TerraformTaskV3@3 + displayName: Select environment + inputs: + provider: azurerm + command: custom + customCommand: workspace + commandOptions: select $(setvars.workspace) + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # service connection + environmentServiceNameAzureRM: deployer + - task: TerraformTaskV3@3 + displayName: Terraform plan + inputs: + provider: azurerm + command: plan + # wait for lock to be released, in case being used by another pipeline run + # https://discuss.hashicorp.com/t/terraform-plan-wait-for-lock-to-be-released/6870/2 + commandOptions: -input=false -lock-timeout=5m -out=$(Build.ArtifactStagingDirectory)/tfplan + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # service connection + environmentServiceNameAzureRM: deployer + # need to publish the tfplan to used by next stage if it's going to run + - publish: "$(Build.ArtifactStagingDirectory)" + displayName: "Publish tfplan for use in TerraformApply" + artifact: savedPlan + condition: | + or( + in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), + eq(variables['setvars.tag_type'], 'test'), + eq(variables['setvars.tag_type'], 'prod') + ) + - stage: TerraformApply + dependsOn: TerraformPlan + variables: + - name: workspace + value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.workspace'] ] + - name: tag_type + value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.tag_type'] ] + # only run on dev, test, or prod branches OR if it's a tag for test or prod + condition: | + or( + in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), + eq(variables['tag_type'], 'test'), + eq(variables['tag_type'], 'prod') + ) + jobs: + - deployment: Apply + condition: succeeded() + environment: Approval + variables: + - name: workspace + value: $[ stageDependencies.TerraformPlan.Plan.outputs['setvars.workspace'] ] + - name: tag_type + value: $[ stageDependencies.TerraformPlan.Plan.outputs['setvars.tag_type'] ] + strategy: + runOnce: + deploy: + steps: + - checkout: self + - download: current + displayName: "Download plan file published from TerraformPlan" + artifact: savedPlan + - task: TerraformInstaller@0 + displayName: Install Terraform + inputs: + terraformVersion: 1.3.1 + # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformTask/TerraformTaskV3#readme + - task: TerraformTaskV3@3 + displayName: Terraform init + inputs: + provider: azurerm + command: init + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow + commandOptions: -input=false + # service connection + backendServiceArm: deployer + # needs to match main.tf + backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform + backendAzureRmStorageAccountName: courtesycardsterraform + backendAzureRmContainerName: tfstate + backendAzureRmKey: terraform.tfstate + - task: TerraformTaskV3@3 + displayName: Select environment + inputs: + provider: azurerm + command: custom + customCommand: workspace + commandOptions: select $(workspace) + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # service connection + environmentServiceNameAzureRM: deployer + - task: TerraformTaskV3@3 + displayName: Terraform apply + inputs: + provider: azurerm + command: apply + # (ditto the lock comment above) + commandOptions: -input=false -lock-timeout=5m $(Pipeline.Workspace)/savedPlan/tfplan + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # service connection + environmentServiceNameAzureRM: deployer From 26b95e5441eaae213fa8ad97c503692e0f34840c Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 21:53:41 +0000 Subject: [PATCH 17/57] refactor: extract out agency-specific name --- terraform/pipeline/azure-pipelines.yml | 2 ++ terraform/pipeline/deploy.yml | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/pipeline/azure-pipelines.yml index dc04ced3..c1ea4714 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/pipeline/azure-pipelines.yml @@ -26,3 +26,5 @@ pool: stages: - template: deploy.yml + parameters: + agency_card: courtesy-cards diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index f4b4ece8..c4d9921a 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -1,3 +1,13 @@ +parameters: + - name: agency_card + type: string + +variables: + - name: RESOURCE_GROUP + value: ${{ parameters.agency_card }}-eligibility-terraform + - name: STORAGE_ACCOUNT + value: ${{ replace(parameters.agency_card, '-', '') }}terraform + stages: - stage: TerraformPlan jobs: @@ -41,8 +51,8 @@ stages: # service connection backendServiceArm: deployer # needs to match main.tf - backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform - backendAzureRmStorageAccountName: courtesycardsterraform + backendAzureRmResourceGroupName: $(RESOURCE_GROUP) + backendAzureRmStorageAccountName: $(STORAGE_ACCOUNT) backendAzureRmContainerName: tfstate backendAzureRmKey: terraform.tfstate - task: TerraformTaskV3@3 @@ -123,8 +133,8 @@ stages: # service connection backendServiceArm: deployer # needs to match main.tf - backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform - backendAzureRmStorageAccountName: courtesycardsterraform + backendAzureRmResourceGroupName: $(RESOURCE_GROUP) + backendAzureRmStorageAccountName: $(STORAGE_ACCOUNT) backendAzureRmContainerName: tfstate backendAzureRmKey: terraform.tfstate - task: TerraformTaskV3@3 From 393953a81bd100e3d71a45b998f828f4803714b7 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 21:54:40 +0000 Subject: [PATCH 18/57] chore: format template code --- terraform/pipeline/deploy.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index c4d9921a..a7b72be2 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -81,11 +81,11 @@ stages: displayName: "Publish tfplan for use in TerraformApply" artifact: savedPlan condition: | - or( - in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), - eq(variables['setvars.tag_type'], 'test'), - eq(variables['setvars.tag_type'], 'prod') - ) + or( + in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), + eq(variables['setvars.tag_type'], 'test'), + eq(variables['setvars.tag_type'], 'prod') + ) - stage: TerraformApply dependsOn: TerraformPlan variables: From 01b2936216e0c6c529b9e2b37db5ed8384fd0967 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 22:14:12 +0000 Subject: [PATCH 19/57] refactor: variables don't work in templates, try pipeline --- terraform/pipeline/azure-pipelines.yml | 6 +++++- terraform/pipeline/deploy.yml | 18 +++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/pipeline/azure-pipelines.yml index c1ea4714..a6df05fb 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/pipeline/azure-pipelines.yml @@ -24,7 +24,11 @@ pr: pool: vmImage: ubuntu-latest +variables: + agency_card: courtesy-cards + stages: - template: deploy.yml parameters: - agency_card: courtesy-cards + resource_group: $(agency_card)-eligibility-terraform + storage_account: ${{ replace(agency_card, '-', '') }}terraform diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index a7b72be2..bf89e83f 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -1,12 +1,8 @@ parameters: - - name: agency_card + - name: resource_group + type: string + - name: storage_account type: string - -variables: - - name: RESOURCE_GROUP - value: ${{ parameters.agency_card }}-eligibility-terraform - - name: STORAGE_ACCOUNT - value: ${{ replace(parameters.agency_card, '-', '') }}terraform stages: - stage: TerraformPlan @@ -51,8 +47,8 @@ stages: # service connection backendServiceArm: deployer # needs to match main.tf - backendAzureRmResourceGroupName: $(RESOURCE_GROUP) - backendAzureRmStorageAccountName: $(STORAGE_ACCOUNT) + backendAzureRmResourceGroupName: ${{ parameters.resource_group }} + backendAzureRmStorageAccountName: ${{ parameters.storage_account }} backendAzureRmContainerName: tfstate backendAzureRmKey: terraform.tfstate - task: TerraformTaskV3@3 @@ -133,8 +129,8 @@ stages: # service connection backendServiceArm: deployer # needs to match main.tf - backendAzureRmResourceGroupName: $(RESOURCE_GROUP) - backendAzureRmStorageAccountName: $(STORAGE_ACCOUNT) + backendAzureRmResourceGroupName: ${{ parameters.resource_group }} + backendAzureRmStorageAccountName: ${{ parameters.storage_account }} backendAzureRmContainerName: tfstate backendAzureRmKey: terraform.tfstate - task: TerraformTaskV3@3 From 100697e9b42a31879e6669970356faa6eb22691f Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 22:17:41 +0000 Subject: [PATCH 20/57] chore: fix syntax for using variable in expression --- terraform/pipeline/azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/pipeline/azure-pipelines.yml index a6df05fb..7bb3a281 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/pipeline/azure-pipelines.yml @@ -31,4 +31,4 @@ stages: - template: deploy.yml parameters: resource_group: $(agency_card)-eligibility-terraform - storage_account: ${{ replace(agency_card, '-', '') }}terraform + storage_account: ${{ replace(variables.agency_card, '-', '') }}terraform From bc2379a397d37702da963f4c93ff95a4aa51746a Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 22:25:22 +0000 Subject: [PATCH 21/57] chore: prevent this noise from appearing in terraform plan --- terraform/suppress.arm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/suppress.arm.json b/terraform/suppress.arm.json index 95fd5b7d..51bfb6a3 100644 --- a/terraform/suppress.arm.json +++ b/terraform/suppress.arm.json @@ -3,7 +3,7 @@ "contentVersion": "1.0.0.0", "parameters": { "metricAlertID": { - "type": "string" + "type": "String" } }, "resources": [ From 2ea9a89e44378ceadb765fce9569a87454f3c1c6 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 23:08:36 +0000 Subject: [PATCH 22/57] refactor: variablize Terraform state resources --- terraform/main.tf | 4 ++-- terraform/pipeline/azure-pipelines.yml | 4 ++-- terraform/pipeline/deploy.yml | 8 ++++++++ terraform/variables.tf | 10 ++++++++++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 0403446e..2aea8447 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -10,8 +10,8 @@ terraform { backend "azurerm" { # needs to match pipeline/azure-pipelines.yml - resource_group_name = "courtesy-cards-eligibility-terraform" - storage_account_name = "courtesycardsterraform" + resource_group_name = var.TF_RESOURCE_GROUP + storage_account_name = var.TF_STORAGE_ACCOUNT container_name = "tfstate" key = "terraform.tfstate" } diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/pipeline/azure-pipelines.yml index 7bb3a281..1c9a904d 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/pipeline/azure-pipelines.yml @@ -25,10 +25,10 @@ pool: vmImage: ubuntu-latest variables: - agency_card: courtesy-cards + agency_card: mst-courtesy-cards stages: - template: deploy.yml parameters: resource_group: $(agency_card)-eligibility-terraform - storage_account: ${{ replace(variables.agency_card, '-', '') }}terraform + storage_account: ${{ replace(variables.agency_card, '-', '') }}tf diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index bf89e83f..3202b164 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -17,6 +17,10 @@ stages: value: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/')] - name: TARGET value: $[variables['System.PullRequest.TargetBranch']] + - name: TF_VAR_TF_RESOURCE_GROUP + value: ${{ parameters.resource_group }} + - name: TF_VAR_TF_STORAGE_ACCOUNT + value: ${{ parameters.storage_account }} steps: # set the workspace variable at runtime (rather than build time) so that all the necessary variables are available, and we can use Python # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/set-variables-scripts?view=azure-devops&tabs=bash#about-tasksetvariable @@ -89,6 +93,10 @@ stages: value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.workspace'] ] - name: tag_type value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.tag_type'] ] + - name: TF_VAR_TF_RESOURCE_GROUP + value: ${{ parameters.resource_group }} + - name: TF_VAR_TF_STORAGE_ACCOUNT + value: ${{ parameters.storage_account }} # only run on dev, test, or prod branches OR if it's a tag for test or prod condition: | or( diff --git a/terraform/variables.tf b/terraform/variables.tf index f253713c..c62d0495 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -33,3 +33,13 @@ variable "IP_ADDRESS_WHITELIST_PROD" { type = list(string) default = [] } + +variable "TF_RESOURCE_GROUP" { + description = "The name of the resource group used for Terraform backend" + type = string +} + +variable "TF_STORAGE_ACCOUNT" { + description = "The name of the storage account used for Terraform backend" + type = string +} From ee93b21fd21a1e277710e36d73348c6cd85a616e Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 23:39:00 +0000 Subject: [PATCH 23/57] refactor: extract out subscription name from init script --- terraform/init.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/terraform/init.sh b/terraform/init.sh index ece98f21..66add8e7 100755 --- a/terraform/init.sh +++ b/terraform/init.sh @@ -4,14 +4,15 @@ set -e ENV=$1 +SUBSCRIPTION=$2 -if [ $# -ne 1 ]; then - echo "Usage: $0 " +if [ $# -ne 2 ]; then + echo "Usage: $0 " exit 1 fi echo "Setting the subscription for the Azure CLI..." -az account set --subscription="MST IT" +az account set --subscription="$SUBSCRIPTION" printf "Intializing Terraform...\n\n" terraform init From 3803ffa222bfa0be8585eeebdc40ee40c13fc402 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 16:08:30 +0000 Subject: [PATCH 24/57] fix: move agency-specific backend configuration to command options backend configuration cannot use named values such as variables --- terraform/main.tf | 3 --- terraform/pipeline/deploy.yml | 10 ++++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 2aea8447..0343b806 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -9,9 +9,6 @@ terraform { } backend "azurerm" { - # needs to match pipeline/azure-pipelines.yml - resource_group_name = var.TF_RESOURCE_GROUP - storage_account_name = var.TF_STORAGE_ACCOUNT container_name = "tfstate" key = "terraform.tfstate" } diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index 3202b164..70fec352 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -47,7 +47,10 @@ stages: command: init workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: -input=false + commandOptions: | + -input=false \ + -backend-config="resource_group=${{ parameters.resource_group }}" \ + -backend-config="storage_account=${{ parameters.storage_account }}" # service connection backendServiceArm: deployer # needs to match main.tf @@ -133,7 +136,10 @@ stages: command: init workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: -input=false + commandOptions: | + -input=false \ + -backend-config="resource_group=${{ parameters.resource_group }}" \ + -backend-config="storage_account=${{ parameters.storage_account }}" # service connection backendServiceArm: deployer # needs to match main.tf From 8af3d3252eab1b35e78375e94fc44a7760089157 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 17:13:44 +0000 Subject: [PATCH 25/57] feat(terraform): create agency-specific backend config for local dev the pipeline doesn't need the backend config block in main.tf --- terraform/init.sh | 10 ++++++---- terraform/mst/config.azurerm.tfbackend | 2 ++ terraform/mst/env | 1 + terraform/pipeline/deploy.yml | 10 ++-------- 4 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 terraform/mst/config.azurerm.tfbackend create mode 100644 terraform/mst/env diff --git a/terraform/init.sh b/terraform/init.sh index 66add8e7..5f54cc1c 100755 --- a/terraform/init.sh +++ b/terraform/init.sh @@ -3,19 +3,21 @@ set -e -ENV=$1 -SUBSCRIPTION=$2 +ENV="$1" +AGENCY="$2" if [ $# -ne 2 ]; then - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi +source "$AGENCY/env" + echo "Setting the subscription for the Azure CLI..." az account set --subscription="$SUBSCRIPTION" printf "Intializing Terraform...\n\n" -terraform init +terraform init -backend-config="$AGENCY/config.azurerm.tfbackend" printf "\n\nSelecting the Terraform workspace...\n" # matching logic in pipeline/workspace.py diff --git a/terraform/mst/config.azurerm.tfbackend b/terraform/mst/config.azurerm.tfbackend new file mode 100644 index 00000000..39ede935 --- /dev/null +++ b/terraform/mst/config.azurerm.tfbackend @@ -0,0 +1,2 @@ +resource_group_name="mst-courtesy-cards-eligibility-terraform" +storage_account_name="mstcourtesycardstf" diff --git a/terraform/mst/env b/terraform/mst/env new file mode 100644 index 00000000..29896c6d --- /dev/null +++ b/terraform/mst/env @@ -0,0 +1 @@ +SUBSCRIPTION="MST IT" diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index 70fec352..3202b164 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -47,10 +47,7 @@ stages: command: init workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: | - -input=false \ - -backend-config="resource_group=${{ parameters.resource_group }}" \ - -backend-config="storage_account=${{ parameters.storage_account }}" + commandOptions: -input=false # service connection backendServiceArm: deployer # needs to match main.tf @@ -136,10 +133,7 @@ stages: command: init workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: | - -input=false \ - -backend-config="resource_group=${{ parameters.resource_group }}" \ - -backend-config="storage_account=${{ parameters.storage_account }}" + commandOptions: -input=false # service connection backendServiceArm: deployer # needs to match main.tf From e5545dd6c60d3b35f164d4a82b445ed64f863d8d Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 17:15:27 +0000 Subject: [PATCH 26/57] refactor(terraform): create workspace if it doesn't exist --- terraform/init.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/terraform/init.sh b/terraform/init.sh index 5f54cc1c..712a7797 100755 --- a/terraform/init.sh +++ b/terraform/init.sh @@ -20,11 +20,15 @@ printf "Intializing Terraform...\n\n" terraform init -backend-config="$AGENCY/config.azurerm.tfbackend" printf "\n\nSelecting the Terraform workspace...\n" + # matching logic in pipeline/workspace.py -if [ "$ENV" = "prod" ]; then - terraform workspace select default -else - terraform workspace select "$ENV" +WORKSPACE=$([[ "$ENV" == "prod" ]] && echo "default" || echo "$ENV") + +# if the workspace exists, this check will select it +WORKSPACE_EXISTS=$(terraform workspace select "$WORKSPACE" 2> /dev/null; echo $?) +# creating a new workspace also selects it +if [ "$WORKSPACE_EXISTS" -ne 0 ]; then + terraform workspace new "$WORKSPACE" fi echo "Done!" From 3bbc6a7d6db703455eee49d63932f3df3bd187ab Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 17:34:19 +0000 Subject: [PATCH 27/57] refactor(terraform): remove unneeded variables --- terraform/pipeline/deploy.yml | 8 -------- terraform/variables.tf | 10 ---------- 2 files changed, 18 deletions(-) diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index 3202b164..bf89e83f 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -17,10 +17,6 @@ stages: value: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/')] - name: TARGET value: $[variables['System.PullRequest.TargetBranch']] - - name: TF_VAR_TF_RESOURCE_GROUP - value: ${{ parameters.resource_group }} - - name: TF_VAR_TF_STORAGE_ACCOUNT - value: ${{ parameters.storage_account }} steps: # set the workspace variable at runtime (rather than build time) so that all the necessary variables are available, and we can use Python # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/set-variables-scripts?view=azure-devops&tabs=bash#about-tasksetvariable @@ -93,10 +89,6 @@ stages: value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.workspace'] ] - name: tag_type value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.tag_type'] ] - - name: TF_VAR_TF_RESOURCE_GROUP - value: ${{ parameters.resource_group }} - - name: TF_VAR_TF_STORAGE_ACCOUNT - value: ${{ parameters.storage_account }} # only run on dev, test, or prod branches OR if it's a tag for test or prod condition: | or( diff --git a/terraform/variables.tf b/terraform/variables.tf index c62d0495..f253713c 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -33,13 +33,3 @@ variable "IP_ADDRESS_WHITELIST_PROD" { type = list(string) default = [] } - -variable "TF_RESOURCE_GROUP" { - description = "The name of the resource group used for Terraform backend" - type = string -} - -variable "TF_STORAGE_ACCOUNT" { - description = "The name of the storage account used for Terraform backend" - type = string -} From 3ad714ee9782123b8b63649b47d9e3b7fbe0d5a8 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 17:41:36 +0000 Subject: [PATCH 28/57] refactor(terraform): rename local backend config file --- terraform/init.sh | 2 +- terraform/mst/{config.azurerm.tfbackend => local.tfbackend} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename terraform/mst/{config.azurerm.tfbackend => local.tfbackend} (100%) diff --git a/terraform/init.sh b/terraform/init.sh index 712a7797..bdb4bff2 100755 --- a/terraform/init.sh +++ b/terraform/init.sh @@ -17,7 +17,7 @@ echo "Setting the subscription for the Azure CLI..." az account set --subscription="$SUBSCRIPTION" printf "Intializing Terraform...\n\n" -terraform init -backend-config="$AGENCY/config.azurerm.tfbackend" +terraform init -backend-config="$AGENCY/local.tfbackend" printf "\n\nSelecting the Terraform workspace...\n" diff --git a/terraform/mst/config.azurerm.tfbackend b/terraform/mst/local.tfbackend similarity index 100% rename from terraform/mst/config.azurerm.tfbackend rename to terraform/mst/local.tfbackend From f94534bc9f34b1f8d806158aa2575d3b11ae6ded Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 17:45:53 +0000 Subject: [PATCH 29/57] refactor: move agency-specific pipeline into folder, simplify params --- terraform/{pipeline => mst}/azure-pipelines.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) rename terraform/{pipeline => mst}/azure-pipelines.yml (64%) diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/mst/azure-pipelines.yml similarity index 64% rename from terraform/pipeline/azure-pipelines.yml rename to terraform/mst/azure-pipelines.yml index 1c9a904d..90c1b67f 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/mst/azure-pipelines.yml @@ -24,11 +24,8 @@ pr: pool: vmImage: ubuntu-latest -variables: - agency_card: mst-courtesy-cards - stages: - - template: deploy.yml + - template: ../pipeline/deploy.yml parameters: - resource_group: $(agency_card)-eligibility-terraform - storage_account: ${{ replace(variables.agency_card, '-', '') }}tf + resource_group: mst-courtesy-cards-eligibility-terraform + storage_account: mstcourtesycardstf From 8ad23238e3b9ff90e6a28a9ffe7f525641a0ba61 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 18:09:06 +0000 Subject: [PATCH 30/57] chore(pre-commit): autoupdate hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.4.0 → v4.5.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.4.0...v4.5.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3388b308..3b665fea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: stages: [commit-msg] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] From 0a7caab0a809f723684470aaf99ffaa7baadfe2a Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 17:51:49 +0000 Subject: [PATCH 31/57] chore: run terraform fmt --- terraform/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 0343b806..b0877652 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -9,8 +9,8 @@ terraform { } backend "azurerm" { - container_name = "tfstate" - key = "terraform.tfstate" + container_name = "tfstate" + key = "terraform.tfstate" } } From 220bb3966b3c3edad43f5c63dff85129a854982e Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 18:05:48 +0000 Subject: [PATCH 32/57] chore(terraform): update comment documenting where terraform version is --- terraform/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/main.tf b/terraform/main.tf index b0877652..a996a3ec 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,5 +1,5 @@ terraform { - // see version in pipeline/azure-pipelines.yml + // see version in pipeline/deploy.yml required_providers { azurerm = { From bb616285e28d532d13388e8b8b3e46ad3e98d74e Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 18:29:44 +0000 Subject: [PATCH 33/57] feat(terraform): pass variables to Terraform from pipeline pass filepath of variable template as runtime parameter --- terraform/mst/azure-pipelines.yml | 1 + terraform/mst/tfvars.yml | 2 ++ terraform/pipeline/deploy.yml | 5 +++++ 3 files changed, 8 insertions(+) create mode 100644 terraform/mst/tfvars.yml diff --git a/terraform/mst/azure-pipelines.yml b/terraform/mst/azure-pipelines.yml index 90c1b67f..69a5eb21 100644 --- a/terraform/mst/azure-pipelines.yml +++ b/terraform/mst/azure-pipelines.yml @@ -29,3 +29,4 @@ stages: parameters: resource_group: mst-courtesy-cards-eligibility-terraform storage_account: mstcourtesycardstf + variable_file: ../mst/tfvars.yml diff --git a/terraform/mst/tfvars.yml b/terraform/mst/tfvars.yml new file mode 100644 index 00000000..fe92b7bc --- /dev/null +++ b/terraform/mst/tfvars.yml @@ -0,0 +1,2 @@ +variables: + TF_VAR_AGENCY_CARD: 'mst-courtesy-cards' diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index bf89e83f..6e2fa45b 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -3,9 +3,14 @@ parameters: type: string - name: storage_account type: string + - name: variable_file + type: string + stages: - stage: TerraformPlan + variables: + - template: ${{ parameters.variable_file }} jobs: - job: Plan variables: From 9a3a2670d46d3930de1df23e40484477b7a4c2c0 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 20:07:36 +0000 Subject: [PATCH 34/57] feat(terraform): add usage of variable template to TerraformApply stage --- terraform/pipeline/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index 6e2fa45b..3498db32 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -90,6 +90,7 @@ stages: - stage: TerraformApply dependsOn: TerraformPlan variables: + - template: ${{ parameters.variable_file }} - name: workspace value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.workspace'] ] - name: tag_type From 0728b4adbda68a1a2464ed8d090c7d2d1dedc8d3 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 21:00:04 +0000 Subject: [PATCH 35/57] refactor(pipeline): move agency-specific pipeline values into file --- terraform/mst/azure-pipelines.yml | 2 -- terraform/mst/tfvars.yml | 2 ++ terraform/pipeline/deploy.yml | 12 ++++-------- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/terraform/mst/azure-pipelines.yml b/terraform/mst/azure-pipelines.yml index 69a5eb21..4bea08ec 100644 --- a/terraform/mst/azure-pipelines.yml +++ b/terraform/mst/azure-pipelines.yml @@ -27,6 +27,4 @@ pool: stages: - template: ../pipeline/deploy.yml parameters: - resource_group: mst-courtesy-cards-eligibility-terraform - storage_account: mstcourtesycardstf variable_file: ../mst/tfvars.yml diff --git a/terraform/mst/tfvars.yml b/terraform/mst/tfvars.yml index fe92b7bc..4c77fb4d 100644 --- a/terraform/mst/tfvars.yml +++ b/terraform/mst/tfvars.yml @@ -1,2 +1,4 @@ variables: + TF_RESOURCE_GROUP: 'mst-courtesy-cards-eligibility-terraform' + TF_STORAGE_ACCOUNT: 'mstcourtesycardstf' TF_VAR_AGENCY_CARD: 'mst-courtesy-cards' diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index 3498db32..43207958 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -1,8 +1,4 @@ parameters: - - name: resource_group - type: string - - name: storage_account - type: string - name: variable_file type: string @@ -52,8 +48,8 @@ stages: # service connection backendServiceArm: deployer # needs to match main.tf - backendAzureRmResourceGroupName: ${{ parameters.resource_group }} - backendAzureRmStorageAccountName: ${{ parameters.storage_account }} + backendAzureRmResourceGroupName: $(TF_RESOURCE_GROUP) + backendAzureRmStorageAccountName: $(TF_STORAGE_ACCOUNT) backendAzureRmContainerName: tfstate backendAzureRmKey: terraform.tfstate - task: TerraformTaskV3@3 @@ -135,8 +131,8 @@ stages: # service connection backendServiceArm: deployer # needs to match main.tf - backendAzureRmResourceGroupName: ${{ parameters.resource_group }} - backendAzureRmStorageAccountName: ${{ parameters.storage_account }} + backendAzureRmResourceGroupName: $(TF_RESOURCE_GROUP) + backendAzureRmStorageAccountName: $(TF_STORAGE_ACCOUNT) backendAzureRmContainerName: tfstate backendAzureRmKey: terraform.tfstate - task: TerraformTaskV3@3 From 10975081ccaefbef2240c9da9475a563efa8d372 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 22:04:34 +0000 Subject: [PATCH 36/57] refactor: rename local env file to be clearer --- terraform/init.sh | 2 +- terraform/mst/{env => local.env} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename terraform/mst/{env => local.env} (100%) diff --git a/terraform/init.sh b/terraform/init.sh index bdb4bff2..023fc829 100755 --- a/terraform/init.sh +++ b/terraform/init.sh @@ -11,7 +11,7 @@ if [ $# -ne 2 ]; then exit 1 fi -source "$AGENCY/env" +source "$AGENCY/local.env" echo "Setting the subscription for the Azure CLI..." az account set --subscription="$SUBSCRIPTION" diff --git a/terraform/mst/env b/terraform/mst/local.env similarity index 100% rename from terraform/mst/env rename to terraform/mst/local.env From d6d9407fc34c5d40c1521ce631e56e4768471085 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 22:06:00 +0000 Subject: [PATCH 37/57] refactor: rename file that provides Azure vars --- terraform/mst/azure-pipelines.yml | 2 +- terraform/mst/{tfvars.yml => azure-vars.yml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename terraform/mst/{tfvars.yml => azure-vars.yml} (100%) diff --git a/terraform/mst/azure-pipelines.yml b/terraform/mst/azure-pipelines.yml index 4bea08ec..26d663de 100644 --- a/terraform/mst/azure-pipelines.yml +++ b/terraform/mst/azure-pipelines.yml @@ -27,4 +27,4 @@ pool: stages: - template: ../pipeline/deploy.yml parameters: - variable_file: ../mst/tfvars.yml + variable_file: ../mst/azure-vars.yml diff --git a/terraform/mst/tfvars.yml b/terraform/mst/azure-vars.yml similarity index 100% rename from terraform/mst/tfvars.yml rename to terraform/mst/azure-vars.yml From 3e58216de3c7aad4d5edbf3a3edc2c41e80e7fe5 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 22:09:56 +0000 Subject: [PATCH 38/57] chore: remove outdated comments, add comment clarifying weird path --- terraform/mst/azure-pipelines.yml | 1 + terraform/pipeline/deploy.yml | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/terraform/mst/azure-pipelines.yml b/terraform/mst/azure-pipelines.yml index 26d663de..01564acf 100644 --- a/terraform/mst/azure-pipelines.yml +++ b/terraform/mst/azure-pipelines.yml @@ -27,4 +27,5 @@ pool: stages: - template: ../pipeline/deploy.yml parameters: + # path is relative to the template path (see line above) variable_file: ../mst/azure-vars.yml diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index 43207958..e6a010d6 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -47,7 +47,6 @@ stages: commandOptions: -input=false # service connection backendServiceArm: deployer - # needs to match main.tf backendAzureRmResourceGroupName: $(TF_RESOURCE_GROUP) backendAzureRmStorageAccountName: $(TF_STORAGE_ACCOUNT) backendAzureRmContainerName: tfstate @@ -130,7 +129,6 @@ stages: commandOptions: -input=false # service connection backendServiceArm: deployer - # needs to match main.tf backendAzureRmResourceGroupName: $(TF_RESOURCE_GROUP) backendAzureRmStorageAccountName: $(TF_STORAGE_ACCOUNT) backendAzureRmContainerName: tfstate From af9b5ab36bdd4cda9d992d6733e018690c469500 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 22:26:48 +0000 Subject: [PATCH 39/57] feat: use AGENCY_CARD variable --- terraform/app_service.tf | 2 +- terraform/front_door.tf | 2 +- terraform/uptime.tf | 2 +- terraform/variables.tf | 7 +++++++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/terraform/app_service.tf b/terraform/app_service.tf index bf0a6d67..512b1cc8 100644 --- a/terraform/app_service.tf +++ b/terraform/app_service.tf @@ -12,7 +12,7 @@ locals { resource "azurerm_linux_web_app" "main" { # name needs to be globally unique and is more specific because it's used in the app URL - name = "mst-courtesy-cards-eligibility-server-${local.env_name}" + name = "${var.AGENCY_CARD}-eligibility-server-${local.env_name}" location = data.azurerm_resource_group.main.location resource_group_name = data.azurerm_resource_group.main.name service_plan_id = azurerm_service_plan.main.id diff --git a/terraform/front_door.tf b/terraform/front_door.tf index 772c9ee2..f3cdc1b2 100644 --- a/terraform/front_door.tf +++ b/terraform/front_door.tf @@ -10,7 +10,7 @@ resource "azurerm_cdn_frontdoor_profile" "main" { resource "azurerm_cdn_frontdoor_endpoint" "main" { # used in the front door URL - name = "mst-courtesy-cards-eligibility-server-${local.env_name}" + name = "${var.AGENCY_CARD}-eligibility-server-${local.env_name}" cdn_frontdoor_profile_id = azurerm_cdn_frontdoor_profile.main.id } diff --git a/terraform/uptime.tf b/terraform/uptime.tf index df8f5190..23febae1 100644 --- a/terraform/uptime.tf +++ b/terraform/uptime.tf @@ -5,7 +5,7 @@ module "healthcheck" { action_group_id = azurerm_monitor_action_group.eng_email.id application_insights = azurerm_application_insights.main # not strictly necessary to include the environment name, but helps to make the alerts more clear - name = "mst-courtesy-cards-eligibility-server-${local.env_name}-healthcheck" + name = "${var.AGENCY_CARD}-eligibility-server-${local.env_name}-healthcheck" url = "https://${azurerm_cdn_frontdoor_endpoint.main.host_name}/healthcheck" } diff --git a/terraform/variables.tf b/terraform/variables.tf index f253713c..feb0e364 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,3 +1,5 @@ +# Some of these are defined in /azure-vars.yml. Others are defined in Azure DevOps UI. + # needs to be uppercase "because Azure DevOps will always transform pipeline variables to uppercase environment variables" # https://gaunacode.com/terraform-input-variables-using-azure-devops @@ -33,3 +35,8 @@ variable "IP_ADDRESS_WHITELIST_PROD" { type = list(string) default = [] } + +variable "AGENCY_CARD" { + description = "The name of the agency's card program" + type = string +} From 086aa95f0525da5a329b2d41a421e8641551d747 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 22:35:09 +0000 Subject: [PATCH 40/57] refactor(infra): use AGENCY_CARD variable for environment app service note this is a name change --- terraform/environment.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environment.tf b/terraform/environment.tf index 6daa1a61..2825b7d2 100644 --- a/terraform/environment.tf +++ b/terraform/environment.tf @@ -5,5 +5,5 @@ locals { } data "azurerm_resource_group" "main" { - name = "courtesy-cards-eligibility-${local.env_name}" + name = "${var.AGENCY_CARD}-eligibility-${local.env_name}" } From b33cc738500fc72534bf81070c0962c4c3cfc658 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 22:42:27 +0000 Subject: [PATCH 41/57] refactor(terraform): extract storage account name prefix as variable --- terraform/mst/azure-vars.yml | 1 + terraform/storage.tf | 2 +- terraform/variables.tf | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/terraform/mst/azure-vars.yml b/terraform/mst/azure-vars.yml index 4c77fb4d..c3b24d92 100644 --- a/terraform/mst/azure-vars.yml +++ b/terraform/mst/azure-vars.yml @@ -2,3 +2,4 @@ variables: TF_RESOURCE_GROUP: 'mst-courtesy-cards-eligibility-terraform' TF_STORAGE_ACCOUNT: 'mstcourtesycardstf' TF_VAR_AGENCY_CARD: 'mst-courtesy-cards' + TF_VAR_AGENCY_STORAGE_ACCOUNT_PREFIX: 'mstcceligibility' diff --git a/terraform/storage.tf b/terraform/storage.tf index 0f5b03d6..c751fed3 100644 --- a/terraform/storage.tf +++ b/terraform/storage.tf @@ -1,6 +1,6 @@ resource "azurerm_storage_account" "main" { # name needs to be unique per subscription - name = "mstcceligibility${local.env_name}" + name = "${var.AGENCY_STORAGE_ACCOUNT_PREFIX}${local.env_name}" location = data.azurerm_resource_group.main.location resource_group_name = data.azurerm_resource_group.main.name account_tier = "Standard" diff --git a/terraform/variables.tf b/terraform/variables.tf index feb0e364..976b4a4f 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -40,3 +40,8 @@ variable "AGENCY_CARD" { description = "The name of the agency's card program" type = string } + +variable "AGENCY_STORAGE_ACCOUNT_PREFIX" { + description = "The prefix to the name of the storage account for each environment" + type = string +} From c09ddea3b5f3699a92afcfb79c3ff20f856f6625 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 22:50:15 +0000 Subject: [PATCH 42/57] refactor(terraform): make ETL-related names more generic --- terraform/mst/azure-vars.yml | 1 + terraform/roles.tf | 6 +++--- terraform/variables.tf | 7 ++++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/terraform/mst/azure-vars.yml b/terraform/mst/azure-vars.yml index c3b24d92..34f0b669 100644 --- a/terraform/mst/azure-vars.yml +++ b/terraform/mst/azure-vars.yml @@ -3,3 +3,4 @@ variables: TF_STORAGE_ACCOUNT: 'mstcourtesycardstf' TF_VAR_AGENCY_CARD: 'mst-courtesy-cards' TF_VAR_AGENCY_STORAGE_ACCOUNT_PREFIX: 'mstcceligibility' + TF_VAR_AGENCY_CARD_DATA_ETL_FILE : 'velocity.csv' diff --git a/terraform/roles.tf b/terraform/roles.tf index 87f02868..92008dba 100644 --- a/terraform/roles.tf +++ b/terraform/roles.tf @@ -1,14 +1,14 @@ -resource "azurerm_role_assignment" "velocity_etl" { +resource "azurerm_role_assignment" "agency_card_data_etl" { count = local.is_prod ? 1 : 0 description = "This role assignment gives write access only for the path of the hashed data file." scope = azurerm_storage_container.config.resource_manager_id role_definition_name = "Storage Blob Data Contributor" - principal_id = var.VELOCITY_ETL_APP_OBJECT_ID + principal_id = var.AGENCY_CARD_DATA_ETL_APP_OBJECT_ID condition = < Date: Tue, 10 Oct 2023 22:52:44 +0000 Subject: [PATCH 43/57] chore: reorder variables to be more organized --- terraform/variables.tf | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index 0ceec42b..cc47c2b0 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -3,13 +3,8 @@ # needs to be uppercase "because Azure DevOps will always transform pipeline variables to uppercase environment variables" # https://gaunacode.com/terraform-input-variables-using-azure-devops -variable "DEPLOYER_APP_OBJECT_ID" { - description = "Object ID from the Azure DevOps deployer service principal application in Active Directory" - type = string -} - -variable "ENGINEERING_GROUP_OBJECT_ID" { - description = "Object ID from the engineering group (cal-itp-compiler) in Azure Active Directory" +variable "AGENCY_CARD" { + description = "The name of the agency's card program" type = string } @@ -23,6 +18,21 @@ variable "AGENCY_CARD_DATA_ETL_FILE" { type = string } +variable "AGENCY_STORAGE_ACCOUNT_PREFIX" { + description = "The prefix to the name of the storage account for each environment" + type = string +} + +variable "DEPLOYER_APP_OBJECT_ID" { + description = "Object ID from the Azure DevOps deployer service principal application in Active Directory" + type = string +} + +variable "ENGINEERING_GROUP_OBJECT_ID" { + description = "Object ID from the engineering group (cal-itp-compiler) in Azure Active Directory" + type = string +} + variable "IP_ADDRESS_WHITELIST_DEV" { description = "List of IP addresses allowed to connect to the app service, in CIDR notation: https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/linux_web_app#ip_address. By default, all IP addresses are allowed." type = list(string) @@ -40,13 +50,3 @@ variable "IP_ADDRESS_WHITELIST_PROD" { type = list(string) default = [] } - -variable "AGENCY_CARD" { - description = "The name of the agency's card program" - type = string -} - -variable "AGENCY_STORAGE_ACCOUNT_PREFIX" { - description = "The prefix to the name of the storage account for each environment" - type = string -} From 50fbd286fa8224e4f0381f5a894722b3c699d283 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 22:56:15 +0000 Subject: [PATCH 44/57] chore: make description for ETL app object ID variable more generic --- terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index cc47c2b0..64133e90 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -9,7 +9,7 @@ variable "AGENCY_CARD" { } variable "AGENCY_CARD_DATA_ETL_APP_OBJECT_ID" { - description = "Object ID from the registered application for the Velocity server ETL uploading: https://cloudsight.zendesk.com/hc/en-us/articles/360016785598-Azure-finding-your-service-principal-object-ID" + description = "Object ID from the registered application for the Agency Card server ETL uploading: https://cloudsight.zendesk.com/hc/en-us/articles/360016785598-Azure-finding-your-service-principal-object-ID" type = string } From e1ae46e9ca7f24254dd8fdcf1c14715a75ee7e0b Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 10 Oct 2023 23:01:16 +0000 Subject: [PATCH 45/57] refactor(terraform): extract separate variable for resource group name --- terraform/environment.tf | 2 +- terraform/mst/azure-vars.yml | 1 + terraform/variables.tf | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/terraform/environment.tf b/terraform/environment.tf index 2825b7d2..af11ee4a 100644 --- a/terraform/environment.tf +++ b/terraform/environment.tf @@ -5,5 +5,5 @@ locals { } data "azurerm_resource_group" "main" { - name = "${var.AGENCY_CARD}-eligibility-${local.env_name}" + name = "${var.AGENCY_RESOURCE_GROUP}-eligibility-${local.env_name}" } diff --git a/terraform/mst/azure-vars.yml b/terraform/mst/azure-vars.yml index 34f0b669..480f1057 100644 --- a/terraform/mst/azure-vars.yml +++ b/terraform/mst/azure-vars.yml @@ -2,5 +2,6 @@ variables: TF_RESOURCE_GROUP: 'mst-courtesy-cards-eligibility-terraform' TF_STORAGE_ACCOUNT: 'mstcourtesycardstf' TF_VAR_AGENCY_CARD: 'mst-courtesy-cards' + TF_VAR_AGENCY_RESOURCE_GROUP: 'courtesy-cards' TF_VAR_AGENCY_STORAGE_ACCOUNT_PREFIX: 'mstcceligibility' TF_VAR_AGENCY_CARD_DATA_ETL_FILE : 'velocity.csv' diff --git a/terraform/variables.tf b/terraform/variables.tf index 64133e90..692c2567 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -18,6 +18,11 @@ variable "AGENCY_CARD_DATA_ETL_FILE" { type = string } +variable "AGENCY_RESOURCE_GROUP" { + description = "The prefix to the name of the resource group for each environment" + type = string +} + variable "AGENCY_STORAGE_ACCOUNT_PREFIX" { description = "The prefix to the name of the storage account for each environment" type = string From c22124edee8f50647a5ea63d079232f4747f9e83 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Wed, 11 Oct 2023 17:52:43 +0000 Subject: [PATCH 46/57] chore: make naming more consistent --- terraform/environment.tf | 2 +- terraform/mst/azure-vars.yml | 12 ++++++------ terraform/variables.tf | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/terraform/environment.tf b/terraform/environment.tf index af11ee4a..fe33aac3 100644 --- a/terraform/environment.tf +++ b/terraform/environment.tf @@ -5,5 +5,5 @@ locals { } data "azurerm_resource_group" "main" { - name = "${var.AGENCY_RESOURCE_GROUP}-eligibility-${local.env_name}" + name = "${var.AGENCY_RESOURCE_GROUP_PREFIX}-eligibility-${local.env_name}" } diff --git a/terraform/mst/azure-vars.yml b/terraform/mst/azure-vars.yml index 480f1057..9d2326b6 100644 --- a/terraform/mst/azure-vars.yml +++ b/terraform/mst/azure-vars.yml @@ -1,7 +1,7 @@ variables: - TF_RESOURCE_GROUP: 'mst-courtesy-cards-eligibility-terraform' - TF_STORAGE_ACCOUNT: 'mstcourtesycardstf' - TF_VAR_AGENCY_CARD: 'mst-courtesy-cards' - TF_VAR_AGENCY_RESOURCE_GROUP: 'courtesy-cards' - TF_VAR_AGENCY_STORAGE_ACCOUNT_PREFIX: 'mstcceligibility' - TF_VAR_AGENCY_CARD_DATA_ETL_FILE : 'velocity.csv' + TF_RESOURCE_GROUP: "mst-courtesy-cards-eligibility-terraform" + TF_STORAGE_ACCOUNT: "mstcourtesycardstf" + TF_VAR_AGENCY_CARD: "mst-courtesy-cards" + TF_VAR_AGENCY_RESOURCE_GROUP_PREFIX: "courtesy-cards" + TF_VAR_AGENCY_STORAGE_ACCOUNT_PREFIX: "mstcceligibility" + TF_VAR_AGENCY_CARD_DATA_ETL_FILE: "velocity.csv" diff --git a/terraform/variables.tf b/terraform/variables.tf index 692c2567..da63ec03 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -18,7 +18,7 @@ variable "AGENCY_CARD_DATA_ETL_FILE" { type = string } -variable "AGENCY_RESOURCE_GROUP" { +variable "AGENCY_RESOURCE_GROUP_PREFIX" { description = "The prefix to the name of the resource group for each environment" type = string } From 7f8177934490e4029014307020a2764a5b2d9e5d Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 12 Oct 2023 00:05:54 +0000 Subject: [PATCH 47/57] feat: add files for local development of SBMTD Terraform --- terraform/sbmtd/local.env | 1 + terraform/sbmtd/local.tfbackend | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 terraform/sbmtd/local.env create mode 100644 terraform/sbmtd/local.tfbackend diff --git a/terraform/sbmtd/local.env b/terraform/sbmtd/local.env new file mode 100644 index 00000000..a3351160 --- /dev/null +++ b/terraform/sbmtd/local.env @@ -0,0 +1 @@ +SUBSCRIPTION="Mobility Pass eligibility Server" diff --git a/terraform/sbmtd/local.tfbackend b/terraform/sbmtd/local.tfbackend new file mode 100644 index 00000000..37b20b35 --- /dev/null +++ b/terraform/sbmtd/local.tfbackend @@ -0,0 +1,2 @@ +resource_group_name="sbmtd-mobility-pass-tf" +storage_account_name="sbmtdmobilitypasstf" From 93bf003e174e6e18f37be653e8c9d008e029c03a Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 12 Oct 2023 16:48:42 +0000 Subject: [PATCH 48/57] feat: add SBMTD-specific pipeline definition and variables --- terraform/sbmtd/azure-pipelines.yml | 31 +++++++++++++++++++++++++++++ terraform/sbmtd/azure-vars.yml | 7 +++++++ 2 files changed, 38 insertions(+) create mode 100644 terraform/sbmtd/azure-pipelines.yml create mode 100644 terraform/sbmtd/azure-vars.yml diff --git a/terraform/sbmtd/azure-pipelines.yml b/terraform/sbmtd/azure-pipelines.yml new file mode 100644 index 00000000..f66cd8a4 --- /dev/null +++ b/terraform/sbmtd/azure-pipelines.yml @@ -0,0 +1,31 @@ +trigger: + branches: + include: + - dev + - test + - prod + tags: + include: + - 20??.??.?*-rc?* + - 20??.??.?* + # only run for changes to Terraform files + paths: + include: + - terraform/* + +pr: + branches: + include: + - "*" + paths: + include: + - terraform/* + +pool: + vmImage: ubuntu-latest + +stages: + - template: ../pipeline/deploy.yml + parameters: + # path is relative to the template path (see line above) + variable_file: ../sbmtd/azure-vars.yml diff --git a/terraform/sbmtd/azure-vars.yml b/terraform/sbmtd/azure-vars.yml new file mode 100644 index 00000000..2b19445b --- /dev/null +++ b/terraform/sbmtd/azure-vars.yml @@ -0,0 +1,7 @@ +variables: + TF_RESOURCE_GROUP: "sbmtd-mobility-pass-tf" + TF_STORAGE_ACCOUNT: "sbmtdmobilitypasstf" + TF_VAR_AGENCY_CARD: "sbmtd-mobility-pass" + TF_VAR_AGENCY_RESOURCE_GROUP_PREFIX: "sbmtd-mobility-pass" + TF_VAR_AGENCY_STORAGE_ACCOUNT_PREFIX: "sbmtdmobilitypass" + TF_VAR_AGENCY_CARD_DATA_ETL_FILE: "mobilitypass.csv" From 9514c56e3e862e826e0afbe468da40ce1c9ee274 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 12 Oct 2023 21:42:25 +0000 Subject: [PATCH 49/57] docs(infra): update access restrictions to mention /static path --- terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/README.md b/terraform/README.md index 401950e8..f29625d8 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -30,7 +30,7 @@ For browsing the [Azure portal](https://portal.azure.com), you can [switch your ## Access restrictions -We restrict which IP addresses that can access the app service by using a Web Application Firewall (WAF) configured on a Front Door. There is an exception for the `/healthcheck` path, which can be accessed by any IP address. +We restrict which IP addresses that can access the app service by using a Web Application Firewall (WAF) configured on a Front Door. There is an exception for the `/healthcheck` and `/static` paths, which can be accessed by any IP address. The app service itself gives access only to our Front Door and to Azure availability tests. From b165ec688eda0ff14936d00c52fb6c46f5ff478b Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 12 Oct 2023 21:59:15 +0000 Subject: [PATCH 50/57] docs(infra): update docs on init script taking agency param --- terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/README.md b/terraform/README.md index f29625d8..dd78a123 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -105,7 +105,7 @@ Terraform is [`plan`](https://www.terraform.io/cli/commands/plan)'d when code is 1. [Initialize Terraform.](https://www.terraform.io/cli/commands/init) You can also use this script later to switch between [environments](#environments). ```sh - ./init.sh + ./init.sh ``` 1. Make changes to Terraform files. From f871893f6520348dc4b182f69c36f9b1d615133a Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 12 Oct 2023 22:12:59 +0000 Subject: [PATCH 51/57] docs(infra): update docs that mention environment resource groups remove Build Status badge; it seems to not render correctly --- terraform/README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/terraform/README.md b/terraform/README.md index dd78a123..68006ecb 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -17,9 +17,9 @@ The following things in Azure are managed outside of Terraform: | Environment | Azure Resource Group | Terraform Workspace | Git Branch | | ----------- | --------------------------------- | ------------------- | ---------- | -| Dev | `courtesy-cards-eligibility-dev` | `dev` | `dev` | -| Test | `courtesy-cards-eligibility-test` | `test` | `test` | -| Prod | `courtesy-cards-eligibility-prod` | `default` | `prod` | +| Dev | `$(AGENCY_RESOURCE_GROUP_PREFIX)-eligibility-dev` | `dev` | `dev` | +| Test | `$(AGENCY_RESOURCE_GROUP_PREFIX)-eligibility-test` | `test` | `test` | +| Prod | `$(AGENCY_RESOURCE_GROUP_PREFIX)-eligibility-prod` | `default` | `prod` | All resources in these Resource Groups should be reflected in Terraform in this repository. The exceptions are: @@ -70,17 +70,21 @@ See [`Failures`](https://docs.microsoft.com/en-us/azure/azure-monitor/app/asp-ne After [setting up the Azure CLI](#making-changes), you can use the following command to [stream live logs](https://docs.microsoft.com/en-us/azure/app-service/troubleshoot-diagnostic-logs#in-local-terminal): ```sh +az webapp log tail --resource-group --name 2>&1 | grep -v /healthcheck +``` + +e.g. + +```bash az webapp log tail --resource-group courtesy-cards-eligibility-prod --name mst-courtesy-cards-eligibility-server-prod 2>&1 | grep -v /healthcheck ``` ### SCM -[Docker logs](https://mst-courtesy-cards-eligibility-server-dev.scm.azurewebsites.net/api/logs/docker) +Docker logs can be viewed in the Advanced Tools for the instance. The URL pattern is `https://.scm.azurewebsites.net/api/logs/docker` ## Making changes -[![Build Status](https://dev.azure.com/mstransit/courtesy-cards/_apis/build/status/cal-itp.eligibility-server?branchName=dev)](https://dev.azure.com/mstransit/courtesy-cards/_build/latest?definitionId=1&branchName=dev) - Terraform is [`plan`](https://www.terraform.io/cli/commands/plan)'d when code is pushed to any branch on GitHub, then [`apply`](https://www.terraform.io/cli/commands/apply)'d when merged to `dev`. While other automation for this project is done through GitHub Actions, we use an Azure Pipeline (above) for a couple of reasons: - Easier authentication with the Azure API using a service connnection From 4a35bafda155bdfbc827345a7e9c0fc632b0637f Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 12 Oct 2023 22:14:38 +0000 Subject: [PATCH 52/57] docs(infra): update reference to Terraform version location --- terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/README.md b/terraform/README.md index 68006ecb..a161985b 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -96,7 +96,7 @@ Terraform is [`plan`](https://www.terraform.io/cli/commands/plan)'d when code is 1. Install dependencies: - [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - - [Terraform](https://www.terraform.io/downloads) - see exact version in [`pipeline/azure-pipelines.yml`](pipeline/azure-pipelines.yml) + - [Terraform](https://www.terraform.io/downloads) - see exact version in [`pipeline/deploy.yml`](pipeline/deploy.yml) 1. [Authenticate using the Azure CLI](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/azure_cli). From 8c713d2c373ba06ba0b1494ab2e5205bc7948cb6 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 12 Oct 2023 22:31:27 +0000 Subject: [PATCH 53/57] docs(infra): add details on steps to be done manually --- terraform/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/terraform/README.md b/terraform/README.md index a161985b..59d2ac19 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -125,4 +125,12 @@ Terraform is [`plan`](https://www.terraform.io/cli/commands/plan)'d when code is The steps we took to set up MST's environment are documented in [a separate Google Doc](https://docs.google.com/document/d/12uzuKyvyabHAOaeQc6k2jQIG5pQprdEyBpfST_dY2ME/edit#heading=h.1vs880ltbo58). -This is not a complete step-by-step guide; more a list of things to remember. This may be useful as part of incident response. +In general, the steps that must be done manually before the pipeline can be run are: + +- Create Resource Group and storage account dedicated to the Terraform state +- Create container in storage account for Terraform state +- Create environment Resource Group for each environment, Region: West US + - We create these manually to avoid having to give the pipeline service connection permissions for creating resource groups +- Create Terraform workspace for each environment +- Trigger a pipeline run to verify `plan` and `apply` +- Known chicken-and-egg problem: Terraform both creates the Key Vault and expects a secret within it, so will always fail on the first deploy. Add the Benefits slack email secret and re-run the pipeline. From 7f547e2563e324234d752a50629ec2da18a057c7 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 12 Oct 2023 22:53:09 +0000 Subject: [PATCH 54/57] docs(infra): move terraform README into docs file structure update README to point to online documentation --- docs/deployment/.pages | 2 + docs/deployment/infrastructure.md | 136 ++++++++++++++++++++++++++++++ terraform/README.md | 135 +---------------------------- 3 files changed, 139 insertions(+), 134 deletions(-) create mode 100644 docs/deployment/.pages create mode 100644 docs/deployment/infrastructure.md diff --git a/docs/deployment/.pages b/docs/deployment/.pages new file mode 100644 index 00000000..49da12a9 --- /dev/null +++ b/docs/deployment/.pages @@ -0,0 +1,2 @@ +nav: + - infrastructure.md diff --git a/docs/deployment/infrastructure.md b/docs/deployment/infrastructure.md new file mode 100644 index 00000000..59d2ac19 --- /dev/null +++ b/docs/deployment/infrastructure.md @@ -0,0 +1,136 @@ +# Infrastructure + +The infrastructure is configured as code via [Terraform](https://www.terraform.io/), for [various reasons](https://techcommunity.microsoft.com/t5/fasttrack-for-azure/the-benefits-of-infrastructure-as-code/ba-p/2069350). + +## Architecture + +## Resources outside of Terraform + +The following things in Azure are managed outside of Terraform: + +- Subcriptions +- Active Directory (users, groups, service principals, etc.) +- Service connections +- Configuration files, stored as blobs + +## Environments + +| Environment | Azure Resource Group | Terraform Workspace | Git Branch | +| ----------- | --------------------------------- | ------------------- | ---------- | +| Dev | `$(AGENCY_RESOURCE_GROUP_PREFIX)-eligibility-dev` | `dev` | `dev` | +| Test | `$(AGENCY_RESOURCE_GROUP_PREFIX)-eligibility-test` | `test` | `test` | +| Prod | `$(AGENCY_RESOURCE_GROUP_PREFIX)-eligibility-prod` | `default` | `prod` | + +All resources in these Resource Groups should be reflected in Terraform in this repository. The exceptions are: + +- Secrets, such as values under [Key Vault](https://azure.microsoft.com/en-us/services/key-vault/). [`prevent_destroy`](https://developer.hashicorp.com/terraform/tutorials/state/resource-lifecycle#prevent-resource-deletion) is used on these Resources. +- Things managed outside of [Terraform](#resources-outside-of-terraform) + +For browsing the [Azure portal](https://portal.azure.com), you can [switch your `Default subscription filter`](https://docs.microsoft.com/en-us/azure/azure-portal/set-preferences). + +## Access restrictions + +We restrict which IP addresses that can access the app service by using a Web Application Firewall (WAF) configured on a Front Door. There is an exception for the `/healthcheck` and `/static` paths, which can be accessed by any IP address. + +The app service itself gives access only to our Front Door and to Azure availability tests. + +## Monitoring + +We have [ping tests](https://docs.microsoft.com/en-us/azure/azure-monitor/app/monitor-web-app-availability) set up to notify about availability of each environment. Alerts go to [#benefits-notify](https://cal-itp.slack.com/archives/C022HHSEE3F). + +## Logs + +Logs can be found a couple of places: + +### Azure App Service Logs + +[Open the `Logs` for the environment you are interested in.](https://docs.google.com/document/d/11EPDIROBvg7cRtU2V42c6VBxcW_o8HhcyORALNtL_XY/edit#heading=h.6pxjhslhxwvj) The following tables are likely of interest: + +- `AppServiceConsoleLogs`: `stdout` and `stderr` coming from the container +- `AppServiceHTTPLogs`: requests coming through App Service +- `AppServicePlatformLogs`: deployment information + +For some pre-defined queries, click `Queries`, then `Group by: Query type`, and look under `Query pack queries`. + +### [Azure Monitor Logs](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/data-platform-logs) + +[Open the `Logs` for the environment you are interested in.](https://docs.google.com/document/d/11EPDIROBvg7cRtU2V42c6VBxcW_o8HhcyORALNtL_XY/edit#heading=h.n0oq4r1jo7zs) + +The following [tables](https://docs.microsoft.com/en-us/azure/azure-monitor/app/opencensus-python#telemetry-type-mappings) are likely of interest: + +- `requests` +- `traces` + +In the latter two, you should see recent log output. Note [there is some latency](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/data-ingestion-time). + +See [`Failures`](https://docs.microsoft.com/en-us/azure/azure-monitor/app/asp-net-exceptions#diagnose-failures-using-the-azure-portal) in the sidebar (or `exceptions` under `Logs`) for application errors/exceptions. + +### Live tail + +After [setting up the Azure CLI](#making-changes), you can use the following command to [stream live logs](https://docs.microsoft.com/en-us/azure/app-service/troubleshoot-diagnostic-logs#in-local-terminal): + +```sh +az webapp log tail --resource-group --name 2>&1 | grep -v /healthcheck +``` + +e.g. + +```bash +az webapp log tail --resource-group courtesy-cards-eligibility-prod --name mst-courtesy-cards-eligibility-server-prod 2>&1 | grep -v /healthcheck +``` + +### SCM + +Docker logs can be viewed in the Advanced Tools for the instance. The URL pattern is `https://.scm.azurewebsites.net/api/logs/docker` + +## Making changes + +Terraform is [`plan`](https://www.terraform.io/cli/commands/plan)'d when code is pushed to any branch on GitHub, then [`apply`](https://www.terraform.io/cli/commands/apply)'d when merged to `dev`. While other automation for this project is done through GitHub Actions, we use an Azure Pipeline (above) for a couple of reasons: + +- Easier authentication with the Azure API using a service connnection +- Log output is hidden, avoiding accidentally leaking secrets + +### Local development + +1. Get access to the Azure account. +1. Install dependencies: + + - [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) + - [Terraform](https://www.terraform.io/downloads) - see exact version in [`pipeline/deploy.yml`](pipeline/deploy.yml) + +1. [Authenticate using the Azure CLI](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/azure_cli). + + ```sh + az login + ``` + +1. Outside the [dev container](https://docs.calitp.org/eligibility-server/getting-started/), navigate to the `terraform/` directory. +1. Create a [`terraform.tfvars` file](https://developer.hashicorp.com/terraform/language/values/variables#variable-definitions-tfvars-files) and specify the [variables](variables.tf). +1. [Initialize Terraform.](https://www.terraform.io/cli/commands/init) You can also use this script later to switch between [environments](#environments). + + ```sh + ./init.sh + ``` + +1. Make changes to Terraform files. +1. Preview the changes, as necessary. + + ```sh + terraform plan + ``` + +1. Submit the changes via pull request. + +## Azure environment setup + +The steps we took to set up MST's environment are documented in [a separate Google Doc](https://docs.google.com/document/d/12uzuKyvyabHAOaeQc6k2jQIG5pQprdEyBpfST_dY2ME/edit#heading=h.1vs880ltbo58). + +In general, the steps that must be done manually before the pipeline can be run are: + +- Create Resource Group and storage account dedicated to the Terraform state +- Create container in storage account for Terraform state +- Create environment Resource Group for each environment, Region: West US + - We create these manually to avoid having to give the pipeline service connection permissions for creating resource groups +- Create Terraform workspace for each environment +- Trigger a pipeline run to verify `plan` and `apply` +- Known chicken-and-egg problem: Terraform both creates the Key Vault and expects a secret within it, so will always fail on the first deploy. Add the Benefits slack email secret and re-run the pipeline. diff --git a/terraform/README.md b/terraform/README.md index 59d2ac19..b9708d60 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -1,136 +1,3 @@ # Infrastructure -The infrastructure is configured as code via [Terraform](https://www.terraform.io/), for [various reasons](https://techcommunity.microsoft.com/t5/fasttrack-for-azure/the-benefits-of-infrastructure-as-code/ba-p/2069350). - -## Architecture - -## Resources outside of Terraform - -The following things in Azure are managed outside of Terraform: - -- Subcriptions -- Active Directory (users, groups, service principals, etc.) -- Service connections -- Configuration files, stored as blobs - -## Environments - -| Environment | Azure Resource Group | Terraform Workspace | Git Branch | -| ----------- | --------------------------------- | ------------------- | ---------- | -| Dev | `$(AGENCY_RESOURCE_GROUP_PREFIX)-eligibility-dev` | `dev` | `dev` | -| Test | `$(AGENCY_RESOURCE_GROUP_PREFIX)-eligibility-test` | `test` | `test` | -| Prod | `$(AGENCY_RESOURCE_GROUP_PREFIX)-eligibility-prod` | `default` | `prod` | - -All resources in these Resource Groups should be reflected in Terraform in this repository. The exceptions are: - -- Secrets, such as values under [Key Vault](https://azure.microsoft.com/en-us/services/key-vault/). [`prevent_destroy`](https://developer.hashicorp.com/terraform/tutorials/state/resource-lifecycle#prevent-resource-deletion) is used on these Resources. -- Things managed outside of [Terraform](#resources-outside-of-terraform) - -For browsing the [Azure portal](https://portal.azure.com), you can [switch your `Default subscription filter`](https://docs.microsoft.com/en-us/azure/azure-portal/set-preferences). - -## Access restrictions - -We restrict which IP addresses that can access the app service by using a Web Application Firewall (WAF) configured on a Front Door. There is an exception for the `/healthcheck` and `/static` paths, which can be accessed by any IP address. - -The app service itself gives access only to our Front Door and to Azure availability tests. - -## Monitoring - -We have [ping tests](https://docs.microsoft.com/en-us/azure/azure-monitor/app/monitor-web-app-availability) set up to notify about availability of each environment. Alerts go to [#benefits-notify](https://cal-itp.slack.com/archives/C022HHSEE3F). - -## Logs - -Logs can be found a couple of places: - -### Azure App Service Logs - -[Open the `Logs` for the environment you are interested in.](https://docs.google.com/document/d/11EPDIROBvg7cRtU2V42c6VBxcW_o8HhcyORALNtL_XY/edit#heading=h.6pxjhslhxwvj) The following tables are likely of interest: - -- `AppServiceConsoleLogs`: `stdout` and `stderr` coming from the container -- `AppServiceHTTPLogs`: requests coming through App Service -- `AppServicePlatformLogs`: deployment information - -For some pre-defined queries, click `Queries`, then `Group by: Query type`, and look under `Query pack queries`. - -### [Azure Monitor Logs](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/data-platform-logs) - -[Open the `Logs` for the environment you are interested in.](https://docs.google.com/document/d/11EPDIROBvg7cRtU2V42c6VBxcW_o8HhcyORALNtL_XY/edit#heading=h.n0oq4r1jo7zs) - -The following [tables](https://docs.microsoft.com/en-us/azure/azure-monitor/app/opencensus-python#telemetry-type-mappings) are likely of interest: - -- `requests` -- `traces` - -In the latter two, you should see recent log output. Note [there is some latency](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/data-ingestion-time). - -See [`Failures`](https://docs.microsoft.com/en-us/azure/azure-monitor/app/asp-net-exceptions#diagnose-failures-using-the-azure-portal) in the sidebar (or `exceptions` under `Logs`) for application errors/exceptions. - -### Live tail - -After [setting up the Azure CLI](#making-changes), you can use the following command to [stream live logs](https://docs.microsoft.com/en-us/azure/app-service/troubleshoot-diagnostic-logs#in-local-terminal): - -```sh -az webapp log tail --resource-group --name 2>&1 | grep -v /healthcheck -``` - -e.g. - -```bash -az webapp log tail --resource-group courtesy-cards-eligibility-prod --name mst-courtesy-cards-eligibility-server-prod 2>&1 | grep -v /healthcheck -``` - -### SCM - -Docker logs can be viewed in the Advanced Tools for the instance. The URL pattern is `https://.scm.azurewebsites.net/api/logs/docker` - -## Making changes - -Terraform is [`plan`](https://www.terraform.io/cli/commands/plan)'d when code is pushed to any branch on GitHub, then [`apply`](https://www.terraform.io/cli/commands/apply)'d when merged to `dev`. While other automation for this project is done through GitHub Actions, we use an Azure Pipeline (above) for a couple of reasons: - -- Easier authentication with the Azure API using a service connnection -- Log output is hidden, avoiding accidentally leaking secrets - -### Local development - -1. Get access to the Azure account. -1. Install dependencies: - - - [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) - - [Terraform](https://www.terraform.io/downloads) - see exact version in [`pipeline/deploy.yml`](pipeline/deploy.yml) - -1. [Authenticate using the Azure CLI](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/azure_cli). - - ```sh - az login - ``` - -1. Outside the [dev container](https://docs.calitp.org/eligibility-server/getting-started/), navigate to the `terraform/` directory. -1. Create a [`terraform.tfvars` file](https://developer.hashicorp.com/terraform/language/values/variables#variable-definitions-tfvars-files) and specify the [variables](variables.tf). -1. [Initialize Terraform.](https://www.terraform.io/cli/commands/init) You can also use this script later to switch between [environments](#environments). - - ```sh - ./init.sh - ``` - -1. Make changes to Terraform files. -1. Preview the changes, as necessary. - - ```sh - terraform plan - ``` - -1. Submit the changes via pull request. - -## Azure environment setup - -The steps we took to set up MST's environment are documented in [a separate Google Doc](https://docs.google.com/document/d/12uzuKyvyabHAOaeQc6k2jQIG5pQprdEyBpfST_dY2ME/edit#heading=h.1vs880ltbo58). - -In general, the steps that must be done manually before the pipeline can be run are: - -- Create Resource Group and storage account dedicated to the Terraform state -- Create container in storage account for Terraform state -- Create environment Resource Group for each environment, Region: West US - - We create these manually to avoid having to give the pipeline service connection permissions for creating resource groups -- Create Terraform workspace for each environment -- Trigger a pipeline run to verify `plan` and `apply` -- Known chicken-and-egg problem: Terraform both creates the Key Vault and expects a secret within it, so will always fail on the first deploy. Add the Benefits slack email secret and re-run the pipeline. +[Documentation](https://docs.calitp.org/eligibility-server/deployment/infrastructure/) From 2eacbad01d65fd79547051cc2bf3fe5ad3d1f35a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 18:06:44 +0000 Subject: [PATCH 55/57] chore(pre-commit): autoupdate hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/psf/black: 23.9.1 → 23.10.0](https://github.com/psf/black/compare/23.9.1...23.10.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3b665fea..55f11d31 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: - id: check-added-large-files - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 23.10.0 hooks: - id: black types: From eb82b9b47a390e628bf51caa7a13d7e9133a4eb5 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Mon, 23 Oct 2023 20:05:46 +0000 Subject: [PATCH 56/57] docs: add steps about requesting parallelism grant --- docs/deployment/infrastructure.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/deployment/infrastructure.md b/docs/deployment/infrastructure.md index 59d2ac19..9cb745e2 100644 --- a/docs/deployment/infrastructure.md +++ b/docs/deployment/infrastructure.md @@ -127,6 +127,8 @@ The steps we took to set up MST's environment are documented in [a separate Goog In general, the steps that must be done manually before the pipeline can be run are: +- Create an Azure DevOps organization and project +- Request a free grant of parallel jobs using the form at [https://aka.ms/azpipelines-parallelism-request](https://aka.ms/azpipelines-parallelism-request) - Create Resource Group and storage account dedicated to the Terraform state - Create container in storage account for Terraform state - Create environment Resource Group for each environment, Region: West US From 2ecb49cc698d937e5d62f6bfb3cd13e4fa87bd46 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Wed, 25 Oct 2023 20:53:48 +0000 Subject: [PATCH 57/57] fix: use agency card as prefix for key vault name this will rename MST's key vault --- terraform/key_vault.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/key_vault.tf b/terraform/key_vault.tf index 2211c589..5fbe334b 100644 --- a/terraform/key_vault.tf +++ b/terraform/key_vault.tf @@ -1,6 +1,6 @@ resource "azurerm_key_vault" "main" { # name needs to be globally unique - name = "eligibility-server-${local.env_name}" + name = "${var.AGENCY_CARD}-${local.env_name}" location = data.azurerm_resource_group.main.location resource_group_name = data.azurerm_resource_group.main.name sku_name = "standard"