diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy-staged.yml similarity index 66% rename from .github/workflows/deploy.yml rename to .github/workflows/deploy-staged.yml index 96ac67d7..7f6cf9ff 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy-staged.yml @@ -1,28 +1,16 @@ -name: deploy +name: Staged deploy to Test, Preprod and Prod on: - push: - branches: [main] + release: + types: [published] jobs: - dev: - uses: "./.github/workflows/deploy-generic.yml" - with: - env: "dev" - secrets: - kube_namespace: ${{ secrets.KUBE_NAMESPACE }} - kube_cert: ${{ secrets.KUBE_CERT }} - kube_cluster: ${{ secrets.KUBE_CLUSTER }} - kube_token: ${{ secrets.KUBE_TOKEN }} - ecr_role_to_assume: ${{ secrets.DEV_ECR_ROLE_TO_ASSUME }} - secret_key: ${{ secrets.SECRET_KEY }} - catalogue_token: ${{ secrets.CATALOGUE_TOKEN }} - slack_alert_webhook: ${{ secrets.SLACK_ALERT_WEBHOOK }} - azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }} + code-tests: + uses: "./.github/workflows/reusable-tests.yml" - test: - uses: "./.github/workflows/deploy-generic.yml" - needs: dev + deploy-test: + uses: "./.github/workflows/reusable-build-and-deploy.yml" + needs: code-tests with: env: "test" secrets: @@ -36,8 +24,8 @@ jobs: slack_alert_webhook: ${{ secrets.SLACK_ALERT_WEBHOOK }} azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }} - preprod: - uses: "./.github/workflows/deploy-generic.yml" + deploy-preprod: + uses: "./.github/workflows/reusable-build-and-deploy.yml" needs: test with: env: "preprod" @@ -52,8 +40,8 @@ jobs: slack_alert_webhook: ${{ secrets.SLACK_ALERT_WEBHOOK }} azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }} - prod: - uses: "./.github/workflows/deploy-generic.yml" + deploy-prod: + uses: "./.github/workflows/reusable-build-and-deploy.yml" needs: preprod with: env: "prod" diff --git a/.github/workflows/deploy-test-from-branch.yml b/.github/workflows/deploy-test-from-branch.yml new file mode 100644 index 00000000..dd976fcd --- /dev/null +++ b/.github/workflows/deploy-test-from-branch.yml @@ -0,0 +1,35 @@ +name: Deploy to Test from branch + +on: + workflow_dispatch: + inputs: + run-tests: + description: 'Run code tests' + required: false + type: boolean + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + code-tests: + if: ${{ inputs.run-tests }} + uses: "./.github/workflows/reusable-tests.yml" + + deploy-test: + needs: code-tests + if: ${{ always() && !failure() && !cancelled() }} # don't skip if tests are skipped + uses: "./.github/workflows/reusable-build-and-deploy.yml" + with: + env: "test" + secrets: + kube_namespace: ${{ secrets.KUBE_NAMESPACE }} + kube_cert: ${{ secrets.KUBE_CERT }} + kube_cluster: ${{ secrets.KUBE_CLUSTER }} + kube_token: ${{ secrets.KUBE_TOKEN }} + ecr_role_to_assume: ${{ secrets.TEST_ECR_ROLE_TO_ASSUME }} + secret_key: ${{ secrets.SECRET_KEY }} + catalogue_token: ${{ secrets.CATALOGUE_TOKEN }} + slack_alert_webhook: ${{ secrets.SLACK_ALERT_WEBHOOK }} + azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }} diff --git a/.github/workflows/deploy-to-dev.yml b/.github/workflows/deploy-to-dev.yml new file mode 100644 index 00000000..a72e33b7 --- /dev/null +++ b/.github/workflows/deploy-to-dev.yml @@ -0,0 +1,32 @@ +name: Deploy (with tests) to dev +# based on https://jacobian.org/til/github-actions-poetry/ + +on: + workflow_dispatch: + pull_request: + types: [opened, synchronize, reopened] + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + tests: + uses: "./.github/workflows/reusable-tests.yml" + + # Deploys to the 'dev' frontend unless pushed to main + deploy-to-dev: + needs: [tests] + uses: "./.github/workflows/reusable-build-and-deploy.yml" + with: + env: "dev" + secrets: + kube_namespace: ${{ secrets.KUBE_NAMESPACE }} + kube_cert: ${{ secrets.KUBE_CERT }} + kube_cluster: ${{ secrets.KUBE_CLUSTER }} + kube_token: ${{ secrets.KUBE_TOKEN }} + ecr_role_to_assume: ${{ secrets.DEV_ECR_ROLE_TO_ASSUME }} + secret_key: ${{ secrets.SECRET_KEY }} + catalogue_token: ${{ secrets.CATALOGUE_TOKEN }} + slack_alert_webhook: ${{ secrets.SLACK_ALERT_WEBHOOK }} + azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }} diff --git a/.github/workflows/notify-slack-on-failure.yml b/.github/workflows/notify-slack-on-failure.yml new file mode 100644 index 00000000..8eab0367 --- /dev/null +++ b/.github/workflows/notify-slack-on-failure.yml @@ -0,0 +1,20 @@ +name: Check for Deployment failure (Test and above) +# based on: https://github.com/integrations/slack/issues/1563#issuecomment-1588009077 +on: + workflow_run: + workflows: ["Deploy to Test from branch", "Staged deploy to Test, Preprod and Prod"] + types: [completed] + +jobs: + on-failure: + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out' + steps: + - uses: ravsamhq/notify-slack-action@v2 + with: + status: ${{ github.event.workflow_run.conclusion }} + notification_title: " ${{github.event.workflow_run.name}} - ${{github.event.workflow_run.conclusion}} on ${{github.event.workflow_run.head_branch}} - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.event.workflow_run.id}}|View Failure>" + message_format: ":fire: *${{github.event.workflow_run.name}}* ${{github.event.workflow_run.conclusion}} in <${{github.server_url}}/${{github.repository}}/${{github.event.workflow_run.head_branch}}|${{github.repository}}>" + footer: "Linked Repo <${{github.server_url}}/${{github.repository}}|${{github.repository}}> | <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.event.workflow_run.id}}|View Failure>" + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }} diff --git a/.github/workflows/deploy-generic.yml b/.github/workflows/reusable-build-and-deploy.yml similarity index 74% rename from .github/workflows/deploy-generic.yml rename to .github/workflows/reusable-build-and-deploy.yml index 160d8e7c..80f23e88 100644 --- a/.github/workflows/deploy-generic.yml +++ b/.github/workflows/reusable-build-and-deploy.yml @@ -37,8 +37,8 @@ on: required: true jobs: - deploy: - name: Deploy Helm chart into Cloud Platform + build-and-push: + name: Build and push Docker image to CP namespace ECR environment: ${{ inputs.env }} runs-on: ubuntu-latest permissions: @@ -56,22 +56,36 @@ jobs: id: login-to-ecr uses: aws-actions/amazon-ecr-login@v2 - - name: Build Docker image - id: build-docker-image + - name: Output image path + id: image-path env: REGISTRY: ${{ steps.login-to-ecr.outputs.registry }} REPOSITORY: ${{ vars.ECR_REPOSITORY }} IMAGE_TAG: ${{ github.sha }} - run: docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . + run: | + echo "image_path=${REGISTRY}/${REPOSITORY}:${IMAGE_TAG}" >> $GITHUB_OUTPUT + + - name: Build Docker image + id: build-docker-image + env: + IMAGE_PATH: ${{ steps.image-path.outputs.image_path }} + run: docker build -t ${IMAGE_PATH} . - name: Push Docker image to ECR id: push-docker-image-to-ecr env: - REGISTRY: ${{ steps.login-to-ecr.outputs.registry }} - REPOSITORY: ${{ vars.ECR_REPOSITORY }} - IMAGE_TAG: ${{ github.sha }} - run: docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG + IMAGE_PATH: ${{ steps.image-path.outputs.image_path }} + run: docker push ${IMAGE_PATH} + deploy: + name: Deploy Helm chart into Cloud Platform + needs: build-and-push + environment: ${{ inputs.env }} + runs-on: ubuntu-latest + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + steps: - name: Prepare Helm deployment id: prepare-helm-deployment env: @@ -83,9 +97,7 @@ jobs: SENTRY_DSN_WORKAROUND: ${{ vars.SENTRY_DSN_WORKAROUND }} SECRET_KEY: ${{ secrets.SECRET_KEY }} CATALOGUE_TOKEN: ${{ secrets.CATALOGUE_TOKEN }} - IMAGE_TAG: ${{ github.sha }} - REGISTRY: ${{ steps.login-to-ecr.outputs.registry }} - REPOSITORY: ${{ vars.ECR_REPOSITORY }} + IMAGE_PATH: ${{ job.build-and-push.steps.image-path.outputs.image_path }} NAMESPACE: ${{ secrets.KUBE_NAMESPACE }} ENABLE_ANALYTICS: ${{ vars.ENABLE_ANALYTICS }} ANALYTICS_ID: ${{ vars.ANALYTICS_ID }} @@ -121,3 +133,13 @@ jobs: env: KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }} run: kubectl -n ${KUBE_NAMESPACE} apply -f deployments/ + + - name: Slack failure notification + if: ${{ failure() && github.ref == 'refs/heads/main' }} + uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0 + with: + payload: | + {"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/reusable-tests.yml b/.github/workflows/reusable-tests.yml new file mode 100644 index 00000000..2e572112 --- /dev/null +++ b/.github/workflows/reusable-tests.yml @@ -0,0 +1,141 @@ +name: Deploy + +on: + workflow_call: + +jobs: + datahub-client-path-filter: + runs-on: ubuntu-latest + steps: + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + datahub-client: + - 'lib/datahub-client/**' + + datahub-client-tests: + name: Run datahub client tests + needs: datahub-client-path-filter + if: ${{ needs.datahub-client-path-filter.steps.changes.outputs.datahub-client == 'true' }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: lib/datahub-client + strategy: + fail-fast: false + max-parallel: 4 + matrix: + python-version: + - "3.10" + steps: + - name: Checkout repository + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + + - name: Install poetry + run: pipx install poetry + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + python-version: ${{ matrix.python-version }} + cache: poetry + cache-dependency-path: lib/datahub-client/poetry.lock + + - name: Poetry install + run: | + poetry install + + - name: Run tests + run: | + poetry run pytest + + app-unit-tests: + name: Django app unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b + + - name: Install poetry + run: pipx install poetry + + - name: Set up Python + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d + with: + python-version: "3.11" + cache: poetry + cache-dependency-path: ./poetry.lock + + - name: Install project + run: poetry install --no-interaction --no-root + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "21" + + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + + - uses: actions/cache@v4 + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Install npm dependencies + id: install_dependencies + run: npm install + + - name: Collect static files + run: poetry run python manage.py collectstatic --no-input + + - name: Run unit tests with coverage + id: fast-tests + run: TESTING=True poetry run pytest --cov -m 'not slow and not datahub' --doctest-modules + + - name: Set up chromedriver + # https://github.com/marketplace/actions/setup-chromedriver + uses: nanasess/setup-chromedriver@v2.2.2 + + - name: Run selenium tests + id: slow-tests + if: steps.fast-tests.outcome == 'success' + run: TESTING=True poetry run pytest tests/selenium --axe-version 4.9.1 --chromedriver-path /usr/local/bin/chromedriver + + javascript-only-tests: + name: Javascript tests + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "21" + + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + + - name: Load cached npm + uses: actions/cache@v4 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Install dependencies + id: install_dependencies + run: npm install + + - name: Run javascript tests + run: npm test diff --git a/.github/workflows/test-and-deploy.yml b/.github/workflows/test-and-deploy.yml deleted file mode 100644 index fbd3883e..00000000 --- a/.github/workflows/test-and-deploy.yml +++ /dev/null @@ -1,127 +0,0 @@ -name: test -# based on https://jacobian.org/til/github-actions-poetry/ - -on: - workflow_dispatch: - pull_request: - branches: - - main - push: - branches: - - main - -jobs: - unit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: 3.11.1 - - - name: cache poetry install - uses: actions/cache@v4 - with: - path: ~/.local - key: poetry-1.7.1-0 - - - uses: snok/install-poetry@v1 - with: - version: 1.7.1 - virtualenvs-create: true - virtualenvs-in-project: true - - - name: cache deps - id: cache-deps - uses: actions/cache@v4 - with: - path: .venv - key: pydeps-${{ hashFiles('**/poetry.lock') }} - - run: poetry install --no-interaction --no-root - if: steps.cache-deps.outputs.cache-hit != 'true' - - run: poetry install --no-interaction - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: "21" - - - name: Get npm cache directory - id: npm-cache-dir - shell: bash - run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} - - - uses: actions/cache@v4 - id: npm-cache - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - - name: Install dependencies - id: install_dependencies - run: npm install - - - name: collect static files - run: poetry run python manage.py collectstatic --no-input - - - name: run unit tests with coverage - id: fast-tests - run: TESTING=True poetry run pytest --cov -m 'not slow and not datahub' --doctest-modules - - - name: Prepare Selenium - # https://github.com/marketplace/actions/setup-chromedriver - uses: nanasess/setup-chromedriver@v2.2.2 - - - name: run selenium tests - id: slow-tests - if: steps.fast-tests.outcome == 'success' - run: TESTING=True poetry run pytest tests/selenium --axe-version 4.9.1 --chromedriver-path /usr/local/bin/chromedriver - - javascript: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: "21" - - - name: Get npm cache directory - id: npm-cache-dir - shell: bash - run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} - - - uses: actions/cache@v4 - id: npm-cache - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node- - - - name: Install dependencies - id: install_dependencies - run: npm install - - - name: Run javascript tests - run: npm test - - # Deploys to the 'dev' frontend unless pushed to main - deploy-to-dev: - if: github.event.ref != 'refs/heads/main' - uses: "./.github/workflows/deploy-generic.yml" - with: - env: "dev" - secrets: - kube_namespace: ${{ secrets.KUBE_NAMESPACE }} - kube_cert: ${{ secrets.KUBE_CERT }} - kube_cluster: ${{ secrets.KUBE_CLUSTER }} - kube_token: ${{ secrets.KUBE_TOKEN }} - ecr_role_to_assume: ${{ secrets.DEV_ECR_ROLE_TO_ASSUME }} - secret_key: ${{ secrets.SECRET_KEY }} - catalogue_token: ${{ secrets.CATALOGUE_TOKEN }} - slack_alert_webhook: ${{ secrets.SLACK_ALERT_WEBHOOK }} - azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }} diff --git a/deployments/templates/deployment.yml b/deployments/templates/deployment.yml index 0ec9386e..d46f71cb 100644 --- a/deployments/templates/deployment.yml +++ b/deployments/templates/deployment.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: find-moj-data - image: ${REGISTRY}/${REPOSITORY}:${IMAGE_TAG} + image: ${IMAGE_PATH} ports: - name: http containerPort: 8000 @@ -45,7 +45,7 @@ spec: - name: AZURE_REDIRECT_URI value: "$AZURE_REDIRECT_URI" - name: AZURE_AUTHORITY - value: "$AZURE_AUTHORITY" + value: "$AZURE_AUTHORITY" - name: SECRET_KEY valueFrom: secretKeyRef: