Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Streamline GitHub Actions workflows #532

Merged
merged 7 commits into from
Jul 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,28 +1,16 @@
name: deploy
name: Staged deploy to Test, Preprod and Prod

on:
push:
branches: [main]
release:
types: [published]

jobs:
dev:
uses: "./.github/workflows/deploy-generic.yml"
with:
env: "dev"
secrets:
kube_namespace: ${{ secrets.KUBE_NAMESPACE }}
kube_cert: ${{ secrets.KUBE_CERT }}
kube_cluster: ${{ secrets.KUBE_CLUSTER }}
kube_token: ${{ secrets.KUBE_TOKEN }}
ecr_role_to_assume: ${{ secrets.DEV_ECR_ROLE_TO_ASSUME }}
secret_key: ${{ secrets.SECRET_KEY }}
catalogue_token: ${{ secrets.CATALOGUE_TOKEN }}
slack_alert_webhook: ${{ secrets.SLACK_ALERT_WEBHOOK }}
azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }}
code-tests:
uses: "./.github/workflows/reusable-tests.yml"

test:
uses: "./.github/workflows/deploy-generic.yml"
needs: dev
deploy-test:
uses: "./.github/workflows/reusable-build-and-deploy.yml"
needs: code-tests
with:
env: "test"
secrets:
Expand All @@ -36,8 +24,8 @@ jobs:
slack_alert_webhook: ${{ secrets.SLACK_ALERT_WEBHOOK }}
azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }}

preprod:
uses: "./.github/workflows/deploy-generic.yml"
deploy-preprod:
uses: "./.github/workflows/reusable-build-and-deploy.yml"
needs: test
with:
env: "preprod"
Expand All @@ -52,8 +40,8 @@ jobs:
slack_alert_webhook: ${{ secrets.SLACK_ALERT_WEBHOOK }}
azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }}

prod:
uses: "./.github/workflows/deploy-generic.yml"
deploy-prod:
uses: "./.github/workflows/reusable-build-and-deploy.yml"
needs: preprod
with:
env: "prod"
Expand Down
35 changes: 35 additions & 0 deletions .github/workflows/deploy-test-from-branch.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
name: Deploy to Test from branch

on:
workflow_dispatch:
inputs:
run-tests:
description: 'Run code tests'
required: false
type: boolean

concurrency:
group: ${{ github.ref }}
cancel-in-progress: true

jobs:
code-tests:
if: ${{ inputs.run-tests }}
uses: "./.github/workflows/reusable-tests.yml"

deploy-test:
needs: code-tests
if: ${{ always() && !failure() && !cancelled() }} # don't skip if tests are skipped
uses: "./.github/workflows/reusable-build-and-deploy.yml"
with:
env: "test"
secrets:
kube_namespace: ${{ secrets.KUBE_NAMESPACE }}
kube_cert: ${{ secrets.KUBE_CERT }}
kube_cluster: ${{ secrets.KUBE_CLUSTER }}
kube_token: ${{ secrets.KUBE_TOKEN }}
ecr_role_to_assume: ${{ secrets.TEST_ECR_ROLE_TO_ASSUME }}
secret_key: ${{ secrets.SECRET_KEY }}
catalogue_token: ${{ secrets.CATALOGUE_TOKEN }}
slack_alert_webhook: ${{ secrets.SLACK_ALERT_WEBHOOK }}
azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }}
32 changes: 32 additions & 0 deletions .github/workflows/deploy-to-dev.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
name: Deploy (with tests) to dev
# based on https://jacobian.org/til/github-actions-poetry/

on:
workflow_dispatch:
pull_request:
types: [opened, synchronize, reopened]

concurrency:
group: ${{ github.ref }}
cancel-in-progress: true

jobs:
tests:
uses: "./.github/workflows/reusable-tests.yml"

# Deploys to the 'dev' frontend unless pushed to main
deploy-to-dev:
needs: [tests]
uses: "./.github/workflows/reusable-build-and-deploy.yml"
with:
env: "dev"
secrets:
kube_namespace: ${{ secrets.KUBE_NAMESPACE }}
kube_cert: ${{ secrets.KUBE_CERT }}
kube_cluster: ${{ secrets.KUBE_CLUSTER }}
kube_token: ${{ secrets.KUBE_TOKEN }}
ecr_role_to_assume: ${{ secrets.DEV_ECR_ROLE_TO_ASSUME }}
secret_key: ${{ secrets.SECRET_KEY }}
catalogue_token: ${{ secrets.CATALOGUE_TOKEN }}
slack_alert_webhook: ${{ secrets.SLACK_ALERT_WEBHOOK }}
azure_client_secret: ${{ secrets.AZURE_CLIENT_SECRET }}
20 changes: 20 additions & 0 deletions .github/workflows/notify-slack-on-failure.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
name: Check for Deployment failure (Test and above)
# based on: https://github.com/integrations/slack/issues/1563#issuecomment-1588009077
on:
workflow_run:
workflows: ["Deploy to Test from branch", "Staged deploy to Test, Preprod and Prod"]
types: [completed]

jobs:
on-failure:
runs-on: ubuntu-latest
if: github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out'
steps:
- uses: ravsamhq/notify-slack-action@v2
with:
status: ${{ github.event.workflow_run.conclusion }}
notification_title: " ${{github.event.workflow_run.name}} - ${{github.event.workflow_run.conclusion}} on ${{github.event.workflow_run.head_branch}} - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.event.workflow_run.id}}|View Failure>"
message_format: ":fire: *${{github.event.workflow_run.name}}* ${{github.event.workflow_run.conclusion}} in <${{github.server_url}}/${{github.repository}}/${{github.event.workflow_run.head_branch}}|${{github.repository}}>"
footer: "Linked Repo <${{github.server_url}}/${{github.repository}}|${{github.repository}}> | <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.event.workflow_run.id}}|View Failure>"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }}
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ on:
required: true

jobs:
deploy:
name: Deploy Helm chart into Cloud Platform
build-and-push:
name: Build and push Docker image to CP namespace ECR
environment: ${{ inputs.env }}
runs-on: ubuntu-latest
permissions:
Expand All @@ -56,22 +56,36 @@ jobs:
id: login-to-ecr
uses: aws-actions/amazon-ecr-login@v2

- name: Build Docker image
id: build-docker-image
- name: Output image path
id: image-path
env:
REGISTRY: ${{ steps.login-to-ecr.outputs.registry }}
REPOSITORY: ${{ vars.ECR_REPOSITORY }}
IMAGE_TAG: ${{ github.sha }}
run: docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG .
run: |
echo "image_path=${REGISTRY}/${REPOSITORY}:${IMAGE_TAG}" >> $GITHUB_OUTPUT

- name: Build Docker image
id: build-docker-image
env:
IMAGE_PATH: ${{ steps.image-path.outputs.image_path }}
run: docker build -t ${IMAGE_PATH} .

- name: Push Docker image to ECR
id: push-docker-image-to-ecr
env:
REGISTRY: ${{ steps.login-to-ecr.outputs.registry }}
REPOSITORY: ${{ vars.ECR_REPOSITORY }}
IMAGE_TAG: ${{ github.sha }}
run: docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
IMAGE_PATH: ${{ steps.image-path.outputs.image_path }}
run: docker push ${IMAGE_PATH}

deploy:
name: Deploy Helm chart into Cloud Platform
needs: build-and-push
environment: ${{ inputs.env }}
runs-on: ubuntu-latest
permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout
steps:
- name: Prepare Helm deployment
id: prepare-helm-deployment
env:
Expand All @@ -83,9 +97,7 @@ jobs:
SENTRY_DSN_WORKAROUND: ${{ vars.SENTRY_DSN_WORKAROUND }}
SECRET_KEY: ${{ secrets.SECRET_KEY }}
CATALOGUE_TOKEN: ${{ secrets.CATALOGUE_TOKEN }}
IMAGE_TAG: ${{ github.sha }}
REGISTRY: ${{ steps.login-to-ecr.outputs.registry }}
REPOSITORY: ${{ vars.ECR_REPOSITORY }}
IMAGE_PATH: ${{ job.build-and-push.steps.image-path.outputs.image_path }}
NAMESPACE: ${{ secrets.KUBE_NAMESPACE }}
ENABLE_ANALYTICS: ${{ vars.ENABLE_ANALYTICS }}
ANALYTICS_ID: ${{ vars.ANALYTICS_ID }}
Expand Down Expand Up @@ -121,3 +133,13 @@ jobs:
env:
KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }}
run: kubectl -n ${KUBE_NAMESPACE} apply -f deployments/

- name: Slack failure notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
with:
payload: |
{"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
141 changes: 141 additions & 0 deletions .github/workflows/reusable-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
name: Deploy

on:
workflow_call:

jobs:
datahub-client-path-filter:
runs-on: ubuntu-latest
steps:
- uses: dorny/paths-filter@v3
id: changes
with:
filters: |
datahub-client:
- 'lib/datahub-client/**'

datahub-client-tests:
name: Run datahub client tests
needs: datahub-client-path-filter
if: ${{ needs.datahub-client-path-filter.steps.changes.outputs.datahub-client == 'true' }}
runs-on: ubuntu-latest
defaults:
run:
working-directory: lib/datahub-client
strategy:
fail-fast: false
max-parallel: 4
matrix:
python-version:
- "3.10"
steps:
- name: Checkout repository
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

- name: Install poetry
run: pipx install poetry

- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
with:
python-version: ${{ matrix.python-version }}
cache: poetry
cache-dependency-path: lib/datahub-client/poetry.lock

- name: Poetry install
run: |
poetry install

- name: Run tests
run: |
poetry run pytest

app-unit-tests:
name: Django app unit tests
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b

- name: Install poetry
run: pipx install poetry

- name: Set up Python
uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
with:
python-version: "3.11"
cache: poetry
cache-dependency-path: ./poetry.lock

- name: Install project
run: poetry install --no-interaction --no-root

- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: "21"

- name: Get npm cache directory
id: npm-cache-dir
shell: bash
run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT}

- uses: actions/cache@v4
with:
path: ${{ steps.npm-cache-dir.outputs.dir }}
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-

- name: Install npm dependencies
id: install_dependencies
run: npm install

- name: Collect static files
run: poetry run python manage.py collectstatic --no-input

- name: Run unit tests with coverage
id: fast-tests
run: TESTING=True poetry run pytest --cov -m 'not slow and not datahub' --doctest-modules

- name: Set up chromedriver
# https://github.com/marketplace/actions/setup-chromedriver
uses: nanasess/[email protected]

- name: Run selenium tests
id: slow-tests
if: steps.fast-tests.outcome == 'success'
run: TESTING=True poetry run pytest tests/selenium --axe-version 4.9.1 --chromedriver-path /usr/local/bin/chromedriver

javascript-only-tests:
name: Javascript tests
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: "21"

- name: Get npm cache directory
id: npm-cache-dir
shell: bash
run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT}

- name: Load cached npm
uses: actions/cache@v4
id: npm-cache
with:
path: ${{ steps.npm-cache-dir.outputs.dir }}
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-

- name: Install dependencies
id: install_dependencies
run: npm install

- name: Run javascript tests
run: npm test
Loading
Loading