From e8ca630261849fb80f82cd10def32b2c403a7cd0 Mon Sep 17 00:00:00 2001 From: Erik van Velzen Date: Mon, 15 Jul 2024 15:55:47 +0200 Subject: [PATCH] Docker swarm Deploy test environment to docker swarm. --- .editorconfig | 2 +- .github/workflows/build-push-deploy.yml | 44 ++++++++++++++++-- .github/workflows/get-variables.js | 15 ++++++ docker/compose-prod.yaml | 61 +++++++++++++++++++++++++ docker/config/python.example.env | 1 + src/requirements/local.txt | 2 +- 6 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 docker/compose-prod.yaml diff --git a/.editorconfig b/.editorconfig index 11b226077..5c17cd8d7 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,7 +14,7 @@ indent_size = 4 [*.{yml,yaml}] indent_style = space -indent_size = 2 +indent_size = 4 [*.md] trim_trailing_whitespace = false diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml index 05cf9d853..56b0c6529 100644 --- a/.github/workflows/build-push-deploy.yml +++ b/.github/workflows/build-push-deploy.yml @@ -6,6 +6,7 @@ on: - main - production - acceptance + - swarm workflow_dispatch: permissions: @@ -74,7 +75,7 @@ jobs: --push ## This job takes pretty long and can be split up to parallelize the deploy. - deploy-wagtail: + deploy-wagtail-azure: needs: - build-and-push-images - variables @@ -141,8 +142,7 @@ jobs: --name wagtail-${{ github.ref_name }} --hostname ${{ fromJson(needs.variables.outputs.result).WAGTAIL_HOSTNAME }} - - deploy-next: + deploy-next-azure: needs: - build-and-push-images - variables @@ -195,3 +195,41 @@ jobs: --environment holon-env --name next-${{ github.ref_name }} --hostname www.holontool.nl + + deploy-swarm: + runs-on: ubuntu-latest + environment: ${{ fromJson(needs.variables.outputs.result).GITHUB_ENVIRONMENT }} + needs: + - build-and-push-images + - variables + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + sparse-checkout: docker + - name: Deploy to Docker Swarm + uses: sagebind/docker-swarm-deploy-action@v2 + env: + # Shared + TAG: ${{ needs.variables.outputs.tag }} + WAGTAIL_HOSTNAME: ${{ fromJson(needs.variables.outputs.result).WAGTAIL_HOSTNAME }} + # Wagtail + AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} + MEDIA_LOCATION: ${{ fromJson(needs.variables.outputs.result).MEDIA_LOCATION }} + STATIC_LOCATION: ${{ fromJson(needs.variables.outputs.result).STATIC_LOCATION }} + DB_USER: ${{ fromJson(needs.variables.outputs.result).DB_USER }} + DB_NAME: ${{ fromJson(needs.variables.outputs.result).DB_NAME }} + DB_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + RETURN_SCENARIO: ${{ fromJson(needs.variables.outputs.result).RETURN_SCENARIO }} + SECRET_KEY: "${{ secrets.SECRET_KEY }}" + SENTRY_ENVIRONMENT: ${{ fromJson(needs.variables.outputs.result).SENTRY_ENVIRONMENT }} + DOMAIN_HOST: ${{ fromJson(needs.variables.outputs.result).DOMAIN_HOST }} + EMAIL_HOST_PASSWORD: ${{ secrets.EMAIL_HOST_PASSWORD }} + # NextJS + NEXT_HOSTNAME: ${{ fromJson(needs.variables.outputs.result).NEXT_HOSTNAME }} + NEXT_PUBLIC_TINY_URL_API_KEY: ${{ secrets.TINY_URL_API_KEY }} + with: + remote_host: ssh://root@server.zenmo.com + ssh_private_key: ${{ secrets.SWARM_SSH_PRIVATE_KEY }} + ssh_public_key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ1E4LUG22qgzc8U7oNYGWCn0cyA31+iyX2pck9wcPMS + args: stack deploy --compose-file ./docker/compose-prod.yaml holon-${{ fromJson(needs.variables.outputs.result).GITHUB_ENVIRONMENT.toLowerCase() }} diff --git a/.github/workflows/get-variables.js b/.github/workflows/get-variables.js index 7e44858ee..5998782c6 100644 --- a/.github/workflows/get-variables.js +++ b/.github/workflows/get-variables.js @@ -1,6 +1,7 @@ const configPerBranch = { main: { + GITHUB_ENVIRONMENT: 'Test', DB_NAME: 'holon-wagtail-v2-test', DB_USER: 'holon_wagtail_test', DB_PASSWORD_KEY: 'DB_PASSWORD_TEST', @@ -18,6 +19,7 @@ const configPerBranch = { } }, acceptance: { + GITHUB_ENVIRONMENT: 'Acceptance', DB_NAME: 'holon-wagtail-v2-acceptatie', DB_USER: 'holon_wagtail_acceptance', DB_PASSWORD_KEY: 'DB_PASSWORD_ACCEPTANCE', @@ -35,6 +37,7 @@ const configPerBranch = { } }, production: { + GITHUB_ENVIRONMENT: 'Production', DB_NAME: 'holon-wagtail-v2', DB_USER: 'holon_wagtail_prod', DB_PASSWORD_KEY: 'DB_PASSWORD_PROD', @@ -52,6 +55,18 @@ const configPerBranch = { N_WORKERS: '4', } }, + swarm: { + GITHUB_ENVIRONMENT: 'swarm_test', + DB_NAME: 'holon-test', + DB_USER: 'holon-test', + RETURN_SCENARIO: 'True', + SENTRY_ENVIRONMENT: 'swarm-test', + NEXT_HOSTNAME: 'swarm.holontool.nl', + DOMAIN_HOST: 'https://swarm.holontool.nl', + WAGTAIL_HOSTNAME: 'cms-swarm.holontool.nl', + MEDIA_LOCATION: 'media-test', + STATIC_LOCATION: 'static-test', + }, } module.exports = (branchName) => { diff --git a/docker/compose-prod.yaml b/docker/compose-prod.yaml new file mode 100644 index 000000000..b5c1dfbb4 --- /dev/null +++ b/docker/compose-prod.yaml @@ -0,0 +1,61 @@ +## Compose file for Docker Swarm environments +version: "3.8" + +services: + wagtail: + image: ghcr.io/zenmo/holon-wagtail:${TAG} + environment: + ALLOWED_HOSTS: "*" + AZURE_ACCOUNT_NAME: holonstorage + AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY} + MEDIA_LOCATION: ${MEDIA_LOCATION} + STATIC_LOCATION: ${STATIC_LOCATION} + DB_HOST: postgres + DB_USER: ${DB_USER} + DB_NAME: ${DB_NAME} + DB_PASSWORD: ${DB_PASSWORD} + RETURN_SCENARIO: ${RETURN_SCENARIO} + SECRET_KEY: ${SECRET_KEY} + SENTRY_DSN: "https://764e9f2b886741bcbcfd2acd74a7f7b0@o4505045746384896.ingest.sentry.io/4505045759361024" + SENTRY_ENVIRONMENT: ${SENTRY_ENVIRONMENT} + DOMAIN_HOST: ${DOMAIN_HOST} + N_WORKERS: 8 + EMAIL_HOST_PASSWORD: ${EMAIL_HOST_PASSWORD} + WAGTAILADMIN_BASE_URL: https://${WAGTAIL_HOSTNAME} + labels: + caddy: ${WAGTAIL_HOSTNAME} + caddy.reverse_proxy: "{{upstreams 8000}}" + networks: + - caddy_default + - postgres_default + - default + deploy: + resources: + limits: + cpus: "8" + memory: 8G + + nextjs: + image: ghcr.io/zenmo/holon-nextjs:${TAG} + environment: + WAGTAIL_API_URL: http://wagtail:8000/wt/api/nextjs + NEXT_PUBLIC_WAGTAIL_API_URL: https://${WAGTAIL_HOSTNAME}/wt/api/nextjs + NEXT_PUBLIC_TINY_URL_API_KEY: ${NEXT_PUBLIC_TINY_URL_API_KEY} + labels: + caddy: ${NEXT_HOSTNAME} + caddy.reverse_proxy: "{{upstreams 3000}}" + networks: + - caddy_default + - default + deploy: + resources: + limits: + cpus: "4" + memory: 4G + +networks: + caddy_default: + external: true + postgres_default: + external: true + default: diff --git a/docker/config/python.example.env b/docker/config/python.example.env index c95a7a8b6..c3b3d6dd6 100644 --- a/docker/config/python.example.env +++ b/docker/config/python.example.env @@ -1,3 +1,4 @@ +# For development AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_BUCKET_NAME= diff --git a/src/requirements/local.txt b/src/requirements/local.txt index a2f241264..31456abbc 100644 --- a/src/requirements/local.txt +++ b/src/requirements/local.txt @@ -2,7 +2,7 @@ -r test.txt # Add local extra requirements here (django-debug etc) -pydevd-pycharm~=241.17011.79 +pydevd-pycharm~=242.21829.44 black django-debug-toolbar isort