Merge branch 'acceptance' into production

Zenmo · Aug 28, 2024 · f2c1936 · f2c1936
2 parents 6bed9a3 + 91ea105
commit f2c1936
Show file tree

Hide file tree

Showing 16 changed files with 117 additions and 379 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -14,7 +14,7 @@ indent_size = 4
 
 [*.{yml,yaml}]
 indent_style = space
-indent_size = 2
+indent_size = 4
 
 [*.md]
 trim_trailing_whitespace = false

diff --git a/.github/workflows/build-push-deploy.yml b/.github/workflows/build-push-deploy.yml
@@ -73,125 +73,41 @@ jobs:
           --tag ghcr.io/zenmo/holon-nextjs:${{ needs.variables.outputs.tag }}
           --push
 
-  ## This job takes pretty long and can be split up to parallelize the deploy.
-  deploy-wagtail:
-    needs:
-      - build-and-push-images
-      - variables
-    runs-on: ubuntu-latest
-    steps:
-      - name: Log in to Azure
-        uses: azure/login@v1
-        with:
-          ## https://github.com/Azure/login#configure-a-service-principal-with-a-secret
-          creds: ${{ secrets.AZURE_CREDENTIALS }}
-#      - name: Create app environment
-#        uses: azure/CLI@v1
-#        with:
-#          azcliversion: 2.51.0
-#          inlineScript: >
-#            az containerapp env create
-#            --name holon-env
-#            --location westeurope
-#            --resource-group HOLON-webapp
-      - name: Deploy-Wagtail
-        uses: azure/CLI@v1
-        with:
-          ## Use generic azure/CLI@v1 instead of specific azure/container-apps-deploy-action@v1
-          ## because it supports the options that we want.
-          azcliversion: 2.51.0
-          inlineScript: >
-            az containerapp create
-            --resource-group HOLON-webapp
-            --environment holon-env
-            --name wagtail-${{ github.ref_name }}
-            --env-vars
-            ALLOWED_HOSTS="*"
-            AZURE_ACCOUNT_NAME=holonstorage
-            AZURE_STORAGE_KEY=${{ secrets.AZURE_STORAGE_KEY }}
-            MEDIA_LOCATION=${{ fromJson(needs.variables.outputs.result).MEDIA_LOCATION }}
-            STATIC_LOCATION=${{ fromJson(needs.variables.outputs.result).STATIC_LOCATION }}
-            DB_HOST=${{ secrets.DB_HOST }}
-            DB_USER=${{ fromJson(needs.variables.outputs.result).DB_USER }}
-            DB_NAME=${{ fromJson(needs.variables.outputs.result).DB_NAME }}
-            DB_PASSWORD=${{ secrets[fromJson(needs.variables.outputs.result).DB_PASSWORD_KEY] }}
-            RETURN_SCENARIO=${{ fromJson(needs.variables.outputs.result).RETURN_SCENARIO }}
-            SECRET_KEY="${{ secrets.SECRET_KEY }}"
-            SENTRY_DSN=https://764e9f2b886741bcbcfd2acd74a7f7b0@o4505045746384896.ingest.sentry.io/4505045759361024
-            SENTRY_ENVIRONMENT=${{ fromJson(needs.variables.outputs.result).SENTRY_ENVIRONMENT }}
-            DOMAIN_HOST=${{ fromJson(needs.variables.outputs.result).DOMAIN_HOST }}
-            N_WORKERS=${{ fromJson(needs.variables.outputs.result).wagtail.N_WORKERS }}
-            EMAIL_HOST_PASSWORD=${{ secrets.EMAIL_HOST_PASSWORD }}
-            WAGTAILADMIN_BASE_URL=https://${{ fromJson(needs.variables.outputs.result).WAGTAIL_HOSTNAME }}
-            --target-port 8000
-            --ingress external
-            --image ghcr.io/zenmo/holon-wagtail:${{ needs.variables.outputs.tag }}
-            --cpu ${{ fromJson(needs.variables.outputs.result).wagtail.CPU }}
-            --memory ${{ fromJson(needs.variables.outputs.result).wagtail.MEMORY }}
-            --min-replicas 1
-            --max-replicas 2
-      - name: Bind-Wagtail
-        uses: azure/CLI@v1
-        with:
-          azcliversion: 2.51.0
-          inlineScript: >
-            az containerapp hostname bind
-            --resource-group HOLON-webapp
-            --environment holon-env
-            --name wagtail-${{ github.ref_name }}
-            --hostname ${{ fromJson(needs.variables.outputs.result).WAGTAIL_HOSTNAME }}
-
-
-  deploy-next:
-    needs:
-      - build-and-push-images
-      - variables
-    runs-on: ubuntu-latest
-    steps:
-      - name: Log in to Azure
-        uses: azure/login@v1
-        with:
-          ## https://github.com/Azure/login#configure-a-service-principal-with-a-secret
-          creds: ${{ secrets.AZURE_CREDENTIALS }}
-      - name: Deploy-Next.js
-        uses: azure/CLI@v1
-        with:
-          azcliversion: 2.51.0
-          inlineScript: >
-            az containerapp create
-            --resource-group HOLON-webapp
-            --name next-${{ github.ref_name }}
-            --environment holon-env
-            --env-vars
-            WAGTAIL_API_URL=http://wagtail-${{ github.ref_name }}/wt/api/nextjs
-            NEXT_PUBLIC_WAGTAIL_API_URL=https://${{ fromJson(needs.variables.outputs.result).WAGTAIL_HOSTNAME }}/wt/api/nextjs
-            NEXT_PUBLIC_TINY_URL_API_KEY=${{ secrets.TINY_URL_API_KEY }}
-            --target-port 3000
-            --ingress external
-            --tags branch=${{ github.ref_name }}
-            --image ghcr.io/zenmo/holon-nextjs:${{ needs.variables.outputs.tag }}
-            --cpu 0.25
-            --memory 0.5
-            --min-replicas 1
-            --max-replicas 1
-      - name: Bind-Next.js
-        uses: azure/CLI@v1
-        with:
-          azcliversion: 2.51.0
-          inlineScript: >
-            az containerapp hostname bind
-            --resource-group HOLON-webapp
-            --environment holon-env
-            --name next-${{ github.ref_name }}
-            --hostname ${{ fromJson(needs.variables.outputs.result).NEXT_HOSTNAME }}
-      - name: Bind-www
-        if: github.ref_name == 'production'
-        uses: azure/CLI@v1
-        with:
-          azcliversion: 2.51.0
-          inlineScript: >
-            az containerapp hostname bind
-            --resource-group HOLON-webapp
-            --environment holon-env
-            --name next-${{ github.ref_name }}
-            --hostname www.holontool.nl
+  deploy-swarm:
+      runs-on: ubuntu-latest
+      environment: ${{ fromJson(needs.variables.outputs.result).GITHUB_ENVIRONMENT }}
+      needs:
+          - build-and-push-images
+          - variables
+      steps:
+          -   name: Check out repository
+              uses: actions/checkout@v4
+              with:
+                  sparse-checkout: docker
+          -   name: Deploy to Docker Swarm
+              uses: sagebind/docker-swarm-deploy-action@v2
+              env:
+                  # Shared
+                  TAG: ${{ needs.variables.outputs.tag }}
+                  WAGTAIL_HOSTNAME: ${{ fromJson(needs.variables.outputs.result).WAGTAIL_HOSTNAME }}
+                  # Wagtail
+                  AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }}
+                  MEDIA_LOCATION: ${{ fromJson(needs.variables.outputs.result).MEDIA_LOCATION }}
+                  STATIC_LOCATION: ${{ fromJson(needs.variables.outputs.result).STATIC_LOCATION }}
+                  DB_USER: ${{ fromJson(needs.variables.outputs.result).DB_USER }}
+                  DB_NAME: ${{ fromJson(needs.variables.outputs.result).DB_NAME }}
+                  DB_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+                  RETURN_SCENARIO: ${{ fromJson(needs.variables.outputs.result).RETURN_SCENARIO }}
+                  SECRET_KEY: "${{ secrets.SECRET_KEY }}"
+                  SENTRY_ENVIRONMENT: ${{ fromJson(needs.variables.outputs.result).SENTRY_ENVIRONMENT }}
+                  DOMAIN_HOST: ${{ fromJson(needs.variables.outputs.result).DOMAIN_HOST }}
+                  EMAIL_HOST_PASSWORD: ${{ secrets.EMAIL_HOST_PASSWORD }}
+                  # NextJS
+                  NEXT_HOSTNAME: ${{ fromJson(needs.variables.outputs.result).NEXT_HOSTNAME }}
+                  NEXT_CADDY_HOSTS: ${{ fromJson(needs.variables.outputs.result).NEXT_CADDY_HOSTS }}
+                  NEXT_PUBLIC_TINY_URL_API_KEY: ${{ secrets.TINY_URL_API_KEY }}
+              with:
+                  remote_host: ssh://[email protected]
+                  ssh_private_key: ${{ secrets.SWARM_SSH_PRIVATE_KEY }}
+                  ssh_public_key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ1E4LUG22qgzc8U7oNYGWCn0cyA31+iyX2pck9wcPMS
+                  args: stack deploy --compose-file ./docker/compose-prod.yaml holon-${{ fromJson(needs.variables.outputs.result).GITHUB_ENVIRONMENT }}
diff --git a/.github/workflows/get-variables.js b/.github/workflows/get-variables.js
@@ -1,56 +1,40 @@
 
 const configPerBranch = {
     main: {
-        DB_NAME: 'holon-wagtail-v2-test',
-        DB_USER: 'holon_wagtail_test',
-        DB_PASSWORD_KEY: 'DB_PASSWORD_TEST',
+        GITHUB_ENVIRONMENT: 'test',
+        DB_NAME: 'holon-test',
+        DB_USER: 'holon-test',
         RETURN_SCENARIO: 'True',
-        SENTRY_ENVIRONMENT: 'azure-test',
+        SENTRY_ENVIRONMENT: 'swarm-test',
         NEXT_HOSTNAME: 'test.holontool.nl',
-        DOMAIN_HOST: 'https://test.holontool.nl',
+        NEXT_CADDY_HOSTS: 'test.holontool.nl',
         WAGTAIL_HOSTNAME: 'cms-test.holontool.nl',
         MEDIA_LOCATION: 'media-test',
         STATIC_LOCATION: 'static-test',
-        wagtail: {
-            CPU: '0.25',
-            MEMORY: '0.5',
-            N_WORKERS: '2',
-        }
     },
     acceptance: {
-        DB_NAME: 'holon-wagtail-v2-acceptatie',
-        DB_USER: 'holon_wagtail_acceptance',
-        DB_PASSWORD_KEY: 'DB_PASSWORD_ACCEPTANCE',
+        GITHUB_ENVIRONMENT: 'acceptance',
+        DB_NAME: 'holon-acceptance',
+        DB_USER: 'holon-acceptance',
         RETURN_SCENARIO: 'True',
-        SENTRY_ENVIRONMENT: 'azure-acceptance',
+        SENTRY_ENVIRONMENT: 'swarm-acceptance',
         NEXT_HOSTNAME: 'acceptatie.holontool.nl',
-        DOMAIN_HOST: 'https://acceptatie.holontool.nl',
+        NEXT_CADDY_HOSTS: 'acceptatie.holontool.nl',
         WAGTAIL_HOSTNAME: 'cms-acceptatie.holontool.nl',
         MEDIA_LOCATION: 'media-acceptatie',
         STATIC_LOCATION: 'static-acceptatie',
-        wagtail: {
-            CPU: '0.5',
-            MEMORY: '1',
-            N_WORKERS: '4',
-        }
     },
     production: {
-        DB_NAME: 'holon-wagtail-v2',
-        DB_USER: 'holon_wagtail_prod',
-        DB_PASSWORD_KEY: 'DB_PASSWORD_PROD',
+        GITHUB_ENVIRONMENT: 'production',
+        DB_NAME: 'holon-production',
+        DB_USER: 'holon-production',
         RETURN_SCENARIO: 'False',
-        SENTRY_ENVIRONMENT: 'azure-production',
+        SENTRY_ENVIRONMENT: 'swarm-production',
         NEXT_HOSTNAME: 'holontool.nl',
-        DOMAIN_HOST: 'https://holontool.nl',
+        NEXT_CADDY_HOSTS: 'holontool.nl, www.holontool.nl',
         WAGTAIL_HOSTNAME: 'cms.holontool.nl',
         MEDIA_LOCATION: 'media',
         STATIC_LOCATION: 'static',
-        // It seems we only need to scale Wagtail but not Next.js
-        wagtail: {
-            CPU: '0.5',
-            MEMORY: '1',
-            N_WORKERS: '4',
-        }
     },
 }
 

diff --git a/docker/Dockerfile b/docker/Dockerfile
diff --git a/docker/compose-prod.yaml b/docker/compose-prod.yaml
@@ -0,0 +1,61 @@
+## Compose file for Docker Swarm environments
+version: "3.8"
+
+services:
+    wagtail:
+        image: ghcr.io/zenmo/holon-wagtail:${TAG}
+        environment:
+            ALLOWED_HOSTS: "*"
+            AZURE_ACCOUNT_NAME: holonstorage
+            AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY}
+            MEDIA_LOCATION: ${MEDIA_LOCATION}
+            STATIC_LOCATION: ${STATIC_LOCATION}
+            DB_HOST: postgres
+            DB_USER: ${DB_USER}
+            DB_NAME: ${DB_NAME}
+            DB_PASSWORD: ${DB_PASSWORD}
+            RETURN_SCENARIO: ${RETURN_SCENARIO}
+            SECRET_KEY: ${SECRET_KEY}
+            SENTRY_DSN: "https://764e9f2b886741bcbcfd2acd74a7f7b0@o4505045746384896.ingest.sentry.io/4505045759361024"
+            SENTRY_ENVIRONMENT: ${SENTRY_ENVIRONMENT}
+            DOMAIN_HOST: https://${NEXT_HOSTNAME}
+            N_WORKERS: 8
+            EMAIL_HOST_PASSWORD: ${EMAIL_HOST_PASSWORD}
+            WAGTAILADMIN_BASE_URL: https://${WAGTAIL_HOSTNAME}
+        labels:
+            caddy: ${WAGTAIL_HOSTNAME}
+            caddy.reverse_proxy: "{{upstreams 8000}}"
+        networks:
+            - caddy_default
+            - postgres_default
+            - default
+        deploy:
+            resources:
+                limits:
+                    cpus: "8"
+                    memory: 8G
+
+    nextjs:
+        image: ghcr.io/zenmo/holon-nextjs:${TAG}
+        environment:
+            WAGTAIL_API_URL: http://wagtail:8000/wt/api/nextjs
+            NEXT_PUBLIC_WAGTAIL_API_URL: https://${WAGTAIL_HOSTNAME}/wt/api/nextjs
+            NEXT_PUBLIC_TINY_URL_API_KEY: ${NEXT_PUBLIC_TINY_URL_API_KEY}
+        labels:
+            caddy: ${NEXT_CADDY_HOSTS}
+            caddy.reverse_proxy: "{{upstreams 3000}}"
+        networks:
+            - caddy_default
+            - default
+        deploy:
+            resources:
+                limits:
+                    cpus: "4"
+                    memory: 4G
+
+networks:
+    caddy_default:
+        external: true
+    postgres_default:
+        external: true
+    default:
diff --git a/docker/config/python.example.env b/docker/config/python.example.env
@@ -1,3 +1,4 @@
+# For development
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 AWS_BUCKET_NAME=

diff --git a/docker/files/certs/.gitkeep b/docker/files/certs/.gitkeep
diff --git a/docker/files/config/nginx.conf b/docker/files/config/nginx.conf