Merge remote-tracking branch 'dm3-org/develop' into remove-pending-fr…

…om-BE-and-DS

.github/workflows/README

@@ -0,0 +1,13 @@
+Some notes on handling certain stuff
+
+# Host key verification failed
+
+Before connecting to the server from the deployment pipeline, we make sure the server's host key is in the known_hosts file. This is the command used:
+
+`echo "${{ secrets.HOST_SSH_PUBLIC_KEY }}" > ~/.ssh/known_hosts`
+
+If the server's host key changes, the pipeline will fail with the error message "Host key verification failed". To fix this, log in to the server with ssh from your local machine, and accept the new host key. Then, copy the last line from the known_hosts file on your local machine to the secret HOST_SSH_PUBLIC_KEY in the repository.
+
+If you already logged in to the server from your local machine before, you can find the proper line to copy by running this command:
+
+`ssh-keygen -H -F app.dm3.network`

.github/workflows/deploy.yml

@@ -0,0 +1,157 @@
+name: deploy
+on:
+    workflow_dispatch:
+    push:
+        branches:
+            - testing
+            - develop
+            - main
+
+jobs:
+    messenger-web-deploy:
+        environment: ${{ github.ref_name == 'main' && 'prod' || (github.ref_name == 'develop' && 'staging' || 'testing') }}
+        runs-on: ubuntu-latest
+        env:
+            environment_name: ${{ github.ref_name == 'main' && 'prod' || (github.ref_name == 'develop' && 'staging' || 'testing') }}
+        steps:
+            - uses: actions/checkout@v2
+            - uses: actions/setup-node@v3
+              with:
+                  registry-url: 'https://npm.pkg.github.com'
+                  node-version: 22.0.0
+                  cache: 'yarn'
+            - name: Print environment name
+              run: echo $environment_name
+            - name: Declare some variables
+              shell: bash
+              run: |
+                  echo "sha_short=$(git rev-parse --short "$GITHUB_SHA")" >> "$GITHUB_ENV"
+                  echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> "$GITHUB_ENV"
+                  echo "now=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV
+                  echo "unix_now=$(date +%s)" >> "$GITHUB_ENV"
+            - name: Prepare SSH
+              run: |
+                  mkdir ~/.ssh
+                  echo "${{ secrets.HOST_SSH_PUBLIC_KEY }}" > ~/.ssh/known_hosts
+                  echo "${{ secrets.DO_SSH_KEY }}" > ./ssh-key
+                  chmod 600 ./ssh-key
+            - name: Create .env file
+              env:
+                  TARGET_HOST: ${{ vars.HOST_DOMAIN }}
+                  TARGET_IP: ${{ vars.HOST_IP }}
+              run: |
+                  echo "REACT_APP_ADDR_ENS_SUBDOMAIN=${{ vars.ADDR_ENS_SUBDOMAIN }}" >> ./.env.react
+                  echo "REACT_APP_BACKEND=https://${{ vars.HOST_DOMAIN }}/api" >> ./.env.react
+                  echo "REACT_APP_DEFAULT_DELIVERY_SERVICE=${{ vars.DEFAULT_DELIVERY_SERVICE}}" >> ./.env.react
+                  echo "REACT_APP_DEFAULT_SERVICE=https://${{ vars.HOST_DOMAIN }}/api" >> ./.env.react
+                  echo "REACT_APP_MAINNET_PROVIDER_RPC=${{ secrets.RPC }}" >> ./.env.react
+                  echo "REACT_APP_PROFILE_BASE_URL=https://${{ vars.HOST_DOMAIN }}/api" >> ./.env.react
+                  echo "REACT_APP_RESOLVER_BACKEND=https://${{ vars.HOST_DOMAIN }}/resolver-handler" >> ./.env.react
+                  echo "REACT_APP_USER_ENS_SUBDOMAIN=${{ vars.USER_ENS_SUBDOMAIN }}" >> ./.env.react
+                  echo "REACT_APP_PUBLIC_VAPID_KEY=${{ secrets.REACT_APP_PUBLIC_VAPID_KEY}}" >> ./.env.react
+                  echo "REACT_APP_WALLET_CONNECT_PROJECT_ID=${{ secrets.REACT_APP_WALLET_CONNECT_PROJECT_ID }}" >> ./.env.react
+                  echo "REACT_APP_COMMIT_HASH=${{ env.sha_short }}" >> ./.env.react
+                  echo "REACT_APP_BRANCH=${{ env.branch }}" >> ./.env.react
+                  echo "REACT_APP_BUILD_TIME=${{ env.now }}" >> ./.env.react
+                  echo "REACT_APP_ENVIRONMENT_NAME=${{ env.environment_name }}" >> ./.env.react
+                  echo "REACT_APP_MAINNET_PROVIDER_RPC=${{ secrets.MAINNET_PROVIDER_URL}}" >> ./.env.react
+                  echo "REACT_APP_CHAIN_ID=${{ vars.CHAIN_ID }}" >> ./.env.react
+                  echo "REACT_APP_NONCE=${{ vars.STORAGE_NONCE }}" >> ./.env.react
+                  cat ./.env.react >> ./.env
+                  echo "RESOLVER_ADDRESS=${{ vars.ERC3668_RESOLVER_ADDRESS }}" >> ./.env
+                  echo "SIGNING_PUBLIC_KEY=${{ secrets.SIGNING_PUBLIC_KEY }}" >> ./.env
+                  echo "SIGNING_PRIVATE_KEY=${{ secrets.SIGNING_PRIVATE_KEY }}" >> ./.env
+                  echo "SIGNER_PRIVATE_KEY=${{ secrets.SIGNER_PRIVATE_KEY }}" >> ./.env
+                  echo "SPAM_PROTECTION=${{ secrets.SPAM_PROTECTION }}" >> ./.env
+                  echo "ENCRYPTION_PUBLIC_KEY=${{ secrets.ENCRYPTION_PUBLIC_KEY }}" >> ./.env
+                  echo "ENCRYPTION_PRIVATE_KEY=${{ secrets.ENCRYPTION_PRIVATE_KEY }}" >> ./.env
+                  echo "RPC=${{ secrets.RPC }}" >> ./.env
+                  echo "URL=${{ vars.HOST_DOMAIN }}" >> ./.env
+                  echo "CERT_MAIL=${{ vars.CERT_MAIL }}" >> ./.env
+                  echo "DATABASE_URL=${{ secrets.DATABASE_URL }}" >> ./.env
+                  echo "RESOLVER_SUPPORTED_ADDR_ENS_SUBDOMAINS=${{ vars.RESOLVER_SUPPORTED_ADDR_ENS_SUBDOMAINS }}" >> ./.env
+                  echo "RESOLVER_SUPPORTED_NAME_ENS_SUBDOMAINS=${{ vars.RESOLVER_SUPPORTED_NAME_ENS_SUBDOMAINS }}" >> ./.env
+                  envsubst '${TARGET_HOST} ${TARGET_IP}' < ./docker/nginx.conf > ./nginx.conf
+                  cat ./.env
+            - name: Prepare docker build environment
+              shell: bash
+              run: |
+                  cp ./.env.react packages/messenger-demo/.env
+                  cp ./.env.react packages/messenger-web/.env
+                  docker build --progress=plain -t build -f ./docker/DockerfileBuild .
+                  docker build --progress=plain -t base -f ./docker/DockerfileBase .
+                  docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} -p ${{ secrets.DOCKER_HUB_PAT }}
+                  # production images will have no special suffix -> they are the real deal
+                  if [ $environment_name != "prod" ]; then
+                      echo "docker_suffix=.$environment_name" >> "$GITHUB_ENV"
+                  fi
+            - name: Build and publish backend docker image
+              shell: bash
+              run: |
+                  version=$(NODE_PATH=packages/backend node -p "require('package.json').version")
+                  image_name=dm3-backend
+                  docker build --progress=plain -f ./docker/DockerfilePackages --build-arg="PACKAGE=backend" \
+                    --tag $image_name:latest \
+                    --tag dm3org/$image_name:latest${{ env.docker_suffix }} \
+                    --tag dm3org/$image_name:$version${{ env.docker_suffix }}.latest \
+                    --tag dm3org/$image_name:$version${{ env.docker_suffix }}.${{ env.unix_now }}.${{ env.sha_short }}  .
+                  docker save -o ./$image_name.tar $image_name:latest
+                  docker push --all-tags dm3org/$image_name
+            - name: Build and publish delivery-service docker image
+              shell: bash
+              run: |
+                  version=$(NODE_PATH=packages/delivery-service node -p "require('package.json').version")
+                  image_name=dm3-delivery-service
+                  docker build --progress=plain -f ./docker/DockerfilePackages --build-arg="PACKAGE=delivery-service" \
+                    --tag $image_name:latest \
+                    --tag dm3org/$image_name:latest${{ env.docker_suffix }} \
+                    --tag dm3org/$image_name:$version${{ env.docker_suffix }}.latest \
+                    --tag dm3org/$image_name:$version${{ env.docker_suffix }}.${{ env.unix_now }}.${{ env.sha_short }}  .
+                  docker save -o ./$image_name.tar $image_name:latest
+                  docker push --all-tags dm3org/$image_name
+            - name: Build offchain-resolver docker image
+              shell: bash
+              run: |
+                  version=$(NODE_PATH=packages/delivery-service node -p "require('package.json').version")
+                  image_name=dm3-offchain-resolver
+                  docker build --progress=plain -f ./docker/DockerfilePackages --build-arg="PACKAGE=offchain-resolver" \
+                    --tag $image_name:latest \
+                    --tag dm3org/$image_name:latest${{ env.docker_suffix }} \
+                    --tag dm3org/$image_name:$version${{ env.docker_suffix }}.latest \
+                    --tag dm3org/$image_name:$version${{ env.docker_suffix }}.${{ env.unix_now }}.${{ env.sha_short }}  .
+                  docker save -o ./$image_name.tar $image_name:latest
+                  docker push --all-tags dm3org/$image_name
+            - name: Build messenger-web docker image
+              shell: bash
+              run: |
+                  docker build --progress=plain -f ./docker/DockerfilePackages --tag dm3-messenger-web:latest     --build-arg="PACKAGE=messenger-web" .
+                  docker save -o ./dm3-messenger-web.tar dm3-messenger-web:latest
+            - name: Send files to server
+              run: |
+                  ssh -i ./ssh-key root@${{ vars.HOST_DOMAIN }} "\
+                  rm /home/app/*.tar || true"
+                  rsync -avz -e 'ssh -i ./ssh-key' ./.env app@${{ vars.HOST_DOMAIN }}:/home/app/dm3
+                  rsync -avz -e 'ssh -i ./ssh-key' ./dm3-*.tar app@${{ vars.HOST_DOMAIN }}:/home/app/dm3 
+                  rsync -avz -e 'ssh -i ./ssh-key' ./nginx.conf app@${{ vars.HOST_DOMAIN }}:/home/app/dm3
+                  rsync -avz -e 'ssh -i ./ssh-key' ./docker/docker-compose.yml app@${{ vars.HOST_DOMAIN }}:/home/app/dm3
+            - name: Stop docker on server
+              run: |
+                  ssh -i ./ssh-key app@${{ vars.HOST_DOMAIN }} "\
+                  cd dm3 && docker compose down"
+                  ssh -i ./ssh-key root@${{ vars.HOST_DOMAIN }} "\
+                  systemctl restart docker.service"
+            - name: Load docker images
+              run: |
+                  ssh -i ./ssh-key app@${{ vars.HOST_DOMAIN }} "\
+                  cd dm3 && ls |grep -E 'dm3-.*tar' | xargs --no-run-if-empty -L 1 docker load -i; \
+                  rm dm3-*.tar || true"
+            - name: Configure Firewall
+              run: |
+                  ssh -i ./ssh-key root@${{ vars.HOST_DOMAIN }} "\
+                  ufw allow from 172.18.0.1/16 proto tcp to ${{ vars.HOST_IP}} port 80; 
+                  ufw allow from 172.18.0.1/16 proto tcp to ${{ secrets.IP_ADDRESS }} port 443;
+                  ufw enable"
+            - name: Start docker on server
+              run: |
+                  ssh -i ./ssh-key app@${{ vars.HOST_DOMAIN }} "\
+                  cd dm3 && docker compose --env-file .env up -d && docker system prune -af"

.github/workflows/on-push.yml

@@ -43,12 +43,16 @@ jobs:
               with:
                   name: lib-schema
                   path: packages/lib/**/schema/
+            - uses: actions/upload-artifact@master
+              with:
+                  name: backend-schema
+                  path: packages/backend/src/schema/storage
     backend-test:
         runs-on: ubuntu-latest
         needs: build
         defaults:
             run:
-                working-directory: 'packages/lib'
+                working-directory: 'packages/backend'
         steps:
             - uses: actions/checkout@v1
             - id: workspace-test
@@ -61,7 +65,7 @@ jobs:
         needs: build
         defaults:
             run:
-                working-directory: 'packages/lib'
+                working-directory: 'packages/delivery-service'
         steps:
             - uses: actions/checkout@v1
             - id: workspace-test
@@ -147,6 +151,19 @@ jobs:
               with:
                   workspace-name: '@dm3-org/dm3-lib-shared'
                   package-pat: ${{ secrets.PACKAGE_PAT }}
+    lib-shared-side-test:
+        runs-on: ubuntu-latest
+        needs: build
+        defaults:
+            run:
+                working-directory: 'packages/lib'
+        steps:
+            - uses: actions/checkout@v1
+            - id: workspace-test
+              uses: ./test-action
+              with:
+                  workspace-name: '@dm3-org/dm3-lib-shared'
+                  package-pat: ${{ secrets.PACKAGE_PAT }}
     lib-storage-test:
         runs-on: ubuntu-latest
         needs: build