hicommonwealth · kurtisassad · Sep 10, 2024 · Sep 13, 2024 · Sep 17, 2024 · Sep 17, 2024
diff --git a/.dockerignore b/.dockerignore
@@ -12,7 +12,6 @@ node_modules/
 **/test-results
 **/*.tsbuildinfo
 **/.DS_Store
-*.env
 *.log
 .idea
 .vscode

diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml
@@ -0,0 +1,47 @@
+name: Build Docker Image
+
+on:
+  push:
+    branches:
+      - staging
+      - frick
+      - frack
+      - production
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set Heroku app name based on branch
+        id: set-heroku-app-name
+        run: |
+          BRANCH_NAME=${GITHUB_REF##*/}
+          case $BRANCH_NAME in
+            frick)
+              APP_NAME="commonwealth-frick"
+              ;;
+            frack)
+              APP_NAME="commonwealth-frack"
+              ;;
+            staging)
+              APP_NAME="commonwealth-beta"
+              ;;
+            demo)
+              APP_NAME="commonwealth-demo"
+              ;;
+            production)
+              APP_NAME="commonwealthapp"
+              ;;
+            *)
+              echo "Branch $BRANCH_NAME is not configured for deployment."
+              exit 1
+              ;;
+          esac
+          echo "HEROKU_APP_NAME=$APP_NAME" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v2
+
+      - name: Deploy to Heroku
+        run: |
+          chmod +x ./scripts/deploy-to-heroku.sh
+          ./scripts/deploy-to-heroku.sh "${{ secrets.HEROKU_EMAIL }}" "${{ secrets.HEROKU_API_TOKEN }}" "${{ env.HEROKU_APP_NAME }}"
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -187,7 +187,7 @@ jobs:
       - name: Run serial e2e tests
         run: |
           pnpm -F commonwealth bootstrap-test-db &&
-          (cd packages/commonwealth && NODE_OPTIONS=--max-old-space-size=$(../../scripts/get-max-old-space-size.sh) node --import=extensionless/register --enable-source-maps ./build/server.js) &
+          (cd packages/commonwealth && node --import=extensionless/register --enable-source-maps ./build/server.js) &
           (pnpm -F commonwealth wait-server && pnpm -F commonwealth test-e2e-serial --forbid-only)
 
       - name: Archive code coverage results

diff --git a/Dockerfile b/Dockerfile
diff --git a/Dockerfile.commonwealth_base b/Dockerfile.commonwealth_base
@@ -0,0 +1,48 @@
+FROM datadog-base AS base
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+
+FROM base AS build
+## needed for node-gyp to build
+RUN apt-get update && \
+    apt-get install -y python3 make gcc g++
+COPY . /usr/src/app
+WORKDIR /usr/src/app
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
+RUN pnpm run build
+
+RUN pnpm -F discord-bot run build
+RUN pnpm deploy -F discord-bot --prod /prod/discord-bot
+RUN mv /usr/src/app/packages/discord-bot/build /prod/discord-bot/build
+
+RUN pnpm -F snapshot-listener run build
+RUN pnpm deploy -F snapshot-listener --prod /prod/snapshot-listener
+RUN mv /usr/src/app/packages/snapshot-listener/build /prod/snapshot-listener/build
+
+RUN pnpm deploy -F commonwealth --prod /prod/commonwealth
+RUN mv /usr/src/app/packages/commonwealth/build /prod/commonwealth/build
+
+FROM base AS commonwealth
+ENV NODE_ENV=production
+COPY --from=build /prod/commonwealth /prod/commonwealth
+WORKDIR /prod/commonwealth
+COPY ./scripts/datadog-entrypoint.sh /prod/commonwealth
+RUN chmod +x /prod/commonwealth/datadog-entrypoint.sh
+ENV PORT=$PORT
+
+FROM base AS snapshot-listener
+ENV NODE_ENV=production
+COPY --from=build /prod/snapshot-listener /prod/snapshot-listener
+WORKDIR /prod/snapshot-listener
+COPY ./scripts/datadog-entrypoint.sh /prod/snapshot-listener
+RUN chmod +x /prod/snapshot-listener/datadog-entrypoint.sh
+ENV PORT=$PORT
+
+FROM base AS discord-bot
+ENV NODE_ENV=production
+COPY --from=build /prod/discord-bot /prod/discord-bot
+WORKDIR /prod/discord-bot
+COPY ./scripts/datadog-entrypoint.sh /prod/discord-bot
+RUN chmod +x /prod/discord-bot/datadog-entrypoint.sh
+ENV PORT=$PORT
diff --git a/Dockerfile.datadog b/Dockerfile.datadog
@@ -0,0 +1,28 @@
+FROM node:20-slim AS datadog-base
+
+# Install Datadog dependencies
+RUN apt-get update && apt-get install -y gnupg apt-transport-https gpg-agent curl ca-certificates
+
+# Add Datadog repository and signing keys
+ENV DATADOG_APT_KEYRING="/usr/share/keyrings/datadog-archive-keyring.gpg"
+ENV DATADOG_APT_KEYS_URL="https://keys.datadoghq.com"
+RUN sh -c "echo 'deb [signed-by=${DATADOG_APT_KEYRING}] https://apt.datadoghq.com/ stable 7' > /etc/apt/sources.list.d/datadog.list"
+RUN touch ${DATADOG_APT_KEYRING}
+RUN curl -o /tmp/DATADOG_APT_KEY_CURRENT.public "${DATADOG_APT_KEYS_URL}/DATADOG_APT_KEY_CURRENT.public" && \
+    gpg --ignore-time-conflict --no-default-keyring --keyring ${DATADOG_APT_KEYRING} --import /tmp/DATADOG_APT_KEY_CURRENT.public
+RUN curl -o /tmp/DATADOG_APT_KEY_06462314.public "${DATADOG_APT_KEYS_URL}/DATADOG_APT_KEY_06462314.public" && \
+    gpg --ignore-time-conflict --no-default-keyring --keyring ${DATADOG_APT_KEYRING} --import /tmp/DATADOG_APT_KEY_06462314.public
+RUN curl -o /tmp/DATADOG_APT_KEY_C0962C7D.public "${DATADOG_APT_KEYS_URL}/DATADOG_APT_KEY_C0962C7D.public" && \
+    gpg --ignore-time-conflict --no-default-keyring --keyring ${DATADOG_APT_KEYRING} --import /tmp/DATADOG_APT_KEY_C0962C7D.public
+RUN curl -o /tmp/DATADOG_APT_KEY_F14F620E.public "${DATADOG_APT_KEYS_URL}/DATADOG_APT_KEY_F14F620E.public" && \
+    gpg --ignore-time-conflict --no-default-keyring --keyring ${DATADOG_APT_KEYRING} --import /tmp/DATADOG_APT_KEY_F14F620E.public
+RUN curl -o /tmp/DATADOG_APT_KEY_382E94DE.public "${DATADOG_APT_KEYS_URL}/DATADOG_APT_KEY_382E94DE.public" && \
+    gpg --ignore-time-conflict --no-default-keyring --keyring ${DATADOG_APT_KEYRING} --import /tmp/DATADOG_APT_KEY_382E94DE.public
+
+# Install Datadog Agent
+RUN apt-get update && apt-get -y install --reinstall datadog-agent
+
+# Expose DogStatsD and trace-agent ports
+EXPOSE 8125/udp 8126/tcp
+
+COPY datadog-config/ /etc/datadog-agent/
diff --git a/common_knowledge/Deployment.md b/common_knowledge/Deployment.md
@@ -0,0 +1,31 @@
+# Deployment Flow
+In order to perform a deployment, we need to push to a branch that corresponds with a heroku application. The branches
+are frick, frack, beta. This then runs a github action that takes care of the deployment.
+
+# Deployment Details
+We use Heroku for deploying our applications. The deployment process is automated via GitHub Actions using a Continuous Deployment (CD) pipeline. Every time we push changes to a branch associated with a specific app (e.g., frick, frack, beta, etc.), the CD.yml GitHub action is triggered, which initiates the deployment process.
+
+## Deployment Workflow Overview
+### Base Container Build:
+
+First, our base container image is built using the following command:
+bash
+```shell
+docker build -f Dockerfile.commonwealth_base -t commonwealth_base .
+```
+This image contains the core setup required by all of our apps, ensuring consistency and reducing redundancy across different services.
+### Individual Process Builds:
+
+After building the base container, the individual process-specific Dockerfiles (e.g., Dockerfile.web, Dockerfile.evm_ce) are built and pushed to Heroku using the following command:
+bash
+```shell
+heroku container:push --recursive -a ${heroku_app_name}
+```
+The --recursive flag ensures that all defined processes (web, consumer, evm_ce, etc.) are built and pushed to the appropriate Heroku app.
+### Release on Heroku:
+
+After the container images are successfully pushed, the next step is to release these images to Heroku:
+```shell
+heroku container:release web evm_ce consumer message_relayer knock -a ${heroku_app_name}
+```
+This command releases the defined processes to their respective dynos on Heroku, making the application live.
diff --git a/datadog-config/datadog.yaml b/datadog-config/datadog.yaml
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -51,8 +51,8 @@ services:
       - with-app
     container_name: commonwealth
     build:
-      context: . # Assuming Dockerfile is in the same directory as this docker-compose.yml
-      dockerfile: Dockerfile # Name of the Dockerfile
+      context: .
+      dockerfile: ./Dockerfile.web
     ports:
       - '8080:8080'
     environment:

diff --git a/libs/shared/package.json b/libs/shared/package.json
@@ -22,15 +22,17 @@
     "lint-diff": "NODE_OPTIONS=\"--max-old-space-size=4096\" eslint -c ../../.eslintrc-diff.cjs './src/**/*.{ts,tsx}'"
   },
   "dependencies": {
-    "@canvas-js/core": "^0.10.10",
     "@canvas-js/chain-cosmos": "^0.10.10",
     "@canvas-js/chain-ethereum": "^0.10.10",
     "@canvas-js/chain-solana": "^0.10.10",
     "@canvas-js/chain-substrate": "^0.10.10",
+    "@canvas-js/core": "^0.10.10",
     "@canvas-js/gossiplog": "^0.10.10",
     "@canvas-js/interfaces": "^0.10.10",
     "@canvas-js/signatures": "^0.10.10",
+    "@cosmjs/encoding": "0.32.3",
     "@ipld/dag-json": "^10.2.0",
+    "@polkadot/util": "12.6.2",
     "moment": "^2.23.0",
     "safe-stable-stringify": "^2.4.2"
   },

diff --git a/package.json b/package.json
@@ -9,7 +9,7 @@
   "packageManager": "[email protected]",
   "scripts": {
     "build": "chmod u+x scripts/build.sh && ./scripts/build.sh",
-    "docker-build-commonwealth": "docker build . --target commonwealth --tag commonwealth:latest",
+    "docker-build-commonwealth": "docker build -f Dockerfile.commonwealth_base --tag commonwealth:latest .",
     "chromatic": "chromatic --exit-zero-on-changes",
     "clean": "chmod u+x scripts/clean-modules.sh && ./scripts/clean-modules.sh",
     "db-all": "pnpm -F commonwealth db-all",
@@ -125,7 +125,6 @@
     "prettier-plugin-organize-imports": "^4.0.0",
     "process": "^0.11.10",
     "readline-sync": "^1.4.10",
-    "sequelize-cli": "^6.2.0",
     "sharp": "^0.31.2",
     "sinon": "^15.0.4",
     "source-map-support": "^0.5.21",

diff --git a/packages/commonwealth/Procfile b/packages/commonwealth/Procfile
diff --git a/packages/commonwealth/deploy/dockerfiles/Dockerfile.consumer b/packages/commonwealth/deploy/dockerfiles/Dockerfile.consumer
@@ -0,0 +1,2 @@
+FROM commonwealth
+CMD NODE_ENV=production ./datadog-entrypoint.sh node --import=extensionless/register build/server/workers/commonwealthConsumer/commonwealthConsumer.js
diff --git a/packages/commonwealth/deploy/dockerfiles/Dockerfile.evm_ce b/packages/commonwealth/deploy/dockerfiles/Dockerfile.evm_ce
@@ -0,0 +1,2 @@
+FROM commonwealth
+CMD NODE_ENV=production ./datadog-entrypoint.sh node --import=extensionless/register build/server/workers/evmChainEvents/startEvmPolling.js
diff --git a/packages/commonwealth/deploy/dockerfiles/Dockerfile.knock b/packages/commonwealth/deploy/dockerfiles/Dockerfile.knock
@@ -0,0 +1,2 @@
+FROM commonwealth
+CMD NODE_ENV=production ./datadog-entrypoint.sh node --import=extensionless/register build/server/workers/knock/knockWorker.js
diff --git a/packages/commonwealth/deploy/dockerfiles/Dockerfile.message_relayer b/packages/commonwealth/deploy/dockerfiles/Dockerfile.message_relayer
@@ -0,0 +1,2 @@
+FROM commonwealth
+CMD NODE_ENV=production ./datadog-entrypoint.sh node --import=extensionless/register build/server/workers/messageRelayer/messageRelayer.js
diff --git a/packages/commonwealth/deploy/dockerfiles/Dockerfile.release b/packages/commonwealth/deploy/dockerfiles/Dockerfile.release
@@ -0,0 +1,5 @@
+FROM commonwealth
+RUN apt-get update && apt-get install -y curl # Needed for heroku
+CMD node --import=extensionless/register build/server/scripts/releasePhaseEnvCheck.js && \
+  npx sequelize-cli db:migrate --config server/sequelize.json && \
+  node --import=extensionless/register build/server/scripts/purgeCloudflareCache.js
diff --git a/packages/commonwealth/deploy/dockerfiles/Dockerfile.web b/packages/commonwealth/deploy/dockerfiles/Dockerfile.web
@@ -0,0 +1,2 @@
+FROM commonwealth
+CMD NODE_ENV=production ./datadog-entrypoint.sh node --import=extensionless/register --enable-source-maps ./build/server.js
diff --git a/packages/commonwealth/deploy/environments/.env.public.commonwealth-beta b/packages/commonwealth/deploy/environments/.env.public.commonwealth-beta
@@ -0,0 +1,10 @@
+UNLEASH_FRONTEND_SERVER_URL=https://flag.commonwealth.im/api/frontend
+UNLEASH_FRONTEND_API_TOKEN=*:development.309ede2a4ac83b15b51c0cd6b55acdc9f4edefeda5b6325e2414f9b0
+HEROKU_APP_NAME=commonwealth-beta
+APP_ENV=beta
+NODE_ENV=production
+KNOCK_PUBLIC_API_KEY=pk_RLg22EIJ6jsuci6c7VvBU59gDQJZeFoeBKlOkgJLWvA
+KNOCK_IN_APP_FEED_ID=fc6e68e5-b7b9-49c1-8fab-6dd7e3510ffb
+SERVER_URL=https://qa.commonwealth.im
+MIXPANEL_DEV_TOKEN=312b6c5fadb9a88d98dc1fb38de5d900
+MAGIC_PUBLISHABLE_KEY=pk_live_B0604AA1B8EEFDB4
diff --git a/packages/commonwealth/deploy/environments/.env.public.commonwealth-demo b/packages/commonwealth/deploy/environments/.env.public.commonwealth-demo
@@ -0,0 +1,7 @@
+UNLEASH_FRONTEND_SERVER_URL=https://flag.commonwealth.im/api/frontend
+UNLEASH_FRONTEND_API_TOKEN=*:development.309ede2a4ac83b15b51c0cd6b55acdc9f4edefeda5b6325e2414f9b0
+HEROKU_APP_NAME=commonwealth-demo
+APP_ENV=demo
+NODE_ENV=production
+MIXPANEL_DEV_TOKEN=312b6c5fadb9a88d98dc1fb38de5d900
+MAGIC_PUBLISHABLE_KEY=pk_live_EF89AABAFB87D6F4
diff --git a/packages/commonwealth/deploy/environments/.env.public.commonwealth-frack b/packages/commonwealth/deploy/environments/.env.public.commonwealth-frack
@@ -0,0 +1,10 @@
+UNLEASH_FRONTEND_SERVER_URL=https://flag.commonwealth.im/api/frontend
+UNLEASH_FRONTEND_API_TOKEN=*:development.309ede2a4ac83b15b51c0cd6b55acdc9f4edefeda5b6325e2414f9b0
+HEROKU_APP_NAME=commonwealth-frack
+APP_ENV=frack
+NODE_ENV=production
+SERVER_URL=https://commonwealth-frack.herokuapp.com
+KNOCK_PUBLIC_API_KEY=pk_EkjqgrIByZo85tIqdBkCmihVBtTB_ixY_37oTG_Au1Y
+KNOCK_IN_APP_FEED_ID=fc6e68e5-b7b9-49c1-8fab-6dd7e3510ffb
+MIXPANEL_DEV_TOKEN=312b6c5fadb9a88d98dc1fb38de5d900
+MAGIC_PUBLISHABLE_KEY=pk_live_B0604AA1B8EEFDB4
diff --git a/packages/commonwealth/deploy/environments/.env.public.commonwealth-frick b/packages/commonwealth/deploy/environments/.env.public.commonwealth-frick
@@ -0,0 +1,8 @@
+UNLEASH_FRONTEND_SERVER_URL=https://flag.commonwealth.im/api/frontend
+UNLEASH_FRONTEND_API_TOKEN=*:development.309ede2a4ac83b15b51c0cd6b55acdc9f4edefeda5b6325e2414f9b0
+HEROKU_APP_NAME=commonwealth-frick
+APP_ENV=frick
+NODE_ENV=production
+SERVER_URL=https://commonwealth-frick.herokuapp.com
+MIXPANEL_DEV_TOKEN=312b6c5fadb9a88d98dc1fb38de5d900
+MAGIC_PUBLISHABLE_KEY=pk_live_EF89AABAFB87D6F4
diff --git a/packages/commonwealth/deploy/environments/.env.public.commonwealthapp b/packages/commonwealth/deploy/environments/.env.public.commonwealthapp
@@ -0,0 +1,9 @@
+UNLEASH_FRONTEND_SERVER_URL=https://flag.commonwealth.im/api/frontend
+UNLEASH_FRONTEND_API_TOKEN=*:production.b966a7bcb033f291f1151de9e144129eda0c99553b5e8e781ecc72a0
+HEROKU_APP_NAME=commonwealthapp
+APP_ENV=production
+KNOCK_PUBLIC_API_KEY=pk_ynCCqD_rlxXTO0TvBCYUKYV5BSG5-vHoy451WGDHW5w
+KNOCK_IN_APP_FEED_ID=fc6e68e5-b7b9-49c1-8fab-6dd7e3510ffb
+NODE_ENV=production
+SERVER_URL=https://commonwealth.im
+MAGIC_PUBLISHABLE_KEY=pk_live_B0604AA1B8EEFDB4
diff --git a/packages/commonwealth/package.json b/packages/commonwealth/package.json
@@ -12,7 +12,7 @@
     "async-migrate-comment-version-history": "tsx ./scripts/async-migrate-comment-version-history.ts",
     "async-migrate-thread-version-history": "tsx ./scripts/async-migrate-thread-version-history.ts",
     "add-component-showcase": "tsx ./scripts/add-component-showcase.ts",
-    "archive-outbox": "node --import=extensionless/register --max-old-space-size=$(../../scripts/get-max-old-space-size.sh) build/scripts/archive-outbox.js",
+    "archive-outbox": "node --import=extensionless/register build/scripts/archive-outbox.js",
     "build": "tsc -b ./tsconfig.build.json && tsc-alias -p ./tsconfig.build.json",
     "build-contracts": "npx hardhat compile",
     "bundle": "vite -c ./client/vite.config.ts build",

diff --git a/...th/server/scripts/releasePhaseEnvCheck.ts → ...th/server/scripts/releasePhaseEnvCheck.js b/...th/server/scripts/releasePhaseEnvCheck.ts → ...th/server/scripts/releasePhaseEnvCheck.js
@@ -8,12 +8,10 @@ if (import.meta.url.endsWith(process.argv[1])) {
       // TODO: Add checks for unused env var
       // TODO: Add warning for manually set client FLAGs when APP_ENV !== 'local'/'CI'
       log.info('Environment variables are properly configured');
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
       dispose()('EXIT', true);
     })
     .catch((err) => {
       log.fatal('Environment variables not properly configured!', err);
-      // eslint-disable-next-line @typescript-eslint/no-floating-promises
       dispose()('ERROR', true);
     });
 }
diff --git a/packages/discord-bot/deploy/dockerfiles/Dockerfile.consumer b/packages/discord-bot/deploy/dockerfiles/Dockerfile.consumer
@@ -0,0 +1,2 @@
+FROM discord-bot
+CMD NODE_ENV=production ./datadog-entrypoint.sh node --import=extensionless/register ./build/src/discord-consumer/discordConsumer.js
diff --git a/packages/discord-bot/deploy/dockerfiles/Dockerfile.listener b/packages/discord-bot/deploy/dockerfiles/Dockerfile.listener
@@ -0,0 +1,2 @@
+FROM discord-bot
+CMD NODE_ENV=production ./datadog-entrypoint.sh node --import=extensionless/register ./build/src/discord-listener/discordListener.js
diff --git a/packages/discord-bot/deploy/dockerfiles/Dockerfile.release b/packages/discord-bot/deploy/dockerfiles/Dockerfile.release
@@ -0,0 +1,3 @@
+FROM discord-bot
+RUN apt-get update && apt-get install -y curl
+CMD node --import=extensionless/register ./build/scripts/releasePhaseEnvCheck.js
diff --git a/packages/discord-bot/package.json b/packages/discord-bot/package.json
@@ -23,8 +23,10 @@
     "@hicommonwealth/core": "workspace:*",
     "@hicommonwealth/model": "workspace:*",
     "@hicommonwealth/shared": "workspace:*",
+    "@polkadot/util-crypto": "12.6.2",
     "axios": "^1.3.4",
     "discord.js": "^14.16.2",
+    "extensionless": "^1.9.9",
     "moment": "^2.23.0",
     "pg": "^8.11.3",
     "sequelize": "^6.32.1",

diff --git a/packages/snapshot-listener/Procfile b/packages/snapshot-listener/Procfile
diff --git a/packages/snapshot-listener/deploy/dockerfiles/Dockerfile.release b/packages/snapshot-listener/deploy/dockerfiles/Dockerfile.release
@@ -0,0 +1,3 @@
+FROM snapshot-listener
+RUN apt-get update && apt-get install
+CMD node --import=extensionless/register ./build/scripts/releasePhaseEnvCheck.js
diff --git a/packages/snapshot-listener/deploy/dockerfiles/Dockerfile.web b/packages/snapshot-listener/deploy/dockerfiles/Dockerfile.web
@@ -0,0 +1,2 @@
+FROM snapshot-listener
+CMD NODE_ENV=production ./datadog-entrypoint.sh node --import=extensionless/register --enable-source-maps ./build/src/index.js