feat: post-deployment tests (#242)

.github/workflows/cd-to-infra.yml

@@ -65,10 +65,17 @@ jobs:
             elif [[ "${{ github.event.action }}" == "released" ]]; then
               DEPLOY_ENV="prod"
             fi
+            # Run fast tests post-deployment
+            TEST_SELECTOR="fast"
           else
             DEPLOY_ENV="qa"
+            # Run all tests post-deployment to QA
+            TEST_SELECTOR="."
           fi
           if [[ -n "$DEPLOY_ENV" ]]; then
+            # Schedule deployment
             make DEPLOY_ENV="$DEPLOY_ENV" DEPLOY_TAG=${{ needs.publish.outputs.deploy_tag }} schedule-k8s-deployment
+            # Schedule post-deployment tests
+            make DEPLOY_ENV="$DEPLOY_ENV" TEST_SELECTOR="$TEST_SELECTOR" schedule-tests
           fi
           echo "DEPLOY_ENV is $DEPLOY_ENV"

.github/workflows/manual-deploy-to-infra.yml

@@ -26,12 +26,17 @@ env:
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    env:
+      DEPLOY_ENV: ${{ github.event.inputs.environment }}
+      DEPLOY_TAG: ${{ github.event.inputs.tag }}
     steps:
       -
         uses: actions/checkout@v3
       -
         name: Schedule k8s deployment
-        env:
-          DEPLOY_ENV: ${{ github.event.inputs.environment }}
-          DEPLOY_TAG: ${{ github.event.inputs.tag }}
         run: make schedule-k8s-deployment
+      -
+        name: Schedule post-deployment tests
+        run: |
+          # Just run "fast" tests for manual deployments
+          make TEST_SELECTOR="fast" schedule-tests

Makefile

@@ -21,6 +21,9 @@ DEPLOY_TAG ?= latest
 # Whether or not this is a manual deployment
 MANUAL_DEPLOY ?= false
 
+# Test selector
+TEST_SELECTOR ?= .
+
 .PHONY: all
 all: build check-fmt check-clippy test
 
@@ -98,3 +101,7 @@ schedule-ecs-deployment:
 .PHONY: schedule-k8s-deployment
 schedule-k8s-deployment:
 	./ci-scripts/schedule_k8s_deploy.sh "${DEPLOY_ENV}" "${DEPLOY_TAG}"
+
+.PHONY: schedule-tests
+schedule-tests:
+	./ci-scripts/schedule_tests.sh "${DEPLOY_ENV}" "${TEST_SELECTOR}"

ci-scripts/schedule_tests.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+id=$(uuidgen)
+job_id=$(uuidgen)
+# Schedule tests for 10 minutes in the future to allow the network to stabilize. This assumes that the deployment is
+# successful with the right image being deployed, which might not always be the case if the deployment fails for some
+# reason. In the future, this can be done better via the CD manager, which will check for the network being ready with
+# the right image before scheduling tests.
+now=$(date +%s%N -d "10 minutes")
+ttl=$(date +%s -d "14 days")
+network=${1-dev}
+test_selector=${2-.}
+
+docker run --rm -i \
+  -e "AWS_REGION=$AWS_REGION" \
+  -e "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" \
+  -e "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" \
+  -v ~/.aws:/root/.aws \
+  -v "$PWD":/aws \
+  amazon/aws-cli dynamodb put-item --table-name "ceramic-$network-ops" --item \
+  "{                                                        \
+    \"id\":     {\"S\": \"$id\"},                           \
+    \"job\":    {\"S\": \"$job_id\"},                       \
+    \"ts\":     {\"N\": \"$now\"},                          \
+    \"ttl\":    {\"N\": \"$ttl\"},                          \
+    \"stage\":  {\"S\": \"queued\"},                        \
+    \"type\":   {\"S\": \"workflow\"},                      \
+    \"params\": {                                           \
+      \"M\": {                                              \
+        \"name\":     {\"S\": \"Post-Deployment Tests\"},   \
+        \"org\":      {\"S\": \"3box\"},                    \
+        \"repo\":     {\"S\": \"ceramic-tests\"},           \
+        \"ref\":      {\"S\": \"main\"},                    \
+        \"workflow\": {\"S\": \"run-durable.yml\"},         \
+        \"inputs\":   {                                     \
+          \"M\": {                                          \
+            \"test_selector\": {\"S\": \"$test_selector\"}  \
+          }                                                 \
+        }                                                   \
+      }                                                     \
+    }                                                       \
+  }"