Full upgrade test

.github/workflows/chain_upgrade-test.yml

@@ -0,0 +1,25 @@
+name: "Test EmpowerChain Upgrade"
+
+on:
+  workflow_dispatch:
+  # TODO: Maybe do on a cron schedule, every week or something?
+
+jobs:
+  test-testnet-upgrade:
+    name: Test EmpowerChain Testnet Upgrade
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: earthly/actions-setup@v1
+        with:
+          version: 0.6.30
+
+      - name: Download a single artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: exported-testnet-genesis
+          path: chain
+
+      - run: earthly --ci --no-cache +upgrade-test --UPGRADE_NAME=v2 # Rely on defaults for other args/env vars
+        working-directory: ./chain

chain/Earthfile

@@ -192,7 +192,7 @@ ibc-conformance-test:
     SAVE ARTIFACT /root/.interchaintest/reports/* AS LOCAL build/interchaintest-reports/
 
 node-base:
-    ARG CURRENT_VERSION=v1.0.1-rc0 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG CURRENT_VERSION=v1.0.1-rc1 # TODO: Any way to get the correct version for mainnet, testnet?
     RUN apt-get update -yq \
         && apt-get install --no-install-recommends -yq \
         curl tar wget clang pkg-config libssl-dev jq \
@@ -202,8 +202,9 @@ node-base:
     RUN unzip empowerd-${CURRENT_VERSION}-linux-amd64.zip -d /root/binaries
     ENV PATH $PATH:/root/binaries
 
+# earthly +prep-genesis --CHAIN_ID=empowerchain-1 --STATE_SYNC_RPC=https://empower-rpc.polkachu.com:443 --STATE_SYNC_PEER=074640d8f2bc981fd201badd19c5b6dba38e00be@65.108.238.219:17456
 prep-genesis:
-    ARG CURRENT_VERSION=v1.0.1-rc0 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG CURRENT_VERSION=v1.0.1-rc1 # TODO: Any way to get the correct version for mainnet, testnet?
     ARG CHAIN_ID=circulus-1 #empowerchain-1
     ARG STATE_SYNC_RPC=https://empower-testnet-rpc.polkachu.com:443 #https://empower-rpc.polkachu.com:443 # TODO: Switch to Empower's own STATE SYNC RPC
     ARG STATE_SYNC_PEER=[email protected]:17456 #[email protected]:17456 # TODO: Switch to Empower's own state sync peer
@@ -240,12 +241,16 @@ prep-genesis:
     SAVE ARTIFACT exported-genesis.json AS LOCAL exported-genesis.json
 
 upgrade-test:
-    ARG CURRENT_VERSION=v1.0.1-rc0 # TODO: Any way to get the correct version for mainnet, testnet?
+    ARG CURRENT_VERSION=v1.0.1-rc1 # TODO: Any way to get the correct version for mainnet, testnet?
     ARG VALPASS=passw0rd
     ARG VALNAME=validator
     ARG --required UPGRADE_NAME # TODO: Could this be parsed from CURRENT_VERSION if we keep a sane naming scheme?
     FROM +node-base --CURRENT_VERSION=${CURRENT_VERSION}
     COPY exported-genesis.json exported-genesis.json
+    COPY +build-with-wasmvm/empowerd ./new_empowerd
+    COPY +build-with-wasmvm/libwasmvm.x86_64.so /usr/local/lib/libwasmvm.x86_64.so
+    COPY +build-with-wasmvm/libwasmvm.aarch64.so /usr/local/lib/libwasmvm.aarch64.so
+    ENV LD_LIBRARY_PATH=/usr/local/lib
 
     RUN empowerd init ${VALNAME}
     # Set up validator key
@@ -256,12 +261,20 @@ upgrade-test:
     RUN mv exported-genesis.json.generated ~/.empowerchain/config/genesis.json
     RUN sed -i -e 's/"voting_period": "172800s"/"voting_period": "30s"/g' ~/.empowerchain/config/genesis.json
 
-    RUN empowerd start > /dev/null 2>&1 & sleep 10; \
-        while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ]; do \
-            echo "Waiting to catch up, current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
+    RUN empowerd start > /tmp/before_upgrade_log 2>&1 & sleep 10; \
+        max_catch_up_attempts=10; \
+        catch_up_count=0; \
+        while [ "`empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $catch_up_count -lt $max_catch_up_attempts ]; do \
+            echo "Waiting to catch up (attempt ${catch_up_count}/${max_catch_up_attempts}, current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
             sleep 5; \
+            catch_up_count=$((catch_up_count+1)); \
         done; \
-        sleep 10; \
+        if [ $catch_up_count -eq $max_catch_up_attempts ]; then \
+            echo "Failed to catch up after ${max_catch_up_attempts} attempts."; \
+            cat /tmp/before_upgrade_log; \
+            exit 1; \
+        fi; \
+        sleep 10; \ # Just for good measure
         UPGRADE_HEIGHT=$(($(empowerd status | jq -r ".SyncInfo.latest_block_height")+20)); \
         echo "Upgrade prop for height $UPGRADE_HEIGHT"; \
         echo "{\n\
@@ -294,16 +307,38 @@ upgrade-test:
           echo "Error: Proposal not passed: $GOV_RESULT"; \
           exit 1; \
         fi; \
-        while [ "`empowerd status | jq -r ".SyncInfo.latest_block_height"`" != ${UPGRADE_HEIGHT} ]; do \
-            echo "Waiting for upgrade height (${UPGRADE_HEIGHT}), current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
+        max_upgrade_wait_attempts=30; \
+        upgrade_wait_count=0; \
+        while [ "`empowerd status | jq -r ".SyncInfo.latest_block_height"`" != ${UPGRADE_HEIGHT} ] && [ $upgrade_wait_count -lt $max_upgrade_wait_attempts ]; do \
+            echo "Waiting for upgrade height (${UPGRADE_HEIGHT}, attempt ${upgrade_wait_count}/${max_upgrade_wait_attempts}), current block: $(empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
             sleep 5; \
+            upgrade_wait_count=$((upgrade_wait_count+1)); \
         done; \
-
-    COPY +build-with-wasmvm/empowerd ./new_empowerd
-    COPY +build-with-wasmvm/libwasmvm.x86_64.so /usr/local/lib/libwasmvm.x86_64.so
-    COPY +build-with-wasmvm/libwasmvm.aarch64.so /usr/local/lib/libwasmvm.aarch64.so
-    ENV LD_LIBRARY_PATH=/usr/local/lib
-
-    # TODO: Next up: start with the new binary and let it run for a few blocks
-
-    RUN false
+        if [ $upgrade_wait_count -eq $max_upgrade_wait_attempts ]; then \
+            echo "Failed to wait for upgrade height ${max_upgrade_wait_attempts} attempts."; \
+            cat /tmp/before_upgrade_log; \
+            exit 1; \
+        fi; \
+        sleep 5; \ # Just for good measure
+        pkill empowerd; \
+        sleep 5; \ # Just for good measure
+        ./new_empowerd start > /tmp/after_upgrade_log 2>&1 & sleep 60; \
+        after_upgrade_max_catch_up_attempts=10; \
+        after_upgrade_catch_up_count=0; \
+        while [ "`./new_empowerd status | jq -r ".SyncInfo.catching_up"`" != "false" ] && [ $after_upgrade_catch_up_count -lt $after_upgrade_max_catch_up_attempts ]; do \
+            echo "Waiting to catch up after upgrade (attempt ${after_upgrade_catch_up_count}/${after_upgrade_max_catch_up_attempts}, current block: $(./new_empowerd status | jq -r ".SyncInfo.latest_block_height")"; \
+            sleep 5; \
+            after_upgrade_catch_up_count=$((after_upgrade_catch_up_count+1)); \
+        done; \
+        if [ $after_upgrade_catch_up_count -eq $after_upgrade_max_catch_up_attempts ]; then \
+            echo "Failed to catch up after the upgrade after ${after_upgrade_max_catch_up_attempts} attempts."; \
+            cat /tmp/after_upgrade_log; \
+            exit 1; \
+        fi; \
+        CURRENT_HEIGHT=$(./new_empowerd status | jq -r ".SyncInfo.latest_block_height"); \
+        if [ $CURRENT_HEIGHT -le $UPGRADE_HEIGHT ]; then \
+            echo "Error: Current height ($CURRENT_HEIGHT) is less than or equal to upgrade height ($UPGRADE_HEIGHT)"; \
+            cat /tmp/after_upgrade_log; \
+            exit 1; \
+        fi; \
+        echo "Upgrade test passed! Halted and upgraded on $UPGRADE_HEIGHT, current height: $CURRENT_HEIGHT";