feat: add sequencer health check

Signed-off-by: Jawad Tariq <[email protected]>

subnet-incal.yml

@@ -175,13 +175,18 @@ services:
         condition: service_healthy
     entrypoint: ""
     command: bash -c "
+      apt-get update && apt-get install -y net-tools &&
       source /contracts/.env &&
       cp -vr /data/data-1/consensus /tmp/node_config/node/sequencer-incal/consensus &&
       cp -vr /data/data-1/libp2p /tmp/node_config/node/sequencer-incal/libp2p &&
       mkdir -p /tmp/node_config/subnet/incal/ &&
       cp -v /data/genesis.json /tmp/node_config/subnet/incal/genesis.json  &&
       echo Topos core contract address=$(printenv TOPOS_CORE_PROXY_CONTRACT_ADDRESS), set manually in config &&
       topos node up --name sequencer-incal --home /tmp/node_config --no-edge-process"
+    healthcheck:
+      test: ["CMD-SHELL", "test $(netstat -ntu | grep ':8545' | wc -l) -gt 0"]
+      interval: 1s
+      retries: 10
     environment:
       - RUST_LOG=info,topos=debug
       - TOOLCHAIN_VERSION=stable

subnet-topos.yml

@@ -235,13 +235,18 @@ services:
         condition: service_completed_successfully
     entrypoint: ""
     command: bash -c "
+      apt-get update && apt-get install -y net-tools &&
       source /contracts/.env &&
       mkdir -p /data/node/sequencer-topos/consensus -p /data/node/sequencer-topos/libp2p -p /data/subnet/topos-sequencer/ &&
       cp -vr /data/node/node-1/consensus /data/node/sequencer-topos/ &&
       cp -vr /data/node/node-1/libp2p /data/node/sequencer-topos/ &&
       cp -vr /data/subnet/topos/genesis.json /data/subnet/topos-sequencer/genesis.json &&
       echo \"Topos Core contract address:\" $(printenv TOPOS_CORE_PROXY_CONTRACT_ADDRESS), set manually in config &&
       topos node up --name sequencer-topos --home /data/ --no-edge-process"
+    healthcheck:
+      test: ["CMD-SHELL", "test $(netstat -ntu | grep ':8545' | wc -l) -gt 0"]
+      interval: 1s
+      retries: 10
     environment:
       - RUST_LOG=info,topos=debug
       - TOOLCHAIN_VERSION=stable

tests/utils.sh

@@ -21,18 +21,13 @@ stop_network () {
 
 check_network_health () {
     # Check if all relevant containers are healthy
-    # TODO: Add healcheck for sequencers, currenly only tce and polygon edge nodes are checked
-    SERVICE_NAMES=$(docker compose config --dry-run --services | grep -e node)
+    SERVICE_NAMES=$(docker compose config --dry-run --services | grep -e node -e sequencer)
     EXPECTED=$(docker compose ps -aq $SERVICE_NAMES | wc -l)
     COUNT=$(docker inspect --format "{{.State.Health.Status }}" $(docker compose ps -aq $SERVICE_NAMES) | grep "^healthy$"|wc -l)
-    echo "Number of Healthy containers: $COUNT"
     if [[ $COUNT -eq $EXPECTED ]]; then
-        echo "All expected containers healthy"
-        return 0
+        echo 0
     else
-        echo "Unhealthy containers"
-        docker compose -f tools/docker-compose.yml ps -a peer boot
-        return 1
+        echo 1
     fi
 }