From 5ab3a0fbef71ed2c141f316b1eeea83fbaf33eae Mon Sep 17 00:00:00 2001 From: sampocs Date: Wed, 28 Aug 2024 11:21:58 -0500 Subject: [PATCH] added relayer readiness probe --- integration-tests/Makefile | 17 +------ .../network/scripts/init-chain.sh | 2 +- .../network/scripts/network-readiness.sh | 28 ++++++++++++ .../{readiness.sh => node-readiness.sh} | 0 .../network/scripts/relayer-readiness.sh | 45 +++++++++++++++++++ .../network/scripts/start-hermes.sh | 3 +- .../network/templates/relayer.yaml | 6 +++ .../network/templates/validator.yaml | 2 +- 8 files changed, 84 insertions(+), 19 deletions(-) create mode 100644 integration-tests/network/scripts/network-readiness.sh rename integration-tests/network/scripts/{readiness.sh => node-readiness.sh} (100%) create mode 100644 integration-tests/network/scripts/relayer-readiness.sh diff --git a/integration-tests/Makefile b/integration-tests/Makefile index dbd5b93f0..dff11dfc3 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -31,22 +31,7 @@ check-empty-namespace: if [ $$POD_COUNT -eq 0 ]; then exit 0; else exit 1; fi wait-for-startup: - @echo "---" && sleep 1 - @printf "Waiting for network to startup..." - @elapsed=0; \ - while true; do \ - not_ready_pods=$$($(KUBECTL) get pods --no-headers -n $(K8S_NAMESPACE) | grep -v '1/1 *Running' | wc -l); \ - if [ $$not_ready_pods -eq 0 ]; then \ - printf "Ready! 🚀\n"; \ - break; \ - fi; \ - if [ $$elapsed -eq 30 ]; then \ - printf "\nThe network's taking longer than expected to startup. Please investigate\n"; \ - exit 1; \ - fi; \ - sleep 1 && printf "."; \ - elapsed=$$((elapsed + 1)); \ - done + @bash network/scripts/network-readiness.sh $(K8S_NAMESPACE) python-install: conda create --name $(VENV_NAME) python=3.11 -y diff --git a/integration-tests/network/scripts/init-chain.sh b/integration-tests/network/scripts/init-chain.sh index f23a7ad9d..f00e67a23 100644 --- a/integration-tests/network/scripts/init-chain.sh +++ b/integration-tests/network/scripts/init-chain.sh @@ -140,7 +140,7 @@ update_stride_genesis() { $BINARY add-consumer-section --validator-public-keys $validator_public_keys jq_inplace '.app_state.ccvconsumer.params.unbonding_period |= "'$UNBONDING_TIME'"' $genesis_json - jq_inplace '.app_state.airdrop.params.period_length_seconds |= "'${AIRDROP_PERIOD_LENGTH}'"' $genesis_json + jq_inplace '.app_state.airdrop.params.period_length_seconds |= "'${AIRDROP_PERIOD_SECONDS}'"' $genesis_json } # Genesis updates specific to non-stride chains diff --git a/integration-tests/network/scripts/network-readiness.sh b/integration-tests/network/scripts/network-readiness.sh new file mode 100644 index 000000000..f16b0b0d9 --- /dev/null +++ b/integration-tests/network/scripts/network-readiness.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eu + +namespace="$1" + +echo "---" && sleep 1 +printf "Waiting for network to startup..." + +# TODO: fix this +trap 'printf "\nInterrupted by user.\n"; exit' INT + +elapsed=0 +while true; do + not_ready_pods=$(kubectl get pods --no-headers -n $namespace | grep -v '1/1 *Running' | wc -l) + + if [[ $not_ready_pods -eq 0 ]]; then + printf "Ready! 🚀\n" + break + fi + if [[ $elapsed -eq 60 ]]; then + printf "\nThe network's taking longer than expected to startup. Please investigate via `kubectl get pods`\n" + exit 1 + fi + + sleep 2 && printf "." + elapsed=$((elapsed + 1)) +done \ No newline at end of file diff --git a/integration-tests/network/scripts/readiness.sh b/integration-tests/network/scripts/node-readiness.sh similarity index 100% rename from integration-tests/network/scripts/readiness.sh rename to integration-tests/network/scripts/node-readiness.sh diff --git a/integration-tests/network/scripts/relayer-readiness.sh b/integration-tests/network/scripts/relayer-readiness.sh new file mode 100644 index 000000000..1b85a2202 --- /dev/null +++ b/integration-tests/network/scripts/relayer-readiness.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -e + +relayer_type="$1" + +chain_id="${CHAIN_NAME_A}-test-1" +relayer_config_file=${HOME}/.relayer/config/config.yaml +hermes_config_file=${HOME}/.hermes/config.toml + +if [[ "$relayer_type" == "relayer" ]]; then + if [[ ! -f $relayer_config_file ]]; then + echo "Config not initialized yet" + exit 1 + fi + if ! rly q channels "$CHAIN_NAME_A" 2>/dev/null | grep -q STATE_OPEN; then + echo "Source channel not open yet" + exit 1 + fi + if ! rly q channels "$CHAIN_NAME_B" 2>/dev/null | grep -q STATE_OPEN; then + echo "Destination channel not open yet" + exit 1 + fi + exit 0 + +elif [[ "$relayer_type" == "hermes" ]]; then + if [[ ! -f $hermes_config_file ]]; then + echo "Config not initialized yet" + exit 1 + fi + open_channels=$(hermes query channels --chain "$chain_id" --show-counterparty 2>/dev/null | grep -o "channel-" | wc -l) + if [[ "$open_channels" == "0" ]]; then + echo "Source channel not open yet" + exit 1 + fi + if [[ "$open_channels" == "1" ]]; then + echo "Destination channel not open yet" + exit 1 + fi + exit 0 + +else + echo "ERROR: Unsupported relayer type $relayer_type" + exit 1 +fi diff --git a/integration-tests/network/scripts/start-hermes.sh b/integration-tests/network/scripts/start-hermes.sh index 938d2b476..629fb7454 100644 --- a/integration-tests/network/scripts/start-hermes.sh +++ b/integration-tests/network/scripts/start-hermes.sh @@ -10,6 +10,8 @@ CHAIN_ID_B=${CHAIN_NAME_B}-test-1 wait_for_node $CHAIN_NAME_A wait_for_node $CHAIN_NAME_B +hermes_config_file=${HOME}/.hermes/config.toml + restore_keys() { mnemonic_a=$(jq -r '.relayers[$index].mnemonic' --argjson index "$CHAIN_A_MNEMONIC_INDEX" ${KEYS_FILE}) mnemonic_b=$(jq -r '.relayers[$index].mnemonic' --argjson index "$CHAIN_B_MNEMONIC_INDEX" ${KEYS_FILE}) @@ -35,7 +37,6 @@ create_path() { main() { # The config is mounted from a configmap which is read-only by default # In order to make it writeable, we need to copy it to a new location - hermes_config_file=${HOME}/.hermes/config.toml mkdir -p $(dirname $hermes_config_file) cp configs/hermes.toml ${hermes_config_file} diff --git a/integration-tests/network/templates/relayer.yaml b/integration-tests/network/templates/relayer.yaml index 4b3785c4e..397361d0f 100644 --- a/integration-tests/network/templates/relayer.yaml +++ b/integration-tests/network/templates/relayer.yaml @@ -44,6 +44,12 @@ spec: value: "{{ $chainMnemonicIndexA }}" - name: CHAIN_B_MNEMONIC_INDEX value: "{{ $chainMnemonicIndexB }}" + readinessProbe: + exec: + command: ["bash", "scripts/relayer-readiness.sh", "{{ $relayer.type }}"] + periodSeconds: 2 + initialDelaySeconds: 30 + timeoutSeconds: 10 resources: limits: cpu: "200m" diff --git a/integration-tests/network/templates/validator.yaml b/integration-tests/network/templates/validator.yaml index fdf269fc1..e08dc0f72 100644 --- a/integration-tests/network/templates/validator.yaml +++ b/integration-tests/network/templates/validator.yaml @@ -75,7 +75,7 @@ spec: command: ["bash", "scripts/create-validator.sh"] readinessProbe: exec: - command: ["bash", "scripts/readiness.sh"] + command: ["bash", "scripts/node-readiness.sh"] periodSeconds: 10 env: {{- include "chain.env" $chain | nindent 10 }}