From 158443e19b8389a93258ea9090e7128ae49a87e3 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Thu, 12 Dec 2024 17:36:18 +0200 Subject: [PATCH 01/13] Fix Value error and minor refactor --- tools/rmb_tester/rmb_tester.py | 47 +++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/tools/rmb_tester/rmb_tester.py b/tools/rmb_tester/rmb_tester.py index 9224af1..bdabf8a 100755 --- a/tools/rmb_tester/rmb_tester.py +++ b/tools/rmb_tester/rmb_tester.py @@ -74,24 +74,33 @@ def send_all(messages): responses_expected = 0 return_queues = [] with alive_bar(len(messages), title='Sending ..', title_length=12) as bar: - for msg in messages: - r.lpush("msgbus.system.local", msg.to_json()) - responses_expected += len(msg.twin_dst) - return_queues += [msg.reply_to] - bar() + with r.pipeline() as pipe: + for msg in messages: + pipe.lpush("msgbus.system.local", msg.to_json()) + responses_expected += len(msg.twin_dst) + return_queues += [msg.reply_to] + bar() + pipe.execute() # Execute all commands in the pipeline at once return responses_expected, return_queues def wait_all(responses_expected, return_queues, timeout): - responses = [] - err_count = 0 - success_count = 0 - with alive_bar(responses_expected, title='Waiting ..', title_length=12) as bar: - for _ in range(responses_expected): - start = timer() - result = r.blpop(return_queues, timeout=timeout) - if not result: - break - timeout = timeout - round(timer() - start, 3) + responses = [] + err_count = 0 + success_count = 0 + start_time = timer() + + with alive_bar(responses_expected, title='Waiting ..', title_length=12) as bar: + while responses_expected > 0: + elapsed_time = timer() - start_time + remaining_time = timeout - elapsed_time + + if remaining_time <= 0: + print("Timeout reached, stopping waiting for responses.") + break + + # Use the remaining time for the blpop timeout + result = r.blpop(return_queues, timeout=remaining_time) + if result: response = Message.from_json(result[1]) responses.append(response) if response.err is not None: @@ -101,7 +110,9 @@ def wait_all(responses_expected, return_queues, timeout): success_count += 1 bar.text(f'received a response from twin {response.twin_src} ✅') bar() - return responses, err_count, success_count + responses_expected -= 1 + + return responses, err_count, success_count def main(): global r @@ -135,7 +146,7 @@ def main(): print(f"received_success: {success_count}") print(f"received_errors: {err_count}") print(f"no response errors (client give up): {no_responses}") - responding = {int(response.twin_src) for response in responses} + responding = {int(response.twin_src) for response in responses if response.twin_src != "" } not_responding = set(args.dest) - responding print(f"twins not responding (twin IDs): {' '.join(map(str, not_responding))}") print(f"elapsed time: {elapsed_time}") @@ -144,7 +155,7 @@ def main(): print("Responses:") print("=======================") for response in responses: - print(response) + print({k: v for k, v in response.__dict__.items() if v}) print("=======================") print("Errors:") print("=======================") From 2780f54a2c21d514ff4ba0e486d6c47146018346 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Thu, 12 Dec 2024 17:37:04 +0200 Subject: [PATCH 02/13] Use zos.system.version instaed of rmb.version to test live nodes --- tools/rmb_tester/test_live_nodes.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/rmb_tester/test_live_nodes.sh b/tools/rmb_tester/test_live_nodes.sh index 77c6459..045227d 100755 --- a/tools/rmb_tester/test_live_nodes.sh +++ b/tools/rmb_tester/test_live_nodes.sh @@ -20,7 +20,7 @@ fi RMB_LOG_FILE="./rmb-peer.log" TIMEOUT="${TIMEOUT:-60}" RMB_BIN="${RMB_BIN:-../../target/x86_64-unknown-linux-musl/release/rmb-peer}" - +VERBOSE="${VERBOSE:-false}" cleanup() { echo "stop all bash managed jobs" jlist=$(jobs -p) @@ -36,7 +36,7 @@ trap cleanup SIGHUP SIGINT SIGQUIT SIGABRT SIGTERM set +e echo "redis-server starting .." -redis-server --port 6379 2>&1 > /dev/null& +redis-server --port 6379 > /dev/null 2>&1 & sleep 3 set -e @@ -45,12 +45,12 @@ echo "rmb-peer starting .." $RMB_BIN -m "$MNEMONIC" --substrate "$SUBSTRATE_URL" --relay "$RELAY_URL" --redis "redis://localhost:6379" --debug &> $RMB_LOG_FILE & # wait till peer establish connection to a relay -timeout --preserve-status 10 tail -f -n0 $RMB_LOG_FILE | grep -qe 'now connected' || (echo "rmb-peer taking too much time to start! check the log at $RMB_LOG_FILE for more info." && cleanup) +timeout --preserve-status 20 tail -f -n0 $RMB_LOG_FILE | grep -qe 'now connected' || (echo "rmb-peer taking too much time to start! check the log at $RMB_LOG_FILE for more info." && cleanup) # start rmb_tester source venv/bin/activate echo "rmb_tester starting .." -python3 ./rmb_tester.py -d $(./scripts/twins.sh --likely-up $1) -c "rmb.version" -t $TIMEOUT -e $TIMEOUT --short +python3 ./rmb_tester.py -d $(./scripts/twins.sh --likely-up $1) -c "zos.system.version" -t "$TIMEOUT" -e "$TIMEOUT" "$(if [[ "$VERBOSE" == "false" ]]; then echo "--short"; fi)" deactivate cleanup From d77b15e5493bdcac02d1f06e20a417078be666d9 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Thu, 12 Dec 2024 17:37:16 +0200 Subject: [PATCH 03/13] update documention --- tools/rmb_tester/README.md | 39 +++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/tools/rmb_tester/README.md b/tools/rmb_tester/README.md index a98f663..db4c1c9 100644 --- a/tools/rmb_tester/README.md +++ b/tools/rmb_tester/README.md @@ -2,30 +2,41 @@ You can find here CLI tools and scripts that can be used for testing and benchmarking [RMB](https://github.com/threefoldtech/rmb-rs). You can use either RMB_Tester, RMB_echo, or both to quickly test the communications over RMB. -## Installation: +## Installation + - clone the repo - create a new env + ```py python3 -m venv venv ``` + - activate the new env + ```py source ./venv/bin/activate ``` + - install dependencies + ```py pip install -r requirements.txt ``` -## Usage: +## Usage + RMB tools comprise two Python programs that can be used independently or in conjunction with each other. ### RMB_Tester + RMB_Tester is a CLI tool that serves as an RMB client to automate the process of crafting a specified number of test messages to be sent to one or more destinations. The number of messages, command, data, destination list, and other parameters can be configured through the command line. The tool will wait for the correct number of responses and report some statistics. Please ensure that there is a process running on the destination side that can handle this command and respond back or use RMB_echo for this purpose. +Also, note that the rmb.version built-in command mentioned in this document is specific to the Rust rmb-peer implementation and is not guaranteed to be available in other RMB implementations. ZOS nodes no longer use the Rust rmb-peer. If you run this tool against a ZOS node, you must use a registered command, such as zos.system.version. + example: + ```sh # We sending to two destinations # The default test command will be used and can be handled by RMB_echo process @@ -35,62 +46,79 @@ python3 ./rmb_tester.py --dest 41 55 to just print the summary use `--short` option to override default command use the `--command` + ```sh # The `rmb.version` command will be handled by RMB process itself python3 ./rmb_tester.py --dest 41 --command rmb.version ``` for all optional args see + ```sh python3 ./rmb_tester.py -h ``` ### RMB_Echo (message handler) + This tool will automate handling the messages coming to $queue and respond with same message back to the source and display the count of processed messages. example: + ```sh python3 ./msg_handler.py ``` or specify the redis queue (command) to handle the messages from + ```sh python3 ./msg_handler.py --queue helloworld ``` for all optional args see + ```sh python3 ./msg_handler.py -h ``` -## Recipes: -### Simple method for testing live nodes: +## Recipes + +### Simple method for testing live nodes + - For simplicity, you can install this tool's dependencies by running the ``install.sh` script: + ```sh ./install ``` you can start testing live nodes if it is reachable over rmb by running `test-live-nodes.sh` script. it takes only one argument, the network name (one of `dev`, `qa`, `test`, `main`) and required to pass set you mnemonic as env var `MNEMONIC`. for testing dev network nodes: + ```sh MNEMONIC="[YOUR MNEMONIC]" ./test_live_nodes.sh dev ``` + optionally, set `TIMEOUT` and/or `RMB_BIN`. `TIMEOUT` : set message ttl and client timeout. default to 60 (for large number of destinations use appropriate value) `RMB_BIN` : set the path of the rmb_peer binary file. default to `../../target/x86_64-unknown-linux-musl/release/rmb-peer` +Additionally, you can set VERBOSE to true (or any non-empty value) to display detailed response and error messages. + ```sh MNEMONIC="[YOUR MNEMONIC]" TIMEOUT=500 ./test_live_nodes.sh main ``` -### More Customized method: +### More Customized method + - Test all dest twins to ensure that they are reachable over RMB + ```sh # The nodes.sh script when used with `--likely-up` option will output the IDs of the online nodes in the network using the gridproxy API. python3 ./rmb_tester.py -d $(./scripts/twins.sh --likely-up main) -c "rmb.version" -t 600 -e 600 ``` + Note: this tool is for testing purposes and not optimized for speed, for large number of destinations use appropriate expiration and timeout values. you can copy and paste all non responsive twins and run `./twinid_to_nodeid.sh` with the list of twins ids for easy lookup node id and verfiying the status (like know if node in standby mode). + ```sh ./scripts/twinid_to_nodeid.sh main 2562 5666 2086 2092 ``` @@ -99,6 +127,7 @@ First arg is network (one of `dev`, `qa`, `test`, `main`) Then you follow it with space separated list of twin ids the output would be like + ```sh twin ID: 2562 node ID: 1419 status: up twin ID: 5666 node ID: 3568 status: up From d9c3be36a968714b898642ac2fa7abeb8f7b4b8e Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Thu, 12 Dec 2024 18:32:32 +0200 Subject: [PATCH 04/13] doc: fix typo --- tools/rmb_tester/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/rmb_tester/README.md b/tools/rmb_tester/README.md index db4c1c9..ecba6e1 100644 --- a/tools/rmb_tester/README.md +++ b/tools/rmb_tester/README.md @@ -87,7 +87,7 @@ python3 ./msg_handler.py -h - For simplicity, you can install this tool's dependencies by running the ``install.sh` script: ```sh -./install +./install.sh ``` you can start testing live nodes if it is reachable over rmb by running `test-live-nodes.sh` script. it takes only one argument, the network name (one of `dev`, `qa`, `test`, `main`) and required to pass set you mnemonic as env var `MNEMONIC`. for testing dev network nodes: From fb9bf80728cc272389025e7834e73d9cbbbe5ae3 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Thu, 12 Dec 2024 21:26:22 +0200 Subject: [PATCH 05/13] Add a clear visual indicator for the outcome and a show the percentage of failures --- tools/rmb_tester/rmb_tester.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/rmb_tester/rmb_tester.py b/tools/rmb_tester/rmb_tester.py index bdabf8a..4632100 100755 --- a/tools/rmb_tester/rmb_tester.py +++ b/tools/rmb_tester/rmb_tester.py @@ -88,6 +88,7 @@ def wait_all(responses_expected, return_queues, timeout): err_count = 0 success_count = 0 start_time = timer() + timedout = False with alive_bar(responses_expected, title='Waiting ..', title_length=12) as bar: while responses_expected > 0: @@ -95,7 +96,7 @@ def wait_all(responses_expected, return_queues, timeout): remaining_time = timeout - elapsed_time if remaining_time <= 0: - print("Timeout reached, stopping waiting for responses.") + timedout = True break # Use the remaining time for the blpop timeout @@ -111,6 +112,8 @@ def wait_all(responses_expected, return_queues, timeout): bar.text(f'received a response from twin {response.twin_src} ✅') bar() responses_expected -= 1 + if timedout: + print("Timeout reached, stopping waiting for responses.") return responses, err_count, success_count @@ -150,6 +153,12 @@ def main(): not_responding = set(args.dest) - responding print(f"twins not responding (twin IDs): {' '.join(map(str, not_responding))}") print(f"elapsed time: {elapsed_time}") + if responses_expected == success_count: + print("🎉 All responses received successfully! 🎉") + else: + missing_responses = (no_responses / responses_expected) * 100 + print(f"⚠️ Warning: {missing_responses:.2f}% of responses are missing! ⚠️") + print("=======================") if not args.short: print("Responses:") From c0d5081ee274feb3d3e19117bb82235f8f72e4e0 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Thu, 12 Dec 2024 21:28:36 +0200 Subject: [PATCH 06/13] Fix: ensure a single rmb-peer instance runs during the test_live_nodes.sh script and that the Redis database is clean --- tools/rmb_tester/test_live_nodes.sh | 46 +++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/tools/rmb_tester/test_live_nodes.sh b/tools/rmb_tester/test_live_nodes.sh index 045227d..47be7a4 100755 --- a/tools/rmb_tester/test_live_nodes.sh +++ b/tools/rmb_tester/test_live_nodes.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash case $1 in - main|dev|qa|test ) # Ok - ;; - *) - # The wrong first argument. - echo 'Expected "dev", "qa", "test", or "main" as second arg' >&2 - exit 1 + main|dev|qa|test ) # Ok + ;; + *) + # The wrong first argument. + echo 'Expected "dev", "qa", "test", or "main" as second arg' >&2 + exit 1 esac @@ -22,11 +22,20 @@ TIMEOUT="${TIMEOUT:-60}" RMB_BIN="${RMB_BIN:-../../target/x86_64-unknown-linux-musl/release/rmb-peer}" VERBOSE="${VERBOSE:-false}" cleanup() { + set +e + if command -v deactivate &> /dev/null; then + deactivate + fi + jlist=$(jobs -p) + plist=$(ps --ppid $$ | awk '/[0-9]/{print $1}' | grep -v -E "^$$|^$(pgrep -f 'ps')|^$(pgrep -f 'awk')|^$(pgrep -f 'grep')$") + pids=${jlist:-$plist} + if [ -n "$pids" ]; then echo "stop all bash managed jobs" - jlist=$(jobs -p) - plist=$(ps --ppid $$ | awk '/[0-9]/{print $1}') - - kill ${jlist:-$plist} + kill $pids + else + echo "No processes to stop." + fi + exit } trap cleanup SIGHUP SIGINT SIGQUIT SIGABRT SIGTERM @@ -38,10 +47,22 @@ echo "redis-server starting .." redis-server --port 6379 > /dev/null 2>&1 & sleep 3 +# clear all databases +echo "Removes all keys in Redis" +redis-cli -p 6379 FLUSHALL set -e +# ensure that RMB is not already running +if pgrep -x $(basename "$RMB_BIN") > /dev/null; then + echo "Another instance of rmb-peer is already running. Killing..." + pkill -x $(basename "$RMB_BIN") +fi + +# ensure the MNEMONIC has no leading or trailing spaces +MNEMONIC="${MNEMONIC#"${MNEMONIC%%[![:space:]]*}"}"; MNEMONIC="${MNEMONIC%"${MNEMONIC##*[![:space:]]}"}" + # start rmb in background -echo "rmb-peer starting .." +echo "rmb-peer starting ("$1"net).." $RMB_BIN -m "$MNEMONIC" --substrate "$SUBSTRATE_URL" --relay "$RELAY_URL" --redis "redis://localhost:6379" --debug &> $RMB_LOG_FILE & # wait till peer establish connection to a relay @@ -50,7 +71,6 @@ timeout --preserve-status 20 tail -f -n0 $RMB_LOG_FILE | grep -qe 'now connected # start rmb_tester source venv/bin/activate echo "rmb_tester starting .." -python3 ./rmb_tester.py -d $(./scripts/twins.sh --likely-up $1) -c "zos.system.version" -t "$TIMEOUT" -e "$TIMEOUT" "$(if [[ "$VERBOSE" == "false" ]]; then echo "--short"; fi)" -deactivate +python3 ./rmb_tester.py -d $(./scripts/twins.sh --likely-up $1) -c "zos.system.version" -t "$TIMEOUT" -e "$TIMEOUT" $(if [[ "$VERBOSE" == "false" ]]; then echo "--short"; fi) cleanup From cb9c4c98c99dc08843b421d56adf5829e204bce6 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Mon, 16 Dec 2024 18:28:08 +0200 Subject: [PATCH 07/13] fix: hardening the script's error handling --- tools/rmb_tester/test_live_nodes.sh | 30 +++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tools/rmb_tester/test_live_nodes.sh b/tools/rmb_tester/test_live_nodes.sh index 47be7a4..1e032e7 100755 --- a/tools/rmb_tester/test_live_nodes.sh +++ b/tools/rmb_tester/test_live_nodes.sh @@ -9,6 +9,12 @@ case $1 in exit 1 esac +if [ -z "$MNEMONIC" ]; then + echo "MNEMONIC is not set" + echo "Please set the MNEMONIC environment variable" + echo "Example: MNEMONIC='...' ./test_live_nodes.sh " + exit 1 +fi if [[ "$1" == "main" ]]; then SUBSTRATE_URL="wss://tfchain.grid.tf:443" @@ -23,27 +29,31 @@ RMB_BIN="${RMB_BIN:-../../target/x86_64-unknown-linux-musl/release/rmb-peer}" VERBOSE="${VERBOSE:-false}" cleanup() { set +e - if command -v deactivate &> /dev/null; then + echo "cleaning up initiated" + if [ -n "$VIRTUAL_ENV" ]; then + echo "deactivating virtual environment" deactivate fi - jlist=$(jobs -p) + # close redis-server + echo "closing redis-server ..." + redis-cli -p 6379 shutdown + jlist=$(jobs -pr) plist=$(ps --ppid $$ | awk '/[0-9]/{print $1}' | grep -v -E "^$$|^$(pgrep -f 'ps')|^$(pgrep -f 'awk')|^$(pgrep -f 'grep')$") pids=${jlist:-$plist} if [ -n "$pids" ]; then - echo "stop all bash managed jobs" + echo "stop rmb-peer and all bash managed jobs" kill $pids else - echo "No processes to stop." + echo "All jobs in this bash session have completed or stoped, so there are none left to clean up." fi - exit } trap cleanup SIGHUP SIGINT SIGQUIT SIGABRT SIGTERM - +echo "starting live nodes rmb test version $(git describe --tags)" # start redis in backgroud and skip errors in case alreday running set +e -echo "redis-server starting .." +echo "redis-server starting ..." redis-server --port 6379 > /dev/null 2>&1 & sleep 3 @@ -66,7 +76,11 @@ echo "rmb-peer starting ("$1"net).." $RMB_BIN -m "$MNEMONIC" --substrate "$SUBSTRATE_URL" --relay "$RELAY_URL" --redis "redis://localhost:6379" --debug &> $RMB_LOG_FILE & # wait till peer establish connection to a relay -timeout --preserve-status 20 tail -f -n0 $RMB_LOG_FILE | grep -qe 'now connected' || (echo "rmb-peer taking too much time to start! check the log at $RMB_LOG_FILE for more info." && cleanup) +if ! timeout --preserve-status 20 tail -f -n0 $RMB_LOG_FILE | grep -qe 'now connected'; then + echo "rmb-peer taking too much time to start! check the log at $RMB_LOG_FILE for more info." + cleanup + exit 1 +fi # start rmb_tester source venv/bin/activate From 5bb41a9ba2d5b25d15a6b03741cecb4f7c8e5186 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Mon, 16 Dec 2024 18:56:18 +0200 Subject: [PATCH 08/13] show the final outcome in diffrent color --- tools/rmb_tester/rmb_tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/rmb_tester/rmb_tester.py b/tools/rmb_tester/rmb_tester.py index 4632100..35c5453 100755 --- a/tools/rmb_tester/rmb_tester.py +++ b/tools/rmb_tester/rmb_tester.py @@ -154,10 +154,10 @@ def main(): print(f"twins not responding (twin IDs): {' '.join(map(str, not_responding))}") print(f"elapsed time: {elapsed_time}") if responses_expected == success_count: - print("🎉 All responses received successfully! 🎉") + print("\033[92m🎉 All responses received successfully! 🎉\033[0m") else: missing_responses = (no_responses / responses_expected) * 100 - print(f"⚠️ Warning: {missing_responses:.2f}% of responses are missing! ⚠️") + print("\033[93m⚠️ Warning: {missing_responses:.2f}% of responses are missing! ⚠️\033[0m") print("=======================") if not args.short: From 963611e89fc37a4afb2869a7472fed3150623002 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Mon, 16 Dec 2024 19:53:02 +0200 Subject: [PATCH 09/13] style: convert double quotes (") to single quotes (') for static strings that don't need variable expansion --- tools/rmb_tester/test_live_nodes.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tools/rmb_tester/test_live_nodes.sh b/tools/rmb_tester/test_live_nodes.sh index 1e032e7..550cb78 100755 --- a/tools/rmb_tester/test_live_nodes.sh +++ b/tools/rmb_tester/test_live_nodes.sh @@ -10,32 +10,32 @@ case $1 in esac if [ -z "$MNEMONIC" ]; then - echo "MNEMONIC is not set" - echo "Please set the MNEMONIC environment variable" - echo "Example: MNEMONIC='...' ./test_live_nodes.sh " + echo 'MNEMONIC is not set' + echo 'Please set the MNEMONIC environment variable' + echo 'Example: MNEMONIC="..." ./test_live_nodes.sh ' exit 1 fi -if [[ "$1" == "main" ]]; then - SUBSTRATE_URL="wss://tfchain.grid.tf:443" - RELAY_URL="wss://relay.grid.tf" +if [[ "$1" == 'main' ]]; then + SUBSTRATE_URL='wss://tfchain.grid.tf:443' + RELAY_URL='wss://relay.grid.tf' else SUBSTRATE_URL="wss://tfchain.$1.grid.tf:443" RELAY_URL="wss://relay.$1.grid.tf" fi -RMB_LOG_FILE="./rmb-peer.log" +RMB_LOG_FILE='./rmb-peer.log' TIMEOUT="${TIMEOUT:-60}" RMB_BIN="${RMB_BIN:-../../target/x86_64-unknown-linux-musl/release/rmb-peer}" VERBOSE="${VERBOSE:-false}" cleanup() { set +e - echo "cleaning up initiated" + echo 'cleaning up initiated' if [ -n "$VIRTUAL_ENV" ]; then - echo "deactivating virtual environment" + echo 'deactivating virtual environment' deactivate fi # close redis-server - echo "closing redis-server ..." + echo 'closing redis-server ...' redis-cli -p 6379 shutdown jlist=$(jobs -pr) plist=$(ps --ppid $$ | awk '/[0-9]/{print $1}' | grep -v -E "^$$|^$(pgrep -f 'ps')|^$(pgrep -f 'awk')|^$(pgrep -f 'grep')$") @@ -53,18 +53,18 @@ trap cleanup SIGHUP SIGINT SIGQUIT SIGABRT SIGTERM echo "starting live nodes rmb test version $(git describe --tags)" # start redis in backgroud and skip errors in case alreday running set +e -echo "redis-server starting ..." +echo 'redis-server starting ...' redis-server --port 6379 > /dev/null 2>&1 & sleep 3 # clear all databases -echo "Removes all keys in Redis" +echo 'Removes all keys in Redis' redis-cli -p 6379 FLUSHALL set -e # ensure that RMB is not already running if pgrep -x $(basename "$RMB_BIN") > /dev/null; then - echo "Another instance of rmb-peer is already running. Killing..." + echo 'Another instance of rmb-peer is already running. Killing...' pkill -x $(basename "$RMB_BIN") fi From e9d52303a1be07a201b10aef7cb77f99fd5e4c62 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Mon, 16 Dec 2024 19:54:24 +0200 Subject: [PATCH 10/13] fix: adding back the f prefix to the string literal --- tools/rmb_tester/rmb_tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/rmb_tester/rmb_tester.py b/tools/rmb_tester/rmb_tester.py index 35c5453..887df91 100755 --- a/tools/rmb_tester/rmb_tester.py +++ b/tools/rmb_tester/rmb_tester.py @@ -157,7 +157,7 @@ def main(): print("\033[92m🎉 All responses received successfully! 🎉\033[0m") else: missing_responses = (no_responses / responses_expected) * 100 - print("\033[93m⚠️ Warning: {missing_responses:.2f}% of responses are missing! ⚠️\033[0m") + print(f"\033[93m⚠️ Warning: {missing_responses:.2f}% of responses are missing! ⚠️\033[0m") print("=======================") if not args.short: From d538658d8254aba1865e3799dc7d9fbe72ff3f27 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Mon, 16 Dec 2024 20:09:16 +0200 Subject: [PATCH 11/13] improve logging and debug output in the script --- tools/rmb_tester/test_live_nodes.sh | 31 +++++++++++++++++++---------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tools/rmb_tester/test_live_nodes.sh b/tools/rmb_tester/test_live_nodes.sh index 550cb78..e86807d 100755 --- a/tools/rmb_tester/test_live_nodes.sh +++ b/tools/rmb_tester/test_live_nodes.sh @@ -27,39 +27,48 @@ RMB_LOG_FILE='./rmb-peer.log' TIMEOUT="${TIMEOUT:-60}" RMB_BIN="${RMB_BIN:-../../target/x86_64-unknown-linux-musl/release/rmb-peer}" VERBOSE="${VERBOSE:-false}" +DEBUG="${DEBUG:-false}" + cleanup() { set +e - echo 'cleaning up initiated' + debug 'cleaning up initiated' if [ -n "$VIRTUAL_ENV" ]; then - echo 'deactivating virtual environment' + debug 'deactivating virtual environment' deactivate fi # close redis-server - echo 'closing redis-server ...' + debug 'closing redis-server ...' redis-cli -p 6379 shutdown jlist=$(jobs -pr) plist=$(ps --ppid $$ | awk '/[0-9]/{print $1}' | grep -v -E "^$$|^$(pgrep -f 'ps')|^$(pgrep -f 'awk')|^$(pgrep -f 'grep')$") pids=${jlist:-$plist} if [ -n "$pids" ]; then - echo "stop rmb-peer and all bash managed jobs" + debug "stop rmb-peer and all bash managed jobs" kill $pids else - echo "All jobs in this bash session have completed or stoped, so there are none left to clean up." + debug "All jobs in this bash session have completed or stoped, so there are none left to clean up." fi } +debug() { + if [[ "$DEBUG" == "true" ]]; then + echo "$@" + fi +} + trap cleanup SIGHUP SIGINT SIGQUIT SIGABRT SIGTERM -echo "starting live nodes rmb test version $(git describe --tags)" +echo "starting live nodes rmb test script version $(git describe --tags)" +echo "network used: $1net" # start redis in backgroud and skip errors in case alreday running set +e -echo 'redis-server starting ...' +debug 'redis-server starting ...' redis-server --port 6379 > /dev/null 2>&1 & sleep 3 # clear all databases -echo 'Removes all keys in Redis' -redis-cli -p 6379 FLUSHALL +debug 'Removes all keys in Redis' +redis-cli -p 6379 FLUSHALL > /dev/null 2>&1 & set -e # ensure that RMB is not already running @@ -72,7 +81,7 @@ fi MNEMONIC="${MNEMONIC#"${MNEMONIC%%[![:space:]]*}"}"; MNEMONIC="${MNEMONIC%"${MNEMONIC##*[![:space:]]}"}" # start rmb in background -echo "rmb-peer starting ("$1"net).." +debug "rmb-peer starting ($1net).." $RMB_BIN -m "$MNEMONIC" --substrate "$SUBSTRATE_URL" --relay "$RELAY_URL" --redis "redis://localhost:6379" --debug &> $RMB_LOG_FILE & # wait till peer establish connection to a relay @@ -84,7 +93,7 @@ fi # start rmb_tester source venv/bin/activate -echo "rmb_tester starting .." +debug "rmb_tester starting .." python3 ./rmb_tester.py -d $(./scripts/twins.sh --likely-up $1) -c "zos.system.version" -t "$TIMEOUT" -e "$TIMEOUT" $(if [[ "$VERBOSE" == "false" ]]; then echo "--short"; fi) cleanup From f21ef9f840d19460e2921e2899fe523d6c2219e0 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Mon, 16 Dec 2024 20:23:27 +0200 Subject: [PATCH 12/13] add precheck for binary path --- tools/rmb_tester/test_live_nodes.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tools/rmb_tester/test_live_nodes.sh b/tools/rmb_tester/test_live_nodes.sh index e86807d..eb0ae1c 100755 --- a/tools/rmb_tester/test_live_nodes.sh +++ b/tools/rmb_tester/test_live_nodes.sh @@ -29,6 +29,13 @@ RMB_BIN="${RMB_BIN:-../../target/x86_64-unknown-linux-musl/release/rmb-peer}" VERBOSE="${VERBOSE:-false}" DEBUG="${DEBUG:-false}" +if [ -f "$RMB_BIN" ]; then + binary_version_output=$( "$RMB_BIN" --version ) +else + echo "rmb-peer binary not found at $RMB_BIN" + exit 1 +fi + cleanup() { set +e debug 'cleaning up initiated' @@ -58,8 +65,10 @@ debug() { trap cleanup SIGHUP SIGINT SIGQUIT SIGABRT SIGTERM -echo "starting live nodes rmb test script version $(git describe --tags)" -echo "network used: $1net" +echo 'starting live nodes rmb test script ...' +echo "network: $1net" +debug "script version: $(git describe --tags)" +debug "rmb-peer version: $binary_version_output" # start redis in backgroud and skip errors in case alreday running set +e debug 'redis-server starting ...' From bf55f61366fc2f4bbb9b1188c19190c456edecb9 Mon Sep 17 00:00:00 2001 From: Sameh Abouel-saad Date: Mon, 16 Dec 2024 20:24:53 +0200 Subject: [PATCH 13/13] update README.md --- tools/rmb_tester/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/rmb_tester/README.md b/tools/rmb_tester/README.md index ecba6e1..5f986ea 100644 --- a/tools/rmb_tester/README.md +++ b/tools/rmb_tester/README.md @@ -100,7 +100,7 @@ optionally, set `TIMEOUT` and/or `RMB_BIN`. `TIMEOUT` : set message ttl and client timeout. default to 60 (for large number of destinations use appropriate value) `RMB_BIN` : set the path of the rmb_peer binary file. default to `../../target/x86_64-unknown-linux-musl/release/rmb-peer` -Additionally, you can set VERBOSE to true (or any non-empty value) to display detailed response and error messages. +Additionally, you can set `VERBOSE` to true (or any non-empty value) to display detailed response and error messages and/or `DEBUG` can be configured to enable debug output. ```sh MNEMONIC="[YOUR MNEMONIC]" TIMEOUT=500 ./test_live_nodes.sh main