Check implementation using Mesh CLI, within GitHub Actions

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+ignore = E501

.github/workflows/check_with_mesh_cli.yml

@@ -0,0 +1,37 @@
+name: Check with Mesh CLI
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          pip3 install requests bottle
+          curl -sSfL https://raw.githubusercontent.com/coinbase/mesh-cli/master/scripts/install.sh | sh -s -- -b "$HOME/.local/bin"
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Build
+        run: |
+          cd $GITHUB_WORKSPACE/cmd/rosetta && go build .
+          cd $GITHUB_WORKSPACE/systemtests && go build ./proxyToObserverAdapter.go
+
+      - name: check:data
+        run: |
+          PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=data --network=testnet
+
+      - name: check:construction
+        run: |
+          PYTHONPATH=. python3 ./systemtests/check_with_mesh_cli.py --mode=construction --network=testnet
+
+

.gitignore

@@ -1,4 +1,7 @@
 cmd/rosetta/rosetta
 cmd/rosetta/logs/**
+systemtests/logs/**
+systemtests/**/check-data/**
+logs/**
 
 **/__pycache__/**

systemtests/check_with_mesh_cli.py

@@ -0,0 +1,155 @@
+import shutil
+import signal
+import subprocess
+import sys
+import time
+from argparse import ArgumentParser
+from typing import Any
+
+import requests
+
+from systemtests import constants
+from systemtests.config import CONFIGURATIONS, Configuration
+
+
+def main() -> int:
+    parser = ArgumentParser()
+    parser.add_argument("--mode", choices=["data", "construction"], required=True)
+    parser.add_argument("--network", choices=CONFIGURATIONS.keys(), required=True)
+    args = parser.parse_args()
+
+    mode = args.mode
+    configuration = CONFIGURATIONS[args.network]
+
+    process_rosetta = run_rosetta(configuration)
+    process_adapter = run_proxy_to_observer_adapter(configuration)
+    process_checker = run_mesh_cli(mode, configuration)
+
+    # Handle termination signals
+    def signal_handler(sig: Any, frame: Any):
+        process_rosetta.kill()
+        process_adapter.kill()
+        process_checker.kill()
+        sys.exit(1)
+
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    # Wait for checker to finish
+    exit_code = process_checker.wait()
+
+    process_rosetta.kill()
+    process_adapter.kill()
+
+    time.sleep(1)
+
+    print(f"Checker finished with exit code: {exit_code}.")
+    return exit_code
+
+
+def run_rosetta(configuration: Configuration):
+    """
+    E.g.
+
+    rosetta --port=7091 --observer-http-url=http://localhost:8080 \
+        --observer-actual-shard=0 --network-id=D --network-name=devnet --native-currency=EGLD \
+        --first-historical-epoch=42 --num-historical-epochs=1
+    """
+
+    current_epoch = get_current_epoch(configuration)
+
+    command = [
+        str(constants.PATH_ROSETTA),
+        f"--port={constants.PORT_ROSETTA}",
+        f"--observer-http-url=http://localhost:{constants.PORT_OBSERVER_SURROGATE}",
+        f"--observer-actual-shard={configuration.network_shard}",
+        f"--network-id={configuration.network_id}",
+        f"--network-name={configuration.network_name}",
+        f"--native-currency={configuration.native_currency}",
+        f"--first-historical-epoch={current_epoch}",
+        f"--num-historical-epochs={configuration.num_historical_epochs}",
+    ]
+
+    return subprocess.Popen(command)
+
+
+def run_proxy_to_observer_adapter(configuration: Configuration):
+    command = [
+        str(constants.PATH_PROXY_TO_OBSERVER_ADAPTER),
+        f"--proxy={configuration.proxy_url}",
+        f"--shard={configuration.network_shard}",
+        f"--sleep={constants.ADAPTER_DELAY_IN_MILLISECONDS}"
+    ]
+
+    return subprocess.Popen(command)
+
+
+def run_mesh_cli(mode: str, configuration: Configuration):
+    if mode == "data":
+        return run_mesh_cli_with_check_data(configuration)
+    elif mode == "construction":
+        return run_mesh_cli_with_check_construction(configuration)
+    else:
+        raise ValueError(f"Unknown mode: {mode}")
+
+
+def run_mesh_cli_with_check_construction(configuration: Configuration):
+    """
+    E.g.
+
+    rosetta-cli check:construction --configuration-file devnet-construction.json \
+        --online-url=http://localhost:7091 --offline-url=http://localhost:7091
+    """
+
+    command = [
+        "rosetta-cli",
+        "check:construction",
+        f"--configuration-file={configuration.check_construction_configuration_file}",
+        f"--online-url=http://localhost:{constants.PORT_ROSETTA}",
+        f"--offline-url=http://localhost:{constants.PORT_ROSETTA}",
+    ]
+
+    return subprocess.Popen(command)
+
+
+def run_mesh_cli_with_check_data(configuration: Configuration):
+    """
+    E.g.
+
+    rosetta-cli check:data --configuration-file devnet-data.json \
+        --online-url=http://localhost:7091 --data-dir=devnet-data
+    """
+
+    shutil.rmtree(configuration.check_data_directory, ignore_errors=True)
+
+    current_epoch = get_current_epoch(configuration)
+    first_historical_epoch = current_epoch - configuration.num_historical_epochs + 1
+    start_block = get_start_of_epoch(configuration, first_historical_epoch)
+
+    command = [
+        "rosetta-cli",
+        "check:data",
+        f"--configuration-file={configuration.check_data_configuration_file}",
+        f"--online-url=http://localhost:{constants.PORT_ROSETTA}",
+        f"--data-dir={configuration.check_data_directory}",
+        f"--start-block={start_block}"
+    ]
+
+    return subprocess.Popen(command)
+
+
+def get_current_epoch(configuration: Configuration) -> int:
+    response = requests.get(f"{configuration.proxy_url}/network/status/{configuration.network_shard}")
+    response.raise_for_status()
+    return response.json()["data"]["status"]["erd_epoch_number"]
+
+
+def get_start_of_epoch(configuration: Configuration, epoch: int) -> int:
+    response = requests.get(f"{configuration.proxy_url}/network/epoch-start/{configuration.network_shard}/by-epoch/{epoch}")
+    response.raise_for_status()
+    return response.json()["data"]["epochStart"]["nonce"]
+
+
+if __name__ == "__main__":
+    exit_code = main()
+    sys.exit(exit_code)

systemtests/config.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Configuration:
+    network_shard: int
+    network_id: str
+    network_name: str
+    native_currency: str
+    num_historical_epochs: int
+    proxy_url: str
+    check_construction_configuration_file: str
+    check_data_configuration_file: str
+    check_data_directory: str
+
+
+CONFIGURATIONS = {
+    "devnet": Configuration(
+        network_shard=0,
+        network_id="D",
+        network_name="untitled",
+        native_currency="EGLD",
+        num_historical_epochs=2,
+        proxy_url="https://devnet-gateway.multiversx.com",
+        check_construction_configuration_file="systemtests/mesh_cli_config/devnet-construction.json",
+        check_data_configuration_file="systemtests/mesh_cli_config/check-data.json",
+        check_data_directory="systemtests/devnet-data",
+    ),
+    "testnet": Configuration(
+        network_shard=0,
+        network_id="T",
+        network_name="untitled",
+        native_currency="EGLD",
+        num_historical_epochs=2,
+        proxy_url="https://testnet-gateway.multiversx.com",
+        check_construction_configuration_file="systemtests/mesh_cli_config/testnet-construction.json",
+        check_data_configuration_file="systemtests/mesh_cli_config/check-data.json",
+        check_data_directory="systemtests/testnet-data",
+    ),
+}

systemtests/constants.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+PATH_REPOSITORY = Path(__file__).parent.parent
+PATH_ROSETTA = PATH_REPOSITORY / "cmd" / "rosetta" / "rosetta"
+PATH_PROXY_TO_OBSERVER_ADAPTER = PATH_REPOSITORY / "systemtests" / "proxyToObserverAdapter"
+PORT_ROSETTA = 7091
+PORT_OBSERVER_SURROGATE = 8080
+ADAPTER_DELAY_IN_MILLISECONDS = 100

systemtests/mesh_cli_config/check-data.json

@@ -0,0 +1,46 @@
+{
+  "network": {
+    "blockchain": "MultiversX",
+    "network": "untitled"
+  },
+  "http_timeout": 30,
+  "max_retries": 10,
+  "retry_elapsed_time": 20,
+  "max_online_connections": 8,
+  "max_sync_concurrency": 4,
+  "tip_delay": 10,
+  "max_reorg_depth": 100,
+  "log_configuration": false,
+  "compression_disabled": false,
+  "l0_in_memory_enabled": false,
+  "error_stack_trace_disabled": false,
+  "coin_supported": false,
+  "construction": null,
+  "data": {
+    "active_reconciliation_concurrency": 8,
+    "inactive_reconciliation_concurrency": 1,
+    "inactive_reconciliation_frequency": 8,
+    "log_blocks": false,
+    "log_transactions": true,
+    "log_balance_changes": true,
+    "log_reconciliations": true,
+    "ignore_reconciliation_error": false,
+    "reconciliation_disabled": false,
+    "reconciliation_drain_disabled": false,
+    "inactive_discrepancy_search_disabled": false,
+    "balance_tracking_disabled": false,
+    "coin_tracking_disabled": false,
+    "status_port": 9091,
+    "results_output_file": "",
+    "pruning_block_disabled": false,
+    "pruning_balance_disabled": false,
+    "initial_balance_fetch_disabled": false,
+    "historical_balance_disabled": false,
+    "end_conditions": {
+      "reconciliation_coverage": {
+        "coverage": 1,
+        "from_tip": true
+      }
+    }
+  }
+}

systemtests/mesh_cli_config/devnet-construction.json

@@ -0,0 +1,35 @@
+{
+  "network": {
+    "blockchain": "MultiversX",
+    "network": "untitled"
+  },
+  "data_directory": "",
+  "http_timeout": 200,
+  "max_retries": 1,
+  "max_online_connections": 120,
+  "max_sync_concurrency": 1,
+  "construction": {
+    "end_conditions": {
+      "transfer": 1
+    },
+    "stale_depth": 10,
+    "broadcast_limit": 5,
+    "constructor_dsl_file": "devnet-construction.ros",
+    "prefunded_accounts": [
+      {
+        "account_identifier": {
+          "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq"
+        },
+        "privkey": "3e4e89e501eb542c12403fb15c52479e8721f2f4dedc3b3ef0f3b47b37de006c",
+        "curve_type": "edwards25519",
+        "currency": {
+          "symbol": "EGLD",
+          "decimals": 18
+        }
+      }
+    ]
+  },
+  "data": {
+    "inactive_discrepancy_search_disabled": true
+  }
+}

systemtests/mesh_cli_config/devnet-construction.ros

@@ -0,0 +1,54 @@
+transfer(1){
+  transfer{
+    transfer.network = {"network":"untitled", "blockchain":"MultiversX"};
+    native_currency = {"symbol":"EGLD", "decimals":18};
+    sender = {
+      "account_identifier": {
+        "address": "erd1ldjsdetjvegjdnda0qw2h62kq6rpvrklkc5pw9zxm0nwulfhtyqqtyc4vq"
+      },
+      "currency": {
+          "symbol": "EGLD",
+          "decimals": 18
+      }
+    };
+
+    max_fee = "50000000000000";
+    max_transfer_amount = "10000000000000000";
+    recipient_amount = random_number({"minimum": "1", "maximum": {{max_transfer_amount}}});
+
+    print_message({"recipient_amount":{{recipient_amount}}});
+
+    sender_amount = 0-{{recipient_amount}};
+    recipient = {
+      "account_identifier": {
+        "address": "erd1xtslmt67utuewwv8jsx729mxjxaa8dvyyzp7492hy99dl7hvcuqq30l98v"
+      },
+      "currency": {
+          "symbol": "EGLD",
+          "decimals": 18
+      }
+    };
+    transfer.confirmation_depth = "10";
+    transfer.operations = [
+      {
+        "operation_identifier":{"index":0},
+        "type":"Transfer",
+        "account":{{sender.account_identifier}},
+        "amount":{
+          "value":{{sender_amount}},
+          "currency":{{native_currency}}
+        }
+      },
+      {
+        "operation_identifier":{"index":1},
+        "related_operations": [{"index": 0}],
+        "type":"Transfer",
+        "account":{{recipient.account_identifier}},
+        "amount":{
+          "value":{{recipient_amount}},
+          "currency":{{native_currency}}
+        }
+      }
+    ];
+  }
+}