I93 early epoch end tests & improvements on the test architecture

.vscode/launch.json

@@ -21,6 +21,14 @@
       "runtimeExecutable": "npm",
       "runtimeArgs": ["run-script", "test-hbbft-contract"]
     },
+    {
+      "name": "testnet-fork-test-create",
+      "type": "node",
+      "request": "launch",
+      "cwd": "${workspaceFolder}",
+      "runtimeExecutable": "npm",
+      "runtimeArgs": ["run-script", "testnet-fork-test-create"]
+    },
     {
       "name": "localnet-start-phonix-test",
       "type": "node",

config/default.json

@@ -1,5 +1,5 @@
 {
-  "network": "alpha2",
+  "network": "nodes-local-mainnet-fork",
   "networkGitRepo" : "https://github.com/DMDcoin/dmdv4-testnet",
   "networkGitRepoBranch" : "main", 
   "openEthereumProfile" : "release",
@@ -19,7 +19,15 @@
       "name": "local",
       "installDir": "",
       "nodesDir": "nodes",
-      "rpc": "http://127.0.0.1:8541",
+      "rpc": "http://127.0.0.1:8540",
+      "blockscout": "http://127.0.0.1:8540",
+      "db" : "127.0.0.1:5432"
+    },
+    {
+      "name": "local2",
+      "installDir": "",
+      "nodesDir": "nodes-local2",
+      "rpc": "http://127.0.0.1:8540",
       "blockscout": "http://127.0.0.1:8540",
       "db" : "127.0.0.1:5432"
     },
@@ -33,7 +41,7 @@
       "nodesDir": "nodes-alpha2",
       "installDir": "alpha2",
       "screenName": "alpha2",
-      "openEthereumBranch": "main",
+      "openEthereumBranch": "dforsten/hbbft_no_failure",
       "openEthereumDeadlockDetection": false
     },
     {
@@ -56,6 +64,34 @@
       "blockscout": "http://38.242.206.145:4000/",
       "db" : "127.0.0.1:5432",
       "installDir": "old-testnet"
+    },
+    {
+      "name": "nodes-local-fork-test",
+      "rpc": "http://localhost:8540",
+      "blockscout": "http://localhost:4000/",
+      "db" : "localhost:5432",
+      "installDir": "nodes-local-fork-test",
+      "nodesDir": "nodes-local-fork-test"
+    },
+    {
+      "name": "nodes-local-test-early-epoch-end",
+      "rpc": "http://localhost:8540",
+      "blockscout": "http://localhost:4000/",
+      "db" : "localhost:5432",
+      "installDir": "nodes-local-test-early-epoch-end",
+      "nodesDir": "nodes-local-test-early-epoch-end",
+      "builder": {
+        "initialValidatorsCount": 16,
+        "nodesCount": 16
+      }
+    },
+    {
+      "name": "nodes-local-mainnet-fork",
+      "rpc": "https://rpc.uniq.diamonds",
+      "blockscout": "http://localhost:4000/",
+      "db" : "localhost:5432",
+      "installDir": "nodes-local-mainnet-fork",
+      "nodesDir": "nodes-local-mainnet-fork"
     }
   ]
 }

db/migrations/0006-header_import-complete.sql

@@ -0,0 +1,7 @@
+DO $$
+    BEGIN
+        ALTER TABLE public.headers ADD COLUMN import_complete boolean;
+    EXCEPTION
+        WHEN duplicate_column THEN RAISE NOTICE 'column import_complete already exists in public.headers, skipping';
+    END
+$$;

package.json

@@ -69,7 +69,7 @@
     "net-unstake": "ts-node src/net/runUnstake.ts",
     "net-contracts-upgrade": "ts-node src/contracts/runUpgradeContracts.ts",
     "net-contracts-test": "ts-node src/contracts/verifySourceCode.ts",
-    "net-print-contract-details": "ts-node src/contracts/printContractDetails.ts",
+    "net-contracts-print-details": "ts-node src/contracts/printContractDetails.ts",
     "net-stake-on-validators": "ts-node src/net/runStakeOnValidators.ts",
     "net-set-registry-fee": "ts-node src/control/runSetRegistryFee.ts",
     "net-show-block-details": "ts-node src/net/showBlockDetails.ts",
@@ -90,32 +90,51 @@
     "net-print-stake-events": "ts-node src/net/runPrintStakeEvents.ts",
     "net-watchdog": "ts-node src/runWatchdog.ts",
     "net-claiming-pot-dilution": "ts-node src/net/runClaimingPotDilution.ts",
+    "net-transfer-abandoned-coins": "ts-node src/net/runTransferAbandonedCoins.ts",
+    "db-fresh": "npm-run-all db-delete-local --parallel db-start-local db-apply-migrations ",
     "db-delete-local": "cd db && docker compose down",
     "db-start-local": "cd db && envsubst < grafana/templates/postgres.yaml > grafana/provisioning/datasources/postgres.yaml && docker compose up",
     "db-grant-grafana-permissions": "ts-node src/db/runGrantGrafanaPermissions.ts",
     "db-update-ts-schema": "npx @databases/pg-schema-cli --database postgres://postgres:[email protected]:5432/postgres --directory src/db/schema",
     "db-fill-from-network": "ts-node src/db/runFillDb.ts",
     "db-delete-tables": "ts-node src/db/delete-tables.ts",
-    "db-apply-migrations": "npx pg-migrations apply -c postgres://postgres:[email protected]:5432/postgres -D db/migrations",
+    "db-apply-migrations": "sleep 3 && npx pg-migrations apply -c postgres://postgres:[email protected]:5432/postgres -D db/migrations",
     "localnet-blockscout-fresh": "cd testnet/blockscout-local && docker compose down && docker compose up",
+    "localnet-fresh-new": "export STAKING_TRANSITION_WINDOW_LENGTH=120 && VALIDATOR_INACTIVITY_THRESHOLD=307584000 && export STAKING_EPOCH_DURATION=432 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=240 && ts-node ./src/localnet/runCreateTestnet.ts",
+    "localnet-fresh-test-early-epoch-end": "export STAKING_TRANSITION_WINDOW_LENGTH=240 && VALIDATOR_INACTIVITY_THRESHOLD=1209600 && export STAKING_EPOCH_DURATION=60000 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=30 && ts-node ./src/localnet/runCreateTestnet.ts --network=nodes-local-test-early-epoch-end",
     "localnet-create-mnemonic": "ts-node src/localnet/create-random-mnemonic.ts",
-    "localnet-start-phonix-test": "ts-node src/localnet/runPhoenixTestNetwork.ts",
+    "localnet-start-phonix-test": "ts-node src/localnet/tests/runPhoenixTestNetwork.ts",
+    "localnet-test-early-epoch-end-test": "ts-node src/localnet/tests/runEarlyEpochEndTestNetwork.ts",
+    "localnet-test-phoenix": "ts-node src/localnet/tests/runPhoenixTestNetwork.ts",
+    "localnet-start-current": "ts-node src/runWatchdog.ts --boot",
+    "testnet-txPermission": "ts-node src/localnet/tests/verifyTXPermission.ts",
+    "testnet-fork-test-create": "ts-node src/fork/runForkedNetworkBuilder.ts --network nodes-local-fork-test",
+    "testnet-fork-mainnet-create": "ts-node src/fork/runForkedNetworkBuilderNet.ts --network nodes-local-mainnet-fork",
     "testnet-start-current": "ts-node src/runWatchdog.ts --boot",
+    "testnet-start-fork-test": "ts-node src/runWatchdog.ts --boot --network=nodes-local-fork-test",
     "testnet-state-analysis": "ts-node testnet/control/runGetCurrentTestnetState.ts",
+    "testnet-delete": "rm -r testnet/nodes || true",
+    "testnet-delete-fork-test": "rm -r testnet/nodes-local-fork-test || true",
+    "testnet-kill-processes": "pkill diamond-node || true",
+    "testnet-testrun-early-epoch-end-test": "npm-run-all testnet-kill-processes build-open-ethereum testnet-delete  localnet-fresh-test-early-epoch-end localnet-test-early-epoch-end-test",
+    "testnet-testrun-phoenix": "npm-run-all build-open-ethereum testnet-delete testnet-kill-processes testnet-fresh-test-phoenix localnet-test-phoenix localnet-start-current",
+    "testnet-testrun-fork-test": "npm-run-all build-open-ethereum testnet-kill-processes testnet-delete-fork-test testnet-fork-test-create",
     "testnet-clear-validator-dbs": "ts-node src/localnet/runClearValidatorsDB.ts",
     "testnet-restore-validator-dbs": "ts-node src/localnet/runRestoreValidatorsDB.ts",
     "testnet-backup-validator-dbs": "ts-node src/localnet/runBackupValidatorDB.ts",
     "testnet-run-fresh-grow-each-epoch": "npm-run-all testnet-fresh net-add-validator-each-epoch-booted",
-    "testnet-fresh": "export STAKING_TRANSITION_WINDOW_LENGTH=120 && export STAKING_EPOCH_DURATION=432 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=240 && cd testnet && ./setup_testnet.py 1 27 && cd -",
-    "testnet-fresh-alpha2": "export STAKING_TRANSITION_WINDOW_LENGTH=60 && export STAKING_EPOCH_DURATION=720 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=60 && cd testnet && ./setup_testnet.py 1 27 && cd -",
-    "testnet-fresh-duo": "export STAKING_TRANSITION_WINDOW_LENGTH=60 && export STAKING_EPOCH_DURATION=120 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=30 && cd testnet && ./setup_testnet.py 2 2 && cd -",
-    "testnet-fresh-single-for-unit-tests": "export STAKING_TRANSITION_WINDOW_LENGTH=120 && export STAKING_EPOCH_DURATION=432 && export MINIMUM_BLOCK_TIME=0 && export MAXIMUM_BLOCK_TIME=600 && cd testnet && ./setup_testnet.py 1 1 && cd -",
-    "testnet-fresh-regression": "export STAKING_TRANSITION_WINDOW_LENGTH=60 && export STAKING_EPOCH_DURATION=120 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=30 && cd testnet && ./setup_testnet.py 6 6 && cd -",
-    "testnet-fresh-grow-me": "export STAKING_TRANSITION_WINDOW_LENGTH=120 && export STAKING_EPOCH_DURATION=600 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=30 && cd testnet && ./setup_testnet.py 1 6 && cd -",
-    "testnet-fresh-no-epoch-switches": "export STAKING_TRANSITION_WINDOW_LENGTH=60 && export STAKING_EPOCH_DURATION=2629800 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=60 && cd testnet && ./setup_testnet.py 4 4 && cd -",
-    "testnet-fresh-12hour": "export STAKING_TRANSITION_WINDOW_LENGTH=600 && export STAKING_EPOCH_DURATION=43200 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=600 && cd testnet && ./setup_testnet.py 25 25 && cd -",
+    "testnet-fresh": "export STAKING_TRANSITION_WINDOW_LENGTH=120 && VALIDATOR_INACTIVITY_THRESHOLD=307584000 && export STAKING_EPOCH_DURATION=432 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=240 && cd testnet && ./setup_testnet.py 1 27 && cd -",
+    "testnet-fresh-upgrade": "export STAKING_TRANSITION_WINDOW_LENGTH=60 && VALIDATOR_INACTIVITY_THRESHOLD=1209600 && export STAKING_EPOCH_DURATION=1440 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=60 && cd testnet && ./setup_testnet.py 1 27 && cd -",
+    "testnet-fresh-alpha2": "export STAKING_TRANSITION_WINDOW_LENGTH=60 && VALIDATOR_INACTIVITY_THRESHOLD=307584000 && export STAKING_EPOCH_DURATION=1200 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=600 && cd testnet && ./setup_testnet.py 1 27 && cd -",
+    "testnet-fresh-duo": "export STAKING_TRANSITION_WINDOW_LENGTH=60 && VALIDATOR_INACTIVITY_THRESHOLD=1209600 && export STAKING_EPOCH_DURATION=120 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=30 && cd testnet && ./setup_testnet.py 2 2 && cd -",
+    "testnet-fresh-single-for-unit-tests": "export STAKING_TRANSITION_WINDOW_LENGTH=120 &&  VALIDATOR_INACTIVITY_THRESHOLD=1209600 && export STAKING_EPOCH_DURATION=432 && export MINIMUM_BLOCK_TIME=0 && export MAXIMUM_BLOCK_TIME=600 && cd testnet && ./setup_testnet.py 1 1 && cd -",
+    "testnet-fresh-regression": "export STAKING_TRANSITION_WINDOW_LENGTH=60 &&  VALIDATOR_INACTIVITY_THRESHOLD=1209600 && export STAKING_EPOCH_DURATION=120 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=30 && cd testnet && ./setup_testnet.py 6 6 && cd -",
+    "testnet-fresh-test-phoenix": "export STAKING_TRANSITION_WINDOW_LENGTH=240 && VALIDATOR_INACTIVITY_THRESHOLD=1209600 && export STAKING_EPOCH_DURATION=60000 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=2 && cd testnet && ./setup_testnet.py 4 4 && cd -",
+    "testnet-fresh-grow-me": "export STAKING_TRANSITION_WINDOW_LENGTH=120 &&  VALIDATOR_INACTIVITY_THRESHOLD=1209600 && export STAKING_EPOCH_DURATION=600 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=30 && cd testnet && ./setup_testnet.py 1 6 && cd -",
+    "testnet-fresh-no-epoch-switches": "export STAKING_TRANSITION_WINDOW_LENGTH=60 && VALIDATOR_INACTIVITY_THRESHOLD=1209600 && export STAKING_EPOCH_DURATION=2629800 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=60 && cd testnet && ./setup_testnet.py 4 4 && cd -",
+    "testnet-fresh-12hour": "export STAKING_TRANSITION_WINDOW_LENGTH=600 &&  VALIDATOR_INACTIVITY_THRESHOLD=1209600 && export STAKING_EPOCH_DURATION=43200 && export MINIMUM_BLOCK_TIME=1 && export MAXIMUM_BLOCK_TIME=600 && cd testnet && ./setup_testnet.py 25 25 && cd -",
     "testnet-test-availability-handling-current": "ts-node src/regression/testAvailabilityHandling.ts",
-    "testnet-test-availability-handling-fresh": "export STAKING_TRANSITION_WINDOW_LENGTH=90 && export STAKING_EPOCH_DURATION=120 && export MINIMUM_BLOCK_TIME=1 && export NETWORK_ID=666001 && cd testnet && ./setup_testnet.py 1 17 && cd - && export NODE_ENV=localhost && npm run testnet-test-availability-handling-current",
+    "testnet-test-availability-handling-fresh": "export STAKING_TRANSITION_WINDOW_LENGTH=90 &&  VALIDATOR_INACTIVITY_THRESHOLD=1209600 && export STAKING_EPOCH_DURATION=120 && export MINIMUM_BLOCK_TIME=1 && export NETWORK_ID=666001 && cd testnet && ./setup_testnet.py 1 17 && cd - && export NODE_ENV=localhost && npm run testnet-test-availability-handling-current",
     "testnet-add-validator-each-epoch": "export NODE_ENV=localhost && npm run net-add-validator-each-epoch",
     "testnet-show-node-table": "ts-node src/remotenet/runCreateReservedPeersFiles.ts",
     "remotenet-create-peers-files": "ts-node src/remotenet/runCreateReservedPeersFiles.ts",

src/analysis/analyseBlockMessages.ts

@@ -7,13 +7,8 @@ import path from "path";
 
 export class AnalyseReport {
 
-
   public results = new Map<string, NodeResult>();
-
   public proposalsPerValidator = new Map<string, number>();
-  // public reportMessage(proposer: string, ) {
-
-  // }
 
   private getNodeResult(node: string): NodeResult {