Merge pull request #15370 from MinaProtocol/fix/unit-tests-ci-DEVELOP

Merge berkeley to develop + #15360

buildkite/scripts/build-hardfork-package.sh

@@ -36,7 +36,7 @@ echo "--- Migrate accounts to new network format"
 #       For now, this is hard-coded to the mainnet -> berkeley migration, but we need to select
 #       a migration to perform in the future.
 # NB: we use sed here instead of jq, because jq is extremely slow at processing this file
-sed -i -e 's/"set_verification_key": "signature"/"set_verification_key": {"auth": "signature", "txn_version": "1"}/' config.json
+sed -i -e 's/"set_verification_key": "signature"/"set_verification_key": {"auth": "signature", "txn_version": "2"}/' config.json
 
 case "${NETWORK_NAME}" in
   mainnet)

dockerfiles/Dockerfile-mina-test-executive

@@ -60,7 +60,7 @@ RUN curl https://baltocdn.com/helm/signing.asc | sudo apt-key add - \
     && sudo apt-get install -y helm
 
 # Get yarn + nodejs
-RUN curl -sL https://deb.nodesource.com/setup_16.x | sudo -E bash - \
+RUN curl -sL https://deb.nodesource.com/setup_20.x | sudo -E bash - \
     && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - \
     && echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list \
     && sudo apt-get update \

dockerfiles/stages/3-toolchain

@@ -102,7 +102,7 @@ RUN curl https://baltocdn.com/helm/signing.asc | apt-key add - \
     && rm -rf /var/lib/apt/lists/*
 
 # --- yarn + nodejs, pinned version
-RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - \
+RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - \
     && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - \
     && echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list \
     && apt update --yes \

scripts/archive/check_db_genesis_balances.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+from dataclasses import dataclass
+from typing import Optional, TypedDict
+import psycopg2
+import json
+import os
+import argparse
+
+name = os.environ.get("DB_NAME", "archive")
+user = os.environ.get("DB_USER", "mina")
+password = os.environ.get("DB_PASSWORD")
+host = os.environ.get("DB_HOST", "localhost")
+port = os.environ.get("DB_PORT", "5432")
+config_file = os.environ.get("CONFIG_FILE")
+
+
+parser = argparse.ArgumentParser(
+    description="Check that the genesis balances in the JSON file match the database."
+)
+parser.add_argument(
+    "--config-file",
+    required=config_file is None,
+    default=config_file,
+    help="Path to the node runtime configuration JSON file containing the ledger.",
+)
+parser.add_argument(
+    "--name",
+    default=name,
+    help="Name of the database to connect to.",
+)
+parser.add_argument(
+    "--user",
+    default=user,
+    help="Name of the user to connect to the database.",
+)
+parser.add_argument(
+    "--password",
+    required=password is None,
+    default=password,
+    help="Password to connect to the database.",
+)
+parser.add_argument(
+    "--host",
+    default=host,
+    help="Host of the database.",
+)
+parser.add_argument("--port", default=port, help="Port of the database.")
+
+args = parser.parse_args()
+
+
+class TimingInfo(TypedDict):
+    initial_minimum_balance: int
+    cliff_time: int
+    cliff_amount: int
+    vesting_period: int
+    vesting_increment: int
+
+
+class LedgerAccount(TypedDict):
+    pk: str
+    balance: int
+    timing: Optional[TimingInfo]
+    delegate: Optional[str]
+    nonce: Optional[int]
+    receipt_chain_hash: Optional[str]
+
+
+count_query = "SELECT COUNT(*) FROM accounts_accessed WHERE block_id = 1;"
+
+query = """
+SELECT pk.value, ac.balance, ti.initial_minimum_balance, ti.cliff_time, ti.cliff_amount,
+       ti.vesting_period, ti.vesting_increment, pk_delegate.value, ac.nonce,
+       ac.receipt_chain_hash
+FROM accounts_accessed ac
+INNER JOIN account_identifiers ai ON ac.account_identifier_id = ai.id
+INNER JOIN public_keys pk ON ai.public_key_id = pk.id
+LEFT JOIN timing_info ti ON ac.timing_id = ti.id AND ai.id = ti.account_identifier_id
+LEFT JOIN public_keys pk_delegate ON ac.delegate_id = pk_delegate.id
+WHERE ac.block_id = 1 AND pk.value = %s
+LIMIT 1;
+"""
+
+
+def normalize_balance(balance) -> int:
+    # split account["balance"] by decimal point
+    balance_str = str(balance)
+    split_balance = balance_str.split(".")
+    if len(split_balance) == 1:
+        balance_str = split_balance[0] + "000000000"
+    elif len(split_balance) == 2:
+        balance_str = split_balance[0] + split_balance[1].ljust(9, "0")
+    return int(balance_str)
+
+
+def row_to_ledger_account(row) -> LedgerAccount:
+    initial_minimum_balance = int(row[2])
+    cliff_time = int(row[3])
+    cliff_amount = int(row[4])
+    vesting_period = int(row[5])
+    vesting_increment = int(row[6])
+    return {
+        "pk": row[0],
+        "balance": int(row[1]),
+        "timing": (
+            TimingInfo(
+                initial_minimum_balance=initial_minimum_balance,
+                cliff_time=cliff_time,
+                cliff_amount=cliff_amount,
+                vesting_period=vesting_period,
+                vesting_increment=vesting_increment,
+            )
+            if initial_minimum_balance != 0
+            or cliff_time != 0
+            or cliff_amount != 0
+            or vesting_period != 0
+            or vesting_increment != 0
+            else None
+        ),
+        "delegate": row[7] if row[7] else None,
+        "nonce": int(row[8]) if row[8] != "0" else None,
+        "receipt_chain_hash": row[9] if row[9] else None,
+    }
+
+
+def json_account_to_ledger_account(account) -> LedgerAccount:
+    return {
+        "pk": account["pk"],
+        "balance": normalize_balance(account["balance"]),
+        "timing": (
+            {
+                "initial_minimum_balance": normalize_balance(
+                    account["timing"]["initial_minimum_balance"]
+                ),
+                "cliff_time": int(account["timing"]["cliff_time"]),
+                "cliff_amount": normalize_balance(account["timing"]["cliff_amount"]),
+                "vesting_period": int(account["timing"]["vesting_period"]),
+                "vesting_increment": normalize_balance(
+                    account["timing"]["vesting_increment"]
+                ),
+            }
+            if "timing" in account
+            else None
+        ),
+        "delegate": account["delegate"] if "delegate" in account else None,
+        "nonce": account["nonce"] if "nonce" in account else None,
+        "receipt_chain_hash": (
+            account["receipt_chain_hash"] if "receipt_chain_hash" in account else None
+        ),
+    }
+
+
+def accounts_match(account_json, account_sql) -> bool:
+    account_json = json_account_to_ledger_account(account_json)
+    account_sql = row_to_ledger_account(account_sql)
+    messages = []
+    if account_json["pk"] != account_sql["pk"]:
+        messages.append(f"pk: {account_json['pk']} != {account_sql['pk']}")
+
+    if account_json["balance"] != account_sql["balance"]:
+        messages.append(
+            f"balance: {account_json['balance']} != {account_sql['balance']}"
+        )
+
+    if account_json["timing"] != account_sql["timing"]:
+        messages.append(
+            f"timing:\n{account_json['timing']} !=\n{account_sql['timing']}"
+        )
+
+    if account_json["delegate"] != account_sql["delegate"]:
+        messages.append(
+            f"delegate: {account_json['delegate']} != {account_sql['delegate']}"
+        )
+        if not (
+            account_json["nonce"] == account_sql["nonce"]
+            if account_json["nonce"]
+            else account_sql["nonce"] == 0
+        ):
+            messages.append(f"nonce: {account_json['nonce']} != {account_sql['nonce']}")
+
+    if len(messages) != 0:
+        print(f"Account with pk '{account['pk']}' does not match the SQL query result.")
+        for message in messages:
+            print(f"\n{message}")
+        return False
+    else:
+        return True
+
+
+with open(args.config_file) as json_file:
+    ledger = json.load(json_file)
+
+ledger_accounts = ledger["ledger"]["accounts"]
+
+with psycopg2.connect(
+    dbname=args.name,
+    user=args.user,
+    password=args.password,
+    host=args.host,
+    port=args.port,
+) as conn:
+    with conn.cursor() as cur:
+        cur.execute(count_query)
+        result = cur.fetchone()
+        count = result[0] if result else 0
+        assert count == len(
+            ledger_accounts
+        ), f"Number of accounts in the JSON file ({len(ledger_accounts)}) does not match the number of accounts in the SQL query ({count})."
+        print(f"Number of accounts in the JSON file and SQL query match ({count}).")
+
+    print("Checking that the genesis balances in the JSON file match the database...")
+    all_accounts_match = True
+    for acc in ledger_accounts:
+        with conn.cursor() as cur:
+            cur.execute(query, (acc["pk"],))
+            result = cur.fetchone()
+            all_accounts_match = all_accounts_match and accounts_match(acc, result)
+
+    assert all_accounts_match, "Some accounts do not match the SQL query result."
+    print("All accounts match the SQL query result.")

scripts/archive/migration/README.md

@@ -0,0 +1,137 @@
+## Berkeley Migration script
+
+Migration Script which simplify migration process by wrapping all 3 phases and 2 steps per each phase into single script. We try to make it as verbose as possible and give user a hints why some parameter is needed and how to obtain it
+
+### Usage
+
+Script has 3 subcommands
+
+- initial - for running first phase of migration where migrated db is an empty schema
+- incremental - for running second phase of migration where incrementally we migrated current mainnet
+- final - for migrating data till fork point
+
+### Dependencies 
+
+Script is very verbose and inform when any of below dependency is missing. For documentation purposes we list them here:
+
+#### General
+
+- jq 
+- wget
+- postgres and psql
+- gsutil
+
+#### Mina related 
+
+All mina related apps can be either downloaded in debian or docker. Advanced users can build them locally, but please remember to rename them and use `DUNE_PROFILE=berkeley_archive_migration` env variable when building berkeley_migration
+
+- mina-berkeley-migration
+- mina-berkeley-migration-verifier
+- mina-migration-replayer
+
+
+### Testing 
+
+The best approach to learn about the tool is to start testing it. Below we present end to end testing cycle for migration script based on umt data
+
+#### Data preparation
+
+Test data is available in one of o1labs bucket : `gs://umt-migration-historical-data`.
+Since berkeley migration script supports all berkeley migration phases. We need to have 3 dumps which represents all 3 stages. Beside that we need a ledger file and precomputed blocks (which are stored in different bucket). Below steps required before testing script:
+
+1. Download and import initial dump:
+```
+    gsutil cp gs://umt-migration-historical-data/o1labs-umt-pre-fork-run-1-archive-dump-2024-02-26_0000.sql.tar.gz .
+
+    tar -xzf o1labs-umt-pre-fork-run-1-archive-dump-2024-02-26_0000.sql.tar.gz o1labs-umt-pre-fork-run-1-archive-dump-2024-02-26_0000.sql
+
+    psql -U postgres -c "create database umt_testing_initial"
+    sed -i 's/archive/umt_testing_initial/g' o1labs-umt-pre-fork-run-1-archive-dump-2024-02-26_0000.sql
+
+    psql -U postgres -d umt_testing_initial < o1labs-umt-pre-fork-run-1-archive-dump-2024-02-26_0000.sql
+```
+
+
+2. Download and import incremental dump:
+```
+    gsutil cp gs://umt-migration-historical-data/o1labs-umt-pre-fork-run-1-archive-dump-2024-02-29_1200.sql.tar.gz .
+
+    tar -xzf o1labs-umt-pre-fork-run-1-archive-dump-2024-02-29_1200.sql.tar.gz o1labs-umt-pre-fork-run-1-archive-dump-2024-02-29_1200.sql
+
+    psql -U postgres  -c "create database umt_testing_increment"
+    sed -i 's/archive/umt_testing_increment/g' o1labs-umt-pre-fork-run-1-archive-dump-2024-02-29_1200.sql
+
+    psql -U postgres -d umt_testing_increment < o1labs-umt-pre-fork-run-1-archive-dump-2024-02-29_1200.sql
+```
+
+
+3. Download and import final dump:
+```
+    gsutil cp gs://umt-migration-historical-data/o1labs-umt-pre-fork-run-1-archive-dump-2024-02-29_1516.sql.tar.gz .
+
+    tar -xzf o1labs-umt-pre-fork-run-1-archive-dump-2024-02-29_1516.sql.tar.gz o1labs-umt-pre-fork-run-1-archive-dump-2024-02-29_1516.sql
+
+    psql -U postgres  -c "create database umt_testing_final"
+    sed -i 's/archive/umt_testing_final/g' o1labs-umt-pre-fork-run-1-archive-dump-2024-02-29_1516.sql
+
+    psql -U postgres -d umt_testing_final < o1labs-umt-pre-fork-run-1-archive-dump-2024-02-29_1516.sql
+```
+
+
+4. Create an empty berkeley dump:
+```
+    cd src/app/archive
+
+    psql -U postgres -c "create database migrated"
+
+    psql -U postgres -d "migrated" < create_schema.sql
+
+```
+
+5. Download Genesis ledger file
+```
+    gsutil cp gs://umt-migration-historical-data/o1labs-umt-pre-fork-run-1-ledger.json .
+```
+
+6. Download fork config file
+```
+    gsutil cp gs://umt-migration-historical-data/fork-umt-02-29-2024.json .
+```
+
+#### Initial migration
+
+```
+./scripts/archive/migration/berkeley_migration.sh initial -g  ../../umt_testing/o1labs-umt-pre-fork-run-1-ledger.json -s postgres://postgres:postgres@localhost:5432/umt_testing_initial -t postgres://postgres:postgres@localhost:5432/migrated -b mina_network_block_data -bs 1000 -n o1labs-umt-pre-fork-run-1
+```
+
+this command should output migration-replayer-XXX.json which should be used in next run
+
+#### Incremental migration
+
+```
+./scripts/archive/migration/berkeley_migration.sh incremental -g  ../../umt_testing/o1labs-umt-pre-fork-run-1-ledger.json -s postgres://postgres:postgres@localhost:5432/umt_testing_increment -t postgres://postgres:postgres@localhost:5432/migrated -b mina_network_block_data -bs 50 -n o1labs-umt-pre-fork-run-1 -r migration-checkpoint-597.json
+```
+
+where:
+
+migration-checkpoint-597.json - is a last checkpoint from initial run
+
+#### Final migration
+
+```
+./scripts/archive/migration/berkeley_migration.sh final -g  ../../umt_testing/o1labs-umt-pre-fork-run-1-ledger.json -s postgres://postgres:postgres@localhost:5432/umt_testing_final -t postgres://postgres:postgres@localhost:5432/migrated -b mina_network_block_data -bs 50 -n o1labs-umt-pre-fork-run-1 -r migration-checkpoint-2381.json -fc ../../umt_testing/fork-umt-02-29-2024.json -f 3NLnD1Yp4MS9LtMXikD1YyySZNVgCXA82b5eQVpmYZ5kyTo4Xsr7
+```
+
+where `3NLnD1Yp4MS9LtMXikD1YyySZNVgCXA82b5eQVpmYZ5kyTo4Xsr7` was extracted from fork config file:
+
+```
+{
+  "proof": {
+    "fork": {
+      "state_hash": "3NLnD1Yp4MS9LtMXikD1YyySZNVgCXA82b5eQVpmYZ5kyTo4Xsr7",
+      "blockchain_length": 1276,
+      "global_slot_since_genesis": 2856
+    }
+  },
+```
+and `migration-checkpoint-2381.json` was last checkpoint from incremental run