ci: prebuild and share a single neard binary between different jobs (#…

…10105)

This should hopefully free up the runners for other jobs, reduce CPU
time these tests take (thus saving money) and test latency (thus saving
even more money!)

.github/workflows/ci.yml

@@ -9,25 +9,23 @@ on:
   merge_group:
 
 jobs:
-  backward_compat:
-    name: "Backward Compatibility"
+  build_binary:
+    name: "Build neard"
     runs-on: ubuntu-22.04-16core
-    defaults:
-      run:
-        working-directory: ./pytest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.11
-          cache: pip
       - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
           prefix-key: "0" # change this to invalidate CI cache
+          shared-key: "cargo_nextest"
           save-if: "false" # use the cache from nextest, but don’t double-save
-          # TODO: the python script should use the quick-release cargo profile, but currently does not
-      - run: pip3 install --user -r requirements.txt
-      - run: python3 tests/sanity/backward_compatible.py
+      - run: cargo build --locked --profile quick-release -p neard --bin neard
+      - uses: actions/upload-artifact@v3
+        with:
+          name: neard
+          path: target/quick-release/neard
+          if-no-files-found: error
+          retention-days: 1
 
   cargo_nextest:
     name: "Cargo Nextest (${{matrix.name}})"
@@ -48,7 +46,6 @@ jobs:
             # them at earliest convenience :)
             flags: "--exclude integration-tests --exclude node-runtime --exclude runtime-params-estimator --exclude near-network --exclude estimator-warehouse"
     timeout-minutes: 90
-
     steps:
       # Some of the tests allocate really sparse maps, so heuristic-based overcommit limits are not
       # appropriate here.
@@ -68,9 +65,20 @@ jobs:
         env:
           RUST_BACKTRACE: short
 
-  db_migration:
-    name: "Database Migration"
-    runs-on: ubuntu-22.04-16core
+  protobuf_backward_compat:
+    name: "Protobuf Backward Compatibility"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: bufbuild/buf-setup-action@1158f4fa81bc02e1ff62abcca6d516c9e24c77da
+      - uses: bufbuild/buf-breaking-action@a074e988ee34efcd4927079e79c611f428354c01
+        with:
+          against: "https://github.com/near/nearcore.git#${{github.event.pull_request.base.sha && format('ref={0}', github.event.pull_request.base.sha) || 'branch=master' }}"
+
+  py_backward_compat:
+    name: "Backward Compatibility"
+    needs: build_binary
+    runs-on: ubuntu-22.04
     defaults:
       run:
         working-directory: ./pytest
@@ -80,31 +88,44 @@ jobs:
         with:
           python-version: 3.11
           cache: pip
-      - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
+      - uses: actions/download-artifact@v3
         with:
-          prefix-key: "0" # change this to invalidate CI cache
-          save-if: "false" # use the cache from nextest, but don’t double-save
+          name: neard
+          path: pytest # NB: this does not account for defaults.run.working-directory
+      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
+      - run: chmod +x "$CURRENT_NEARD"
       - run: pip3 install --user -r requirements.txt
-      - run: python3 tests/sanity/db_migration.py
-
-  protobuf_backward_compat:
-    name: "Protobuf Backward Compatibility"
-    runs-on: ubuntu-latest
+      - run: python3 tests/sanity/backward_compatible.py
 
+  py_db_migration:
+    name: "Database Migration"
+    needs: build_binary
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: ./pytest
     steps:
       - uses: actions/checkout@v4
-      - uses: bufbuild/buf-setup-action@1158f4fa81bc02e1ff62abcca6d516c9e24c77da
-      - uses: bufbuild/buf-breaking-action@a074e988ee34efcd4927079e79c611f428354c01
+      - uses: actions/setup-python@v4
         with:
-          against: "https://github.com/near/nearcore.git#${{github.event.pull_request.base.sha && format('ref={0}', github.event.pull_request.base.sha) || 'branch=master' }}"
+          python-version: 3.11
+          cache: pip
+      - uses: actions/download-artifact@v3
+        with:
+          name: neard
+          path: pytest # NB: this does not account for defaults.run.working-directory
+      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
+      - run: echo "NEAR_ROOT=$PWD" >> "$GITHUB_ENV"
+      - run: chmod +x "$CURRENT_NEARD"
+      - run: pip3 install --user -r requirements.txt
+      - run: python3 tests/sanity/db_migration.py
 
-  sanity_checks:
+  py_sanity_checks:
     name: "Sanity Checks"
     runs-on: ubuntu-22.04-16core
     strategy:
       fail-fast: false
     timeout-minutes: 90
-
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
@@ -114,44 +135,57 @@ jobs:
       - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
           prefix-key: "0" # change this to invalidate CI cache
+          shared-key: "cargo_nextest"
           save-if: "false" # use the cache from nextest, but don’t double-save
-      - run: cargo build -p neard --bin neard --features nightly
-      - name: run spin_up_cluster.py
-        # Note: We're not running spin_up_cluster.py for non-nightly
-        # because spinning up non-nightly clusters is already covered
-        # by other steps in the CI, e.g. upgradable.
-        run: |
-          cd pytest
-          python3 -m pip install --progress-bar off --user -r requirements.txt
-          python3 tests/sanity/spin_up_cluster.py
+      - run: pip3 install --user -r pytest/requirements.txt
+      # This is the only job that uses `--features nightly` so we build this in-line instead of a
+      # separate job like done with the regular neard.
+      - run: cargo build --profile quick-release -p neard --bin neard --features nightly
+      # Note: We're not running spin_up_cluster.py for non-nightly
+      # because spinning up non-nightly clusters is already covered
+      # by other steps in the CI, e.g. upgradable.
+      - run: python3 pytest/tests/sanity/spin_up_cluster.py
+        env:
+          NEAR_ROOT: "target/quick-release"
 
-      - run: cargo build --profile quick-release -p neard --bin neard
+  py_genesis_check:
+    name: "Genesis Changes"
+    needs: build_binary
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          cache: pip
+      - uses: actions/download-artifact@v3
+        with:
+          name: neard
+          path: target/quick-release
+      - run: echo "CURRENT_NEARD=$PWD/target/quick-release/neard" >> "$GITHUB_ENV"
+      - run: chmod +x "$CURRENT_NEARD"
+      - run: pip3 install --user -r pytest/requirements.txt
       - run: python3 scripts/state/update_res.py check
 
+  py_style_check:
+    name: "Style"
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          cache: pip
+      - run: pip3 install --user -r pytest/requirements.txt
       - run: python3 scripts/check_nightly.py
       - run: python3 scripts/check_pytests.py
-
       - run: python3 scripts/fix_nightly_feature_flags.py
-
-      # TODO: this should probably be a separate job
       - run: ./scripts/formatting --check
 
-      # TODO: this should probably be a separate job
-      - name: check rpc_errors_schema.json
-        # TODO: the command ran and suggested are different. why? unify.
-        run: |
-          rm -f target/rpc_errors_schema.json
-          cargo check -p near-jsonrpc --features dump_errors_schema
-          if ! git --no-pager diff --no-index chain/jsonrpc/res/rpc_errors_schema.json target/rpc_errors_schema.json; then
-              set +x
-              echo 'The RPC errors schema reflects outdated typing structure; please run'
-              echo '    ./chain/jsonrpc/build_errors_schema.sh'
-              exit 1
-          fi >&2
-
-  upgradability:
+  py_upgradability:
     name: "Upgradability"
-    runs-on: ubuntu-22.04-16core
+    needs: build_binary
+    runs-on: ubuntu-22.04
     defaults:
       run:
         working-directory: ./pytest
@@ -161,9 +195,24 @@ jobs:
         with:
           python-version: 3.11
           cache: pip
+      - uses: actions/download-artifact@v3
+        with:
+          name: neard
+          path: pytest # NB: this does not account for defaults.run.working-directory
+      - run: echo "CURRENT_NEARD=$PWD/neard" >> "$GITHUB_ENV"
+      - run: chmod +x "$CURRENT_NEARD"
+      - run: pip3 install --user -r requirements.txt
+      - run: python3 tests/sanity/upgradable.py
+
+  rpc_error_schema:
+    name: "RPC Schema"
+    runs-on: ubuntu-22.04-8core
+    steps:
+      - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43
         with:
           prefix-key: "0" # change this to invalidate CI cache
+          shared-key: "cargo_nextest"
           save-if: "false" # use the cache from nextest, but don’t double-save
-      - run: pip3 install --user -r requirements.txt
-      - run: python3 tests/sanity/upgradable.py
+      - run: ./chain/jsonrpc/build_errors_schema.sh
+      - run: git diff --quiet ./chain/jsonrpc/res/rpc_errors_schema.json || exit 1

chain/jsonrpc/build_errors_schema.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 cd "${0%/*}/../.." # ensure we're in the workspace directory
 rm -f target/rpc_errors_schema.json
-cargo build -p near-jsonrpc --features dump_errors_schema
+cargo check -p near-jsonrpc --features dump_errors_schema
 cp target/rpc_errors_schema.json chain/jsonrpc/res/rpc_errors_schema.json

pytest/lib/branches.py

@@ -103,6 +103,17 @@ def escaped(branch):
 
 def _compile_current(branch: str) -> Executables:
     """Compile current branch."""
+    prebuilt_neard = os.environ.get("CURRENT_NEARD")
+    if prebuilt_neard is not None:
+        logger.info(
+            f'Using `CURRENT_NEARD={prebuilt_neard}` neard for branch {branch}')
+        try:
+            path = pathlib.Path(prebuilt_neard).resolve()
+            return Executables(path.parent, path)
+        except OSError as e:
+            logger.exception('Could not use `CURRENT_NEARD`, will build…')
+
+    logger.info(f'Building neard for branch {branch}')
     subprocess.check_call(['cargo', 'build', '-p', 'neard', '--bin', 'neard'],
                           cwd=_REPO_DIR)
     subprocess.check_call(['cargo', 'build', '-p', 'near-test-contracts'],

pytest/tests/sanity/db_migration.py

@@ -96,7 +96,8 @@ def main():
     # Run new node and verify it runs for a few more blocks.
     logging.info("Starting the current node...")
     config = executables.current.node_config()
-    node.binary_name = config['binary_name']
+    node.near_root = executables.current.root
+    node.binary_name = executables.current.neard
     node.start(boot_node=node)
 
     logging.info("Running the current node...")

pytest/tests/sanity/upgradable.py

@@ -144,7 +144,8 @@ def test_upgrade() -> None:
     # Restart stable nodes into new version.
     for i in range(3):
         nodes[i].kill()
-        nodes[i].binary_name = config['binary_name']
+        nodes[i].near_root = executables.current.root
+        nodes[i].binary_name = executables.current.neard
         nodes[i].start(
             boot_node=nodes[0],
             extra_env={"NEAR_TESTS_IMMEDIATE_PROTOCOL_UPGRADE": "1"},

scripts/state/update_res.py

@@ -34,9 +34,13 @@ def main():
 
 def near_init_genesis():
     with tempfile.TemporaryDirectory() as tempdir:
-        subprocess.check_call(
-            ('cargo', 'run', '-p', 'neard', '--bin', 'neard', '--', '--home',
-             tempdir, 'init', '--chain-id', 'sample'))
+        args = ['--home', tempdir, 'init', '--chain-id', 'sample']
+        prebuilt_neard = os.environ.get("CURRENT_NEARD")
+        if prebuilt_neard is not None:
+            subprocess.check_call([prebuilt_neard] + args)
+        else:
+            subprocess.check_call(
+                ['cargo', 'run', '-p', 'neard', '--bin', 'neard', '--'] + args)
         with open(os.path.join(tempdir, 'genesis.json')) as rd:
             genesis = json.load(rd, object_pairs_hook=collections.OrderedDict)
     genesis['records'] = []