chore: nayduck in ci on merge (#12132)

Adding nayduck to ci on merge. Adding only one group of tests for now.
Extracting small portion of tests taking >12m to complete to
very_expensive.txt, fixing them should be a rare event anyway.

---------

Co-authored-by: Andrei <[email protected]>

.github/workflows/nayduck_ci.yml

@@ -0,0 +1,46 @@
+name: CI Nayduck tests
+on:
+  merge_group:
+
+  workflow_dispatch:
+
+jobs:
+  nayduck_tests:
+    runs-on: "ubuntu-latest"
+    environment: development
+    timeout-minutes: 60
+
+    steps:
+      - name: Install JQ json processor
+        run: sudo apt install jq
+
+      - name: Install required python modules
+        run: |
+          pip3 install -r pytest/requirements.txt
+
+      - name: Create nayduck-code file
+        run: |
+          echo ${{ secrets.NAYDUCK_CODE }} > ~/.config/nayduck-code
+
+      - name: Run Nayduck tests and wait for results
+        run: |
+          NEW_TEST=$(python3 scripts/nayduck.py  --test-file nightly/ci.txt)
+          RUN_ID=$(echo $NEW_TEST | grep https | sed -E 's|.*\/run\/([0-9]+)|\1|g')
+
+          # wait all the tests to finish
+          while true; do
+            TEST_RESULTS=$(curl -s https://nayduck.nearone.org/api/run/$RUN_ID)
+            TESTS_NOT_READY=$( jq -e '.tests | .[] | select(.status == "RUNNING" or .status == "PENDING" ) ' <<< ${TEST_RESULTS} )
+            if [ -z "$TESTS_NOT_READY" ]; then break; fi
+            sleep 15
+          done
+
+          UNSUCCESSFUL_TESTS=$(jq -e '.tests | .[] | select(.status != "PASSED" and .status != "IGNORED") ' <<< ${TEST_RESULTS} )
+          if [ -z "$UNSUCCESSFUL_TESTS" ]; then
+            echo "Nayduck CI tests passed."
+            echo "Results available at https://nayduck.nearone.org/#/run/$RUN_ID"
+          else
+            echo "CI Nayduck tests are failing https://nayduck.nearone.org/#/run/$RUN_ID."
+            echo "Fix them before merging"
+            exit 1
+          fi

nightly/ci.txt

@@ -0,0 +1,2 @@
+# TODO: add remaining tests.
+./pytest.txt

nightly/expensive.txt

@@ -17,12 +17,6 @@ expensive --timeout=1800 near-client near_client tests::catching_up::test_catchu
 expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync_height_6 --features nightly
 expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_sanity_blocks_produced
 expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_sanity_blocks_produced --features nightly
-expensive --timeout=3600 near-client near_client tests::catching_up::test_all_chunks_accepted_1000
-expensive --timeout=3600 near-client near_client tests::catching_up::test_all_chunks_accepted_1000 --features nightly
-expensive --timeout=7200 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_slow
-expensive --timeout=7200 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_slow --features nightly
-expensive --timeout=1800 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_rare_epoch_changing
-expensive --timeout=1800 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_rare_epoch_changing --features nightly
 expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_hold
 expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_hold --features nightly
 

nightly/nightly.txt

@@ -1,3 +1,4 @@
 ./sandbox.txt
 ./pytest.txt
 ./expensive.txt
+./very_expensive.txt

nightly/pytest-sanity.txt

@@ -159,9 +159,6 @@ pytest --timeout=300 sanity/rpc_hash.py --features nightly
 pytest sanity/rosetta.py
 pytest sanity/rosetta.py --features nightly
 
-# Make sure Docker image can be build and run
-pytest --skip-build --timeout=1h sanity/docker.py
-
 # This is the test for meta transactions.
 pytest sanity/meta_tx.py --features nightly
 

nightly/very_expensive.txt

@@ -0,0 +1,11 @@
+# Catchup tests
+expensive --timeout=3600 near-client near_client tests::catching_up::test_all_chunks_accepted_1000
+expensive --timeout=3600 near-client near_client tests::catching_up::test_all_chunks_accepted_1000 --features nightly
+expensive --timeout=7200 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_slow
+expensive --timeout=7200 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_slow --features nightly
+expensive --timeout=1800 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_rare_epoch_changing
+expensive --timeout=1800 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_rare_epoch_changing --features nightly
+
+# Make sure Docker image can be build and run
+pytest --skip-build --timeout=1h sanity/docker.py
+

scripts/nayduck.py

@@ -184,7 +184,7 @@ def impl(lines: typing.Iterable[str],
             line = line.rstrip()
             if line.startswith('./') or (include_comments and
                                          line.startswith('#./')):
-                if depth == 3:
+                if depth == 4:
                     print(f'{filename}:{lineno}: ignoring {line}; '
                           f'would exceed depth limit of {depth}')
                 else: