From fe92b5a9310e0d05df02713585e1fc9f9dd90279 Mon Sep 17 00:00:00 2001 From: Aleksandr Logunov Date: Wed, 25 Sep 2024 17:16:28 +0400 Subject: [PATCH] chore: nayduck in ci on merge (#12132) Adding nayduck to ci on merge. Adding only one group of tests for now. Extracting small portion of tests taking >12m to complete to very_expensive.txt, fixing them should be a rare event anyway. --------- Co-authored-by: Andrei <122784628+andrei-near@users.noreply.github.com> --- .github/workflows/nayduck_ci.yml | 46 +++++++++++++++++++++++++++ .github/workflows/nightly_nayduck.yml | 21 ------------ nightly/ci.txt | 2 ++ nightly/expensive.txt | 6 ---- nightly/nightly.txt | 1 + nightly/pytest-sanity.txt | 3 -- nightly/very_expensive.txt | 11 +++++++ scripts/nayduck.py | 2 +- 8 files changed, 61 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/nayduck_ci.yml delete mode 100644 .github/workflows/nightly_nayduck.yml create mode 100644 nightly/ci.txt create mode 100644 nightly/very_expensive.txt diff --git a/.github/workflows/nayduck_ci.yml b/.github/workflows/nayduck_ci.yml new file mode 100644 index 00000000000..d4791764a67 --- /dev/null +++ b/.github/workflows/nayduck_ci.yml @@ -0,0 +1,46 @@ +name: CI Nayduck tests +on: + merge_group: + + workflow_dispatch: + +jobs: + nayduck_tests: + runs-on: "ubuntu-latest" + environment: development + timeout-minutes: 60 + + steps: + - name: Install JQ json processor + run: sudo apt install jq + + - name: Install required python modules + run: | + pip3 install -r pytest/requirements.txt + + - name: Create nayduck-code file + run: | + echo ${{ secrets.NAYDUCK_CODE }} > ~/.config/nayduck-code + + - name: Run Nayduck tests and wait for results + run: | + NEW_TEST=$(python3 scripts/nayduck.py --test-file nightly/ci.txt) + RUN_ID=$(echo $NEW_TEST | grep https | sed -E 's|.*\/run\/([0-9]+)|\1|g') + + # wait all the tests to finish + while true; do + TEST_RESULTS=$(curl -s https://nayduck.nearone.org/api/run/$RUN_ID) + TESTS_NOT_READY=$( jq -e '.tests | .[] | select(.status == "RUNNING" or .status == "PENDING" ) ' <<< ${TEST_RESULTS} ) + if [ -z "$TESTS_NOT_READY" ]; then break; fi + sleep 15 + done + + UNSUCCESSFUL_TESTS=$(jq -e '.tests | .[] | select(.status != "PASSED" and .status != "IGNORED") ' <<< ${TEST_RESULTS} ) + if [ -z "$UNSUCCESSFUL_TESTS" ]; then + echo "Nayduck CI tests passed." + echo "Results available at https://nayduck.nearone.org/#/run/$RUN_ID" + else + echo "CI Nayduck tests are failing https://nayduck.nearone.org/#/run/$RUN_ID." + echo "Fix them before merging" + exit 1 + fi diff --git a/.github/workflows/nightly_nayduck.yml b/.github/workflows/nightly_nayduck.yml deleted file mode 100644 index 0f22a8d55f8..00000000000 --- a/.github/workflows/nightly_nayduck.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Nightly Nayduck tests check -on: - merge_group: - -jobs: - nightly_nayduck_tests: - runs-on: "ubuntu-latest" - timeout-minutes: 10 - - steps: - - name: Install JQ json processor - run: sudo apt install jq - - # In this step we get the latest nightly results from the nayduck server - # and check if there are any non-passing tests - - name: Check if there are any non-passing tests - run: | - NIGHTLY_RESULTS=$(curl -s https://nayduck.nearone.org/api/nightly-events) - UNSUCCESSFUL_TESTS=$(jq -e '.tests | .[][] | select(.[2] != "PASSED" ) ' <<< ${NIGHTLY_RESULTS} ) - if [ -z "$UNSUCCESSFUL_TESTS" ] ; then echo "Nightly Nayduck tests OK"; \ - else echo "Nightly Nayduck tests are failing" && exit 1; fi diff --git a/nightly/ci.txt b/nightly/ci.txt new file mode 100644 index 00000000000..6f4d12d54e8 --- /dev/null +++ b/nightly/ci.txt @@ -0,0 +1,2 @@ +# TODO: add remaining tests. +./pytest.txt \ No newline at end of file diff --git a/nightly/expensive.txt b/nightly/expensive.txt index cfd4cefc9d3..8a2630746bc 100644 --- a/nightly/expensive.txt +++ b/nightly/expensive.txt @@ -17,12 +17,6 @@ expensive --timeout=1800 near-client near_client tests::catching_up::test_catchu expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_random_single_part_sync_height_6 --features nightly expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_sanity_blocks_produced expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_sanity_blocks_produced --features nightly -expensive --timeout=3600 near-client near_client tests::catching_up::test_all_chunks_accepted_1000 -expensive --timeout=3600 near-client near_client tests::catching_up::test_all_chunks_accepted_1000 --features nightly -expensive --timeout=7200 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_slow -expensive --timeout=7200 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_slow --features nightly -expensive --timeout=1800 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_rare_epoch_changing -expensive --timeout=1800 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_rare_epoch_changing --features nightly expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_hold expensive --timeout=1800 near-client near_client tests::catching_up::test_catchup_receipts_sync_hold --features nightly diff --git a/nightly/nightly.txt b/nightly/nightly.txt index d99d8445f57..11285397dab 100644 --- a/nightly/nightly.txt +++ b/nightly/nightly.txt @@ -1,3 +1,4 @@ ./sandbox.txt ./pytest.txt ./expensive.txt +./very_expensive.txt \ No newline at end of file diff --git a/nightly/pytest-sanity.txt b/nightly/pytest-sanity.txt index 0da6c0a2a0a..383ce9bdc69 100644 --- a/nightly/pytest-sanity.txt +++ b/nightly/pytest-sanity.txt @@ -159,9 +159,6 @@ pytest --timeout=300 sanity/rpc_hash.py --features nightly pytest sanity/rosetta.py pytest sanity/rosetta.py --features nightly -# Make sure Docker image can be build and run -pytest --skip-build --timeout=1h sanity/docker.py - # This is the test for meta transactions. pytest sanity/meta_tx.py --features nightly diff --git a/nightly/very_expensive.txt b/nightly/very_expensive.txt new file mode 100644 index 00000000000..6bf4468619c --- /dev/null +++ b/nightly/very_expensive.txt @@ -0,0 +1,11 @@ +# Catchup tests +expensive --timeout=3600 near-client near_client tests::catching_up::test_all_chunks_accepted_1000 +expensive --timeout=3600 near-client near_client tests::catching_up::test_all_chunks_accepted_1000 --features nightly +expensive --timeout=7200 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_slow +expensive --timeout=7200 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_slow --features nightly +expensive --timeout=1800 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_rare_epoch_changing +expensive --timeout=1800 near-client near_client tests::catching_up::test_all_chunks_accepted_1000_rare_epoch_changing --features nightly + +# Make sure Docker image can be build and run +pytest --skip-build --timeout=1h sanity/docker.py + diff --git a/scripts/nayduck.py b/scripts/nayduck.py index f9d1bfaf430..8a66c842f27 100755 --- a/scripts/nayduck.py +++ b/scripts/nayduck.py @@ -184,7 +184,7 @@ def impl(lines: typing.Iterable[str], line = line.rstrip() if line.startswith('./') or (include_comments and line.startswith('#./')): - if depth == 3: + if depth == 4: print(f'{filename}:{lineno}: ignoring {line}; ' f'would exceed depth limit of {depth}') else: