Problem: integration tests in CI are slow (#419)

Solution: run in parallel using pytest markers and job matrix Update tests/integration_tests/conftest.py Signed-off-by: yihuang <[email protected]> setup new cachix name
crypto-org-chain · Mar 12, 2024 · 715c7ea · 715c7ea
1 parent e391540
commit 715c7ea
Show file tree

Hide file tree

Showing 7 changed files with 45 additions and 9 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -58,9 +58,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: cachix/install-nix-action@v20
-      - uses: cachix/cachix-action@v12
+      - uses: cachix/cachix-action@v14
         with:
-          name: ethermint
+          name: crypto-ethermint
       - uses: technote-space/[email protected]
         with:
           PATTERNS: |
@@ -74,9 +74,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: cachix/install-nix-action@v20
-      - uses: cachix/cachix-action@v12
+      - uses: cachix/cachix-action@v14
         with:
-          name: ethermint
+          name: crypto-ethermint
       - uses: technote-space/[email protected]
         with:
           PATTERNS: |

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -86,16 +86,21 @@ jobs:
   integration_tests:
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    strategy:
+      matrix:
+        tests: [unmarked, upgrade, filter]
+    env:
+      TESTS_TO_RUN: ${{ matrix.tests }}
     steps:
       - uses: actions/checkout@v3
       - uses: cachix/install-nix-action@v23
         with:
           nix_path: nixpkgs=channel:nixos-22.11
           extra_nix_config: |
             access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
-      - uses: cachix/cachix-action@v12
+      - uses: cachix/cachix-action@v14
         with:
-          name: ethermint
+          name: crypto-ethermint
           signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
       - uses: technote-space/[email protected]
         with:
@@ -128,9 +133,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: cachix/install-nix-action@v20
-      - uses: cachix/cachix-action@v12
+      - uses: cachix/cachix-action@v14
         with:
-          name: ethermint
+          name: crypto-ethermint
           signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
       - name: 'instantiate integration test env'
         run: nix-store -r $(nix-instantiate tests/integration_tests/shell.nix)

diff --git a/Makefile b/Makefile
@@ -343,6 +343,12 @@ test-import:
 test-rpc:
 	./scripts/integration-test-all.sh -t "rpc" -q 1 -z 1 -s 5 -m "rpc" -r "true"
 
+# possible values:
+# - all: run all integration tests
+# - unmarked: run integration tests that are not marked
+# - marker1,marker2: markers separated by comma, run integration tests that are marked with any of the markers
+TESTS_TO_RUN ?= all
+
 run-integration-tests:
 	@nix-shell ./tests/integration_tests/shell.nix --run ./scripts/run-integration-tests.sh
 

diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
@@ -10,4 +10,13 @@ cd ../tests/integration_tests/hardhat
 HUSKY_SKIP_INSTALL=1 npm install
 npm run typechain
 cd ..
-pytest -vv -s
+
+TESTS_TO_RUN="${TESTS_TO_RUN:-all}"
+
+if [[ "$TESTS_TO_RUN" == "all" ]]; then
+  echo "run all tests"
+  pytest -vv -s
+else
+  echo "run tests matching $TESTS_TO_RUN"
+  pytest -vv -s -m "$TESTS_TO_RUN"
+fi
diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py
@@ -3,6 +3,18 @@
 from .network import setup_ethermint, setup_geth
 
 
+def pytest_configure(config):
+    config.addinivalue_line("markers", "unmarked: fallback mark for unmarked tests")
+    config.addinivalue_line("markers", "upgrade: upgrade tests")
+    config.addinivalue_line("markers", "filter: filter tests")
+
+
+def pytest_collection_modifyitems(items, config):
+    for item in items:
+        if not any(item.iter_markers()):
+            item.add_marker("unmarked")
+
+
 @pytest.fixture(scope="session")
 def ethermint(tmp_path_factory):
     path = tmp_path_factory.mktemp("ethermint")

diff --git a/tests/integration_tests/test_filters.py b/tests/integration_tests/test_filters.py
@@ -15,6 +15,8 @@
     w3_wait_for_new_blocks,
 )
 
+pytestmark = pytest.mark.filter
+
 
 @pytest.fixture(scope="module")
 def custom_ethermint(tmp_path_factory):

diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py
@@ -21,6 +21,8 @@
     wait_for_port,
 )
 
+pytestmark = pytest.mark.upgrade
+
 
 def init_cosmovisor(home):
     """
-Original file line number
+Diff line change
@@ Expand Up / @@ -21,6 +21,8 @@ @@
         wait_for_port,
     )
+    pytestmark = pytest.mark.upgrade
     def init_cosmovisor(home):
         """
@@ Expand Down @@