Problem: integration test takes too long time to complete (fix #489) #…

…578

Solution: divide into multiple jobs

.github/workflows/nix.yml

@@ -6,58 +6,149 @@ on:
       - master
       - release/**
     tags:
-      - '*'
+      - "*"
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-      with:
+      - uses: actions/checkout@v2
+        with:
           submodules: true
-    - uses: cachix/install-nix-action@v12
-    - uses: cachix/cachix-action@v8
-      with:
-        name: crypto-com
-        skipPush: true
-    - name: lint
-      run: make lint-ci
+      - uses: cachix/install-nix-action@v12
+      - uses: cachix/cachix-action@v8
+        with:
+          name: crypto-com
+          skipPush: true
+      - name: lint
+        run: make lint-ci
 
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-      with:
-          submodules: true
-    - uses: cachix/install-nix-action@v12
-    - uses: cachix/cachix-action@v8
-      with:
-        name: crypto-com
-        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
-        skipPush: ${{github.event_name == 'pull_request'}}
-    - name: Run integration tests
-      run: make nix-integration-test
-    - name: Build chain utils
-      run: nix-build -A chain-utils-testnet
-    - name: Upload coverage report
-      uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: ./coverage.*.txt
-        flags: integration_tests
-    - name: Publish docker image
-      if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')
-      run: |
-        nix-build -A pystarportImage docker.nix
-        IMAGE=$(docker load < result)
-        IMAGE=${IMAGE#Loaded image: }
-        if [[ "$GITHUB_REF" = "refs/tags/"* ]]; then
-          TAG=${GITHUB_REF#refs/tags/}
-        else
-          TAG="latest"
-        fi
-        TARGET="docker.pkg.github.com/${{ github.repository }}/chain-main-pystarport:$TAG"
-        docker tag $IMAGE $TARGET
-        docker login -u "$GITHUB_ACTOR" -p "${{ github.token }}" docker.pkg.github.com
-        docker push $TARGET
-        echo "pushed: $TARGET"
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v12
+      - uses: cachix/cachix-action@v8
+        with:
+          name: crypto-com
+          signingKey: "${{ secrets.CACHIX_SIGNING_KEY }}"
+          skipPush: ${{github.event_name == 'pull_request'}}
+      - name: Run integration tests
+        run: make nix-integration-test
+      - name: Build chain utils
+        run: nix-build -A chain-utils-testnet
+      - name: Upload coverage report
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.*.txt
+          flags: integration_tests
+      - name: Publish docker image
+        if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')
+        run: |
+          nix-build -A pystarportImage docker.nix
+          IMAGE=$(docker load < result)
+          IMAGE=${IMAGE#Loaded image: }
+          if [[ "$GITHUB_REF" = "refs/tags/"* ]]; then
+            TAG=${GITHUB_REF#refs/tags/}
+          else
+            TAG="latest"
+          fi
+          TARGET="docker.pkg.github.com/${{ github.repository }}/chain-main-pystarport:$TAG"
+          docker tag $IMAGE $TARGET
+          docker login -u "$GITHUB_ACTOR" -p "${{ github.token }}" docker.pkg.github.com
+          docker push $TARGET
+          echo "pushed: $TARGET"
+
+  test-upgrade:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v12
+      - uses: cachix/cachix-action@v8
+        with:
+          name: crypto-com
+      - name: Run integration tests
+        run: make nix-integration-test-upgrade
+
+  test-ledger:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v12
+      - uses: cachix/cachix-action@v8
+        with:
+          name: crypto-com
+      - name: Run integration tests
+        run: make nix-integration-test-ledger
+
+  test-slow:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v12
+      - uses: cachix/cachix-action@v8
+        with:
+          name: crypto-com
+      - name: Run integration tests
+        run: make nix-integration-test-slow
+
+  test-ibc:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v12
+      - uses: cachix/cachix-action@v8
+        with:
+          name: crypto-com
+      - name: Run integration tests
+        run: make nix-integration-test-ibc
+
+  test-byzantine:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v12
+      - uses: cachix/cachix-action@v8
+        with:
+          name: crypto-com
+      - name: Run integration tests
+        run: make nix-integration-test-byzantine
+
+  test-gov:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v12
+      - uses: cachix/cachix-action@v8
+        with:
+          name: crypto-com
+      - name: Run integration tests
+        run: make nix-integration-test-gov
+
+  test-grpc:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: cachix/install-nix-action@v12
+      - uses: cachix/cachix-action@v8
+        with:
+          name: crypto-com
+      - name: Run integration tests
+        run: make nix-integration-test-grpc

Makefile

@@ -153,7 +153,32 @@ make-proto:
 ###############################################################################
 # nix installation: https://nixos.org/download.html
 nix-integration-test: check-network make-proto
-	nix run -f ./. run-integration-tests -c run-integration-tests
+	nix run -f ./default.nix run-integration-tests -c run-integration-tests
+
+nix-integration-test-upgrade: check-network
+	nix run -f ./default.nix run-integration-tests -c run-integration-tests "pytest -v -m upgrade"	
+
+nix-integration-test-ledger: check-network 
+	nix run -f ./default.nix run-integration-tests -c run-integration-tests "pytest -v -m ledger"		
+
+nix-integration-test-slow: check-network 
+	nix run -f ./default.nix run-integration-tests -c run-integration-tests "pytest -v -m slow"
+
+nix-integration-test-ibc: check-network 
+	nix run -f ./default.nix run-integration-tests -c run-integration-tests "pytest -v -m ibc"
+
+nix-integration-test-byzantine: check-network
+	nix run -f ./default.nix run-integration-tests -c run-integration-tests "pytest -v -m byzantine"
+
+nix-integration-test-gov: check-network 
+	nix run -f ./default.nix run-integration-tests -c run-integration-tests "pytest -v -m gov"
+
+nix-integration-test-grpc: check-network make-proto
+	nix run -f ./default.nix run-integration-tests -c run-integration-tests "pytest -v -m grpc"
+
+nix-integration-test-all: check-network make-proto
+	nix run -f ./default.nix run-integration-tests -c run-integration-tests "pytest -v"
+
 
 nix-build-%: check-network check-os
 	@if [ -e ~/.nix/remote-build-env ]; then \

default.nix

@@ -133,9 +133,13 @@ rec {
     set -e
     export PATH=${ci-env}/bin:$PATH
     export TESTS=${tests_src}/integration_tests
-    pytest -v -n 3 -m 'not ledger and not upgrade' --dist loadscope $TESTS
-    pytest -v -m upgrade $TESTS
-    pytest -v -m ledger $TESTS
+    # check argument exists, then use it, otherwise use default
+    if [ -z $1 ]
+    then 
+      pytest -v -m 'not upgrade and not ledger and not slow and not ibc and not byzantine and not gov and not grpc' $TESTS
+    else 
+      $1 $TESTS
+    fi
   '';
 
   ci-shell = pkgs.mkShell {

integration_tests/conftest.py

@@ -11,6 +11,10 @@ def pytest_configure(config):
     config.addinivalue_line("markers", "ledger: marks tests as ledger hardware test")
     config.addinivalue_line("markers", "grpc: marks grpc tests")
     config.addinivalue_line("markers", "upgrade: marks upgrade tests")
+    config.addinivalue_line("markers", "normal: marks normal tests")
+    config.addinivalue_line("markers", "ibc: marks ibc tests")
+    config.addinivalue_line("markers", "byzantine: marks byzantine tests")
+    config.addinivalue_line("markers", "gov: marks gov tests")
 
 
 def pytest_addoption(parser):

integration_tests/test_basic.py

@@ -1,5 +1,9 @@
+import pytest
+
 from .utils import wait_for_block
 
+pytestmark = pytest.mark.normal
+
 
 def test_simple(cluster):
     """

integration_tests/test_byzantine.py

@@ -20,7 +20,7 @@ def cluster(worker_index, pytestconfig, tmp_path_factory):
     )
 
 
-@pytest.mark.slow
+@pytest.mark.byzantine
 def test_byzantine(cluster):
     """
     - 3 nodes

integration_tests/test_gov.py

@@ -5,8 +5,9 @@
 
 from .utils import parse_events, wait_for_block, wait_for_block_time
 
+pytestmark = pytest.mark.gov
+
 
-@pytest.mark.slow
 @pytest.mark.parametrize("vote_option", ["yes", "no", "no_with_veto", "abstain", None])
 def test_param_proposal(cluster, vote_option):
     """
@@ -98,7 +99,6 @@ def test_param_proposal(cluster, vote_option):
         assert cluster.balance(cluster.address("ecosystem")) == amount
 
 
-@pytest.mark.slow
 def test_deposit_period_expires(cluster):
     """
     - proposal and partially deposit
@@ -156,7 +156,6 @@ def test_deposit_period_expires(cluster):
     assert cluster.balance(cluster.address("ecosystem")) == amount2 - 5000
 
 
-@pytest.mark.slow
 def test_community_pool_spend_proposal(cluster):
     """
     - proposal a community pool spend
@@ -208,7 +207,6 @@ def test_community_pool_spend_proposal(cluster):
     assert cluster.balance(recipient) == old_amount + amount
 
 
-@pytest.mark.slow
 def test_change_vote(cluster):
     """
     - submit proposal with deposit
@@ -263,7 +261,6 @@ def test_change_vote(cluster):
     }
 
 
-@pytest.mark.slow
 def test_inherit_vote(cluster):
     """
     - submit proposal with deposits

integration_tests/test_ibc.py

@@ -11,6 +11,8 @@
 
 from .utils import cluster_fixture, wait_for_block, wait_for_port
 
+pytestmark = pytest.mark.ibc
+
 
 @pytest.fixture(scope="module")
 def cluster(worker_index, pytestconfig, tmp_path_factory):

integration_tests/test_ibc_extended.py

@@ -9,6 +9,8 @@
 
 from .utils import cluster_fixture, wait_for_block, wait_for_port
 
+pytestmark = pytest.mark.ibc
+
 
 @pytest.fixture(scope="module")
 def cluster(worker_index, pytestconfig, tmp_path_factory):

integration_tests/test_multisig.py

@@ -1,10 +1,13 @@
 import json
 
+import pytest
 from pystarport import ports
 from pystarport.proto_python.api_util import ApiUtil
 
 from .utils import wait_for_new_blocks
 
+pytestmark = pytest.mark.normal
+
 
 def test_multi_signature(cluster, tmp_path):
     # prepare files

integration_tests/test_nft.py

@@ -1,5 +1,9 @@
 import json
 
+import pytest
+
+pytestmark = pytest.mark.normal
+
 
 def test_create_nft(cluster):
     """

integration_tests/test_reward.py

@@ -1,5 +1,9 @@
+import pytest
+
 from .utils import wait_for_block, wait_for_new_blocks
 
+pytestmark = pytest.mark.normal
+
 
 def test_reward(cluster):
     # starts with cro1

integration_tests/test_sign_offline.py

@@ -1,9 +1,12 @@
+import pytest
 from chainlibpy import Transaction, Wallet
 from pystarport import ports
 from pystarport.proto_python.api_util import ApiUtil
 
 from .utils import wait_for_new_blocks
 
+pytestmark = pytest.mark.normal
+
 
 def test_sign_offline(cluster):
     """