Refactor unittest and integration-tests and CI

Separate unittest and integration-tests into separate directories for easier management.
Change CI to collect unittest and integration-tests' coverage, merge them, and report together.

Signed-off-by: Tyler Gu <[email protected]>

.github/workflows/unittest.yaml

@@ -7,7 +7,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - name: Install Packages
         run: pip install codespell flake8 
       - name: Check Spelling
@@ -23,11 +23,80 @@ jobs:
           pip install -r requirements.txt
           make
       - name: Run unittest
+        run: >
+          pytest acto
+          --junitxml=pytest.xml
+          --cov-report=
+          --cov=acto
+      - name: Upload coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: .coverage.${{ github.sha }}.unittest
+          path: .coverage
+  integration-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: Install Packages
+        run: pip install codespell flake8 
+      - name: Check Spelling
+        run: codespell --skip="./data"|| true  
+      - name: Check Syntax Error
+        run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.20.5'
+      - name: Install dependencies
         run: |
-          pytest -m "not local and not singleBugReproduction" --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=acto | tee pytest-coverage.txt
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          make
+      - name: Run integration test
+        run: >
+          pytest test/integration_tests
+          --junitxml=pytest.xml
+          --cov-report=
+          --cov=acto
+      - name: Upload coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: .coverage.${{ github.sha }}.integration-test
+          path: .coverage
       - name: Pytest coverage comment
         if: ${{ github.event == 'pull_request' }}
         uses: MishaKav/pytest-coverage-comment@main
         with:
-          pytest-coverage-path: ./pytest-coverage.txt
-          junitxml-path: ./pytest.xml
+          pytest-coverage-path: ./integration-test-coverage.txt
+          junitxml-path: ./pytest.xml
+  coverage-report:
+    runs-on: ubuntu-latest
+    needs: [unittest, integration-test]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download unittest coverage
+        uses: actions/download-artifact@v2
+        with:
+          name: .coverage.${{ github.sha }}.unittest
+      - name: Download integration test coverage
+        uses: actions/download-artifact@v2
+        with:
+          name: .coverage.${{ github.sha }}.integration-test
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+      - name: Install Packages
+        run: pip install coverage
+      - name: Combine coverage
+        run: |
+          coverage combine .coverage.${{ github.sha }}.unittest .coverage.${{ github.sha }}.integration-test
+      - name: Generate coverage report
+        run: coverage report -m --skip-covered > coverage.txt
+      - name: Coverage comment
+        if: ${{ github.event == 'pull_request' }}
+        uses: MishaKav/coverage-comment@main
+        with:
+          coverage-path: ./coverage.txt

test/test_bug_reproduction.py → test/e2e_tests/test_bug_reproduction.py

@@ -3,21 +3,26 @@
 import pathlib
 import queue
 import random
-from typing import Dict, List, Tuple
 import unittest
+from test.utils import BugConfig, all_bugs, check_postdiff_runtime_error
+from typing import Dict, List, Tuple
 
 import pytest
+
 from acto.common import PassResult
 from acto.reproduce import reproduce, reproduce_postdiff
 
-from test.utils import BugConfig, all_bugs, check_postdiff_runtime_error
-
 test_dir = pathlib.Path(__file__).parent.resolve()
 test_data_dir = os.path.join(test_dir, 'test_data')
 
-def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_namespace: int) -> bool:
+
+def run_bug_config(
+        operator_name: str,
+        bug_id: str,
+        bug_config: BugConfig,
+        acto_namespace: int) -> bool:
     '''This function tries to reproduce a bug according to the bug config
-    
+
     Returns:
         if the reproduction is successful
     '''
@@ -32,18 +37,18 @@ def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_
                                   cluster_runtime='KIND',
                                   acto_namespace=acto_namespace)
     if bug_config.difftest:
-        if bug_config.diffdir != None:
+        if bug_config.diffdir is not None:
             diff_repro_dir = os.path.join(test_data_dir, bug_config.diffdir)
             work_dir = f'testrun-{bug_id}-diff'
             reproduce(work_dir,
-                        diff_repro_dir,
-                        operator_config,
-                        cluster_runtime='KIND',
-                        acto_namespace=acto_namespace)
+                      diff_repro_dir,
+                      operator_config,
+                      cluster_runtime='KIND',
+                      acto_namespace=acto_namespace)
         if reproduce_postdiff(work_dir,
-                                operator_config,
-                                cluster_runtime='KIND',
-                                acto_namespace=acto_namespace):
+                              operator_config,
+                              cluster_runtime='KIND',
+                              acto_namespace=acto_namespace):
             reproduced = True
         else:
             print(f"Bug {bug_id} not reproduced!")
@@ -52,10 +57,10 @@ def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_
     if bug_config.declaration:
         if len(normal_run_result) != 0:
             last_error = normal_run_result[-1]
-            if last_error.state_result != None and not isinstance(
+            if last_error.state_result is not None and not isinstance(
                     last_error.state_result, PassResult):
                 reproduced = True
-            elif last_error.recovery_result != None and not isinstance(
+            elif last_error.recovery_result is not None and not isinstance(
                     last_error.recovery_result, PassResult):
                 reproduced = True
         else:
@@ -65,10 +70,10 @@ def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_
     if bug_config.recovery:
         if len(normal_run_result) != 0:
             last_error = normal_run_result[-1]
-            if last_error.recovery_result != None and not isinstance(
+            if last_error.recovery_result is not None and not isinstance(
                     last_error.recovery_result, PassResult):
                 reproduced = True
-            elif last_error.state_result != None and not isinstance(
+            elif last_error.state_result is not None and not isinstance(
                     last_error.state_result, PassResult):
                 reproduced = True
         else:
@@ -80,7 +85,7 @@ def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_
             reproduced = True
         elif len(normal_run_result) != 0:
             last_error = normal_run_result[-1]
-            if last_error.health_result != None and not isinstance(
+            if last_error.health_result is not None and not isinstance(
                     last_error.health_result, PassResult):
                 reproduced = True
         else:
@@ -92,13 +97,17 @@ def run_bug_config(operator_name: str, bug_id: str, bug_config: BugConfig, acto_
     else:
         return False
 
-def run_worker(workqueue: multiprocessing.Queue, acto_namespace: int, reproduction_results: Dict[str, bool]):
+
+def run_worker(
+        workqueue: multiprocessing.Queue, acto_namespace: int,
+        reproduction_results: Dict[str, bool]):
     while True:
         try:
-            bug_tuple: Tuple[str, str, BugConfig] = workqueue.get(block=True, timeout=5)
+            bug_tuple: Tuple[str, str, BugConfig] = workqueue.get(
+                block=True, timeout=5)
         except queue.Empty:
             break
-        
+
         operator_name, bug_id, bug_config = bug_tuple
         reproduced = False
         NUM_RETRY = 3
@@ -113,14 +122,15 @@ def run_worker(workqueue: multiprocessing.Queue, acto_namespace: int, reproducti
 
         reproduction_results[bug_id] = reproduced
 
-@pytest.mark.local
+
+@pytest.mark.all_bug_reproduction
 class TestBugReproduction(unittest.TestCase):
 
     def __init__(self, methodName: str = "runTest") -> None:
         super().__init__(methodName)
 
         # TODO: make _num_workers a command line argument
-        self._num_workers = 2 # Number of workers to run the test
+        self._num_workers = 2  # Number of workers to run the test
 
     def test_all_bugs(self):
         manager = multiprocessing.Manager()
@@ -130,11 +140,14 @@ def test_all_bugs(self):
             for bug_id, bug_config in bugs.items():
                 workqueue.put((operator, bug_id, bug_config))
 
-        reproduction_results: Dict[str, bool] = manager.dict() # workers write reproduction results
-                                                               # to this dict. Bug ID -> if success
+        # workers write reproduction results
+        reproduction_results: Dict[str, bool] = manager.dict()
+        # to this dict. Bug ID -> if success
         processes: List[multiprocessing.Process] = []
         for i in range(self._num_workers):
-            p = multiprocessing.Process(target=run_worker, args=(workqueue, i, reproduction_results))
+            p = multiprocessing.Process(
+                target=run_worker, args=(
+                    workqueue, i, reproduction_results))
             p.start()
             processes.append(p)
 
@@ -145,16 +158,17 @@ def test_all_bugs(self):
         for bug_id, if_reproduced in reproduction_results.items():
             if not if_reproduced:
                 errors.append(bug_id)
-        
+
         self.assertFalse(errors, f'Test failed with {errors}')
 
+
 @pytest.mark.singleBugReproduction
 class TestSingleBugReproduction(unittest.TestCase):
 
     def __init__(self, methodName: str = "runTest") -> None:
         super().__init__(methodName)
 
-        self._num_workers = 1 # Number of workers to run the test
+        self._num_workers = 1  # Number of workers to run the test
 
     def test_all_bugs(self):
         manager = multiprocessing.Manager()
@@ -164,11 +178,14 @@ def test_all_bugs(self):
         bug_id, bug_config = random.choice(list(bugs.items()))
         workqueue.put((operator, bug_id, bug_config))
 
-        reproduction_results: Dict[str, bool] = manager.dict() # workers write reproduction results
-                                                               # to this dict. Bug ID -> if success
+        # workers write reproduction results
+        reproduction_results: Dict[str, bool] = manager.dict()
+        # to this dict. Bug ID -> if success
         processes: List[multiprocessing.Process] = []
         for i in range(self._num_workers):
-            p = multiprocessing.Process(target=run_worker, args=(workqueue, i, reproduction_results))
+            p = multiprocessing.Process(
+                target=run_worker, args=(
+                    workqueue, i, reproduction_results))
             p.start()
             processes.append(p)
 
@@ -179,9 +196,9 @@ def test_all_bugs(self):
         for bug_id, if_reproduced in reproduction_results.items():
             if not if_reproduced:
                 errors.append(bug_id)
-        
+
         self.assertFalse(errors, f'Test failed with {errors}')
 
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()