fix: the array api intelligent tests now use 4 runners (#27193)

To avoid the 6 hour timeout when running the coverage workflow, also added the display-test-results job for summarizing the results similar to that in the regular intelligent tests.
ivy-llc · Nov 2, 2023 · 5546e38 · 5546e38
1 parent 4bf62f3
commit 5546e38
Show file tree

Hide file tree

Showing 5 changed files with 117 additions and 29 deletions.
diff --git a/.github/workflows/array-api-det-coverage.yml b/.github/workflows/array-api-det-coverage.yml
@@ -9,6 +9,10 @@ permissions:
 jobs:
   determine_coverage:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        branch: [ 1, 2, 3, 4 ]
     steps:
       - name: Checkout Ivy 🛎
         uses: actions/checkout@v2
@@ -22,7 +26,7 @@ jobs:
         run: |
           pip install pydriller tqdm
           cd ivy
-          python scripts/determine_tests/array_api_det_coverage.py
+          python scripts/determine_tests/array_api_det_coverage.py ${{ matrix.branch }}
           cd ..
           mkdir tests
           cp ivy/tests.pbz2 tests/
@@ -37,4 +41,4 @@ jobs:
           destination-repository-name: 'Mapping'
           user-email: [email protected]
           commit-message: Update Array API Tests Mapping
-          target-branch: main
+          target-branch: main${{ matrix.branch }}
diff --git a/.github/workflows/array-api-intelligent-tests-pr.yml b/.github/workflows/array-api-intelligent-tests-pr.yml
@@ -5,33 +5,80 @@ on:
 
 permissions:
   actions: read
+
 jobs:
+  display_test_results:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs:
+      - run_tests
+
+    steps:
+      - name: Download all test results
+        uses: actions/download-artifact@v3
+
+      - name: Combined Test Results
+        run: |
+          find . -name "test_results_*.txt" -exec cat {} + > combined_test_results.txt
+          echo "Test results summary:"
+          cat combined_test_results.txt
+
+      - name: New Failures Introduced
+        run: |
+          find . -name "new_failures_*.txt" -exec cat {} + > combined_failures.txt
+          if [ -s combined_failures.txt ]
+          then
+              echo "This PR introduces the following new failing tests:"
+              cat combined_failures.txt
+          else
+              echo "This PR does not introduce any new test failures! Yippee!"
+          fi
+
   run_tests:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        branch: [ 1, 2, 3, 4 ]
+
     steps:
       - name: Checkout Ivy 🛎
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: ivy
           persist-credentials: false
           submodules: "recursive"
           fetch-depth: 100
 
-      - name: Determine Tests
+      - name: Get Job URL
+        uses: Tiryoh/gha-jobid-action@v0
+        id: jobs
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          job_name: ${{ github.job }}
+
+      - name: Determine and Run Tests
         run: |
-          git clone -b main https://github.com/unifyai/Mapping.git --depth 1
+          git clone -b main${{ matrix.branch }} https://github.com/unifyai/Mapping.git --depth 1
           pip install pydriller
           cp Mapping/tests.pbz2 ivy/
           cd ivy
           python scripts/determine_tests/array_api_determine_tests.py
+          python scripts/run_tests/array_api_run_tests_pr.py new_failures_${{ matrix.branch }}.txt | tee test_results_${{ matrix.branch }}.txt
+          cd ..
         continue-on-error: true
 
-      - name: Run Tests
-        id: tests
-        run: |
-          cd ivy
-          python scripts/run_tests/array_api_run_tests_pr.py
-        continue-on-error: true
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: test_results_${{ matrix.branch }}
+          path: ivy/test_results_${{ matrix.branch }}.txt
+
+      - name: Upload New Failures
+        uses: actions/upload-artifact@v3
+        with:
+          name: new_failures_${{ matrix.branch }}
+          path: ivy/new_failures_${{ matrix.branch }}.txt
 
       - name: Check on failures
         if: steps.tests.outcome != 'success'

diff --git a/.github/workflows/array-api-intelligent-tests.yml b/.github/workflows/array-api-intelligent-tests.yml
@@ -7,11 +7,32 @@ on:
 permissions:
   actions: read
 jobs:
+  display_test_results:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs:
+      - run_tests
+
+    steps:
+      - name: Download all test results
+        uses: actions/download-artifact@v3
+
+      - name: Combined Test Results
+        run: |
+          find . -name "test_results_*.txt" -exec cat {} + > combined_test_results.txt
+          echo "Test results summary:"
+          cat combined_test_results.txt
+
   run_tests:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        branch: [ 1, 2, 3, 4 ]
+
     steps:
       - name: Checkout Ivy 🛎
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: ivy
           persist-credentials: false
@@ -29,26 +50,33 @@ jobs:
         env:
           SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }}
         run: |
-          source ./ivy/scripts/shell/clone_mapping.sh main
+          source ./ivy/scripts/shell/clone_mapping.sh main${{ matrix.branch }}
           pip install pydriller pymongo
           cp Mapping/tests.pbz2 ivy/
           cd ivy
-          python scripts/determine_tests/array_api_determine_tests.py
+          python scripts/determine_tests/array_api_determine_tests.py ${{ matrix.branch }}
           cd ..
           cp ivy/tests.pbz2 Mapping/
           cd Mapping
           git add .
           git commit -m "Update Mapping"
-          git push origin main
+          git push origin main${{ matrix.branch }}
         continue-on-error: true
 
       - name: Run Tests
         id: tests
         run: |
           cd ivy
-          python scripts/run_tests/array_api_run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} ${{ github.run_id }} ${{ steps.jobs.outputs.html_url }}
+          set -o pipefail
+          python scripts/run_tests/array_api_run_tests.py ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD }} ${{ secrets.MONGODB_PASSWORD }} ${{ github.run_id }} ${{ steps.jobs.outputs.html_url }} | tee test_results_${{ matrix.branch }}.txt
         continue-on-error: true
 
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: test_results_${{ matrix.branch }}
+          path: ivy/test_results_${{ matrix.branch }}.txt
+
       - name: Check on failures
         if: steps.tests.outcome != 'success'
         run: exit 1
diff --git a/scripts/determine_tests/array_api_det_coverage.py b/scripts/determine_tests/array_api_det_coverage.py
@@ -1,4 +1,5 @@
 import os
+import sys
 import subprocess
 from pydriller import Repository
 from tqdm import tqdm
@@ -8,6 +9,8 @@
 
 def main():
     BACKENDS = ["numpy", "jax", "tensorflow", "torch"]
+    N = 4
+    run_iter = int(sys.argv[1]) - 1
 
     test_names = []
     func_folder = "ivy_tests/array_api_testing/array_api_methods_to_test"
@@ -77,7 +80,11 @@ def main():
         x for x in directories if not (x.endswith("__pycache__") or "hypothesis" in x)
     ]
     directories = set(directories_filtered)
-    for test_backend in tqdm(test_names):
+    num_tests = len(test_names)
+    tests_per_run = num_tests // N
+    start = run_iter * tests_per_run
+    end = num_tests if run_iter == N - 1 else (run_iter + 1) * tests_per_run
+    for test_backend in tqdm(test_names[start:end]):
         test_name, backend = test_backend.split(",")
         command = (
             f"docker run --rm --env IVY_BACKEND={backend} --env "

diff --git a/scripts/run_tests/array_api_run_tests_pr.py b/scripts/run_tests/array_api_run_tests_pr.py
@@ -21,18 +21,20 @@ def main():
             array_api_tests_k_flag += " and not (uint16 or uint32 or uint64)"
         k_flag[backend] = array_api_tests_k_flag
 
-    with open("tests_to_run", "r") as f:
-        for line in f:
-            test, backend = line.split(",")
-            backend = backend.strip("\n")
-            command = f'docker run --rm --env IVY_BACKEND={backend} --env ARRAY_API_TESTS_MODULE="ivy" -v "$(pwd)":/ivy -v "$(pwd)"/.hypothesis:/.hypothesis unifyai/ivy:latest timeout 30m python3 -m pytest {test} -k "{k_flag[backend]}" --tb=short -vv'  # noqa
-            print(f"\n{'*' * 100}")
-            print(f"{line[:-1]}")
-            print(f"{'*' * 100}\n")
-            sys.stdout.flush()
-            ret = os.system(command)
-            if ret != 0:
-                failed = True
+    with open(sys.argv[1], "w") as f_write:
+        with open("tests_to_run", "r") as f:
+            for line in f:
+                test, backend = line.split(",")
+                backend = backend.strip("\n")
+                command = f'docker run --rm --env IVY_BACKEND={backend} --env ARRAY_API_TESTS_MODULE="ivy" -v "$(pwd)":/ivy -v "$(pwd)"/.hypothesis:/.hypothesis unifyai/ivy:latest timeout 30m python3 -m pytest {test} -k "{k_flag[backend]}" --tb=short -vv'  # noqa
+                print(f"\n{'*' * 100}")
+                print(f"{line[:-1]}")
+                print(f"{'*' * 100}\n")
+                sys.stdout.flush()
+                ret = os.system(command)
+                if ret != 0:
+                    failed = True
+                    f_write.write(line)
     if failed:
         sys.exit(1)