Run all-runtimes tests independently to avoid segfault

SeldonIO · Jun 18, 2024 · 3368e3c · 3368e3c
1 parent 36aae4b
commit 3368e3c
Show file tree

Hide file tree

Showing 12 changed files with 151 additions and 129 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -9,51 +9,51 @@ on:
     branches: [master]
 
 jobs:
-  # generate:
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       python-version: ["3.9", "3.10"]
-  #       target: ["dataplane", "model-repository"]
-  #   runs-on: ubuntu-22.04
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Set up Python ${{ matrix.python-version }}
-  #       uses: actions/setup-python@v5
-  #       with:
-  #         python-version: ${{ matrix.python-version }}
-  #     - name: Install Poetry
-  #       uses: snok/install-poetry@v1
-  #       with:
-  #         virtualenvs-create: false
-  #     - name: Install dependencies
-  #       run: poetry install --sync --only dev
-  #     - name: Generate ${{ matrix.target }}
-  #       run: make generate-${{ matrix.target }}
-  #     - name: Check for changes
-  #       run: make lint-no-changes
+  generate:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10"]
+        target: ["dataplane", "model-repository"]
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: false
+      - name: Install dependencies
+        run: poetry install --sync --only dev
+      - name: Generate ${{ matrix.target }}
+        run: make generate-${{ matrix.target }}
+      - name: Check for changes
+        run: make lint-no-changes
 
-  # lint:
-  #   strategy:
-  #     matrix:
-  #       python-version:
-  #         - "3.9"
-  #         - "3.10"
-  #   runs-on: ubuntu-22.04
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Set up Python ${{ matrix.python-version }}
-  #       uses: actions/setup-python@v5
-  #       with:
-  #         python-version: ${{ matrix.python-version }}
-  #     - name: Install Poetry
-  #       uses: snok/install-poetry@v1
-  #       with:
-  #         virtualenvs-create: false
-  #     - name: Install dependencies
-  #       run: poetry install --sync --only dev
-  #     - name: Lint
-  #       run: make lint
+  lint:
+    strategy:
+      matrix:
+        python-version:
+          - "3.9"
+          - "3.10"
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: false
+      - name: Install dependencies
+        run: poetry install --sync --only dev
+      - name: Lint
+        run: make lint
 
   test-mlserver:
     strategy:
@@ -65,7 +65,7 @@ jobs:
         # NOTE: There's no pre-built `grpcio` wheel for Python 3.11 yet
         # https://github.com/grpc/grpc/issues/32454
         python-version:
-          # - "3.9"
+          - "3.9"
           - "3.10"
         is-pr:
           - ${{ github.event_name == 'pull_request' }}
@@ -99,66 +99,63 @@ jobs:
         run: poetry install --only test
       - name: Test
         run: |
-          if [[ "$(uname)" == 'Darwin' ]]; then
+          if [[ ${{ runner.os == 'macOS' }} ]]; then
             source ./hack/activate-libomp-macos.sh
-            poetry run tox -e mlserver -- -- -n auto
-          else
-            poetry run tox -e mlserver -- -- -n auto
           fi
-          
+          poetry run tox -e mlserver
 
-  # test-runtimes:
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       os:
-  #         - ubuntu-22.04
-  #         - macos-13
-  #       python-version:
-  #         - "3.9"
-  #         - "3.10"
-  #       tox-environment:
-  #         - sklearn
-  #         - xgboost
-  #         - lightgbm
-  #         - mlflow
-  #         - huggingface
-  #         - alibi-explain
-  #         - alibi-detect
-  #         - catboost
-  #       is-pr:
-  #         - ${{ github.event_name == 'pull_request' }}
-  #       # exclude:
-  #       #   # MacOS tests take a lot of time, so we will run them only on merge
-  #       #   # From https://github.com/orgs/community/discussions/26253
-  #       #   - is-pr: true
-  #       #     os: macos-13
-  #   runs-on: ${{ matrix.os }}
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Install OpenMP (in MacOS)
-  #       if: runner.os == 'macOS'
-  #       run: brew install libomp
-  #     - name: Set up Python ${{ matrix.python-version }}
-  #       uses: actions/setup-python@v5
-  #       with:
-  #         python-version: ${{ matrix.python-version }}
-  #     - uses: conda-incubator/setup-miniconda@v3
-  #       with:
-  #         auto-update-conda: true
-  #         auto-activate-base: false
-  #     - name: Install Poetry
-  #       uses: snok/install-poetry@v1
-  #       with:
-  #         virtualenvs-create: false
-  #     - name: Install dependencies
-  #       run: poetry install --only test
-  #     - name: Test
-  #       run: |
-  #         if [[ "$(uname)" == 'Darwin' ]]; then
-  #           source ./hack/activate-libomp-macos.sh
-  #         fi
-  #         poetry run tox -c ./runtimes/${{ matrix.tox-environment }} -- -- -n auto
+  test-runtimes:
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-22.04
+          - macos-13
+        python-version:
+          - "3.9"
+          - "3.10"
+        tox-environment:
+          - sklearn
+          - xgboost
+          - lightgbm
+          - mlflow
+          - huggingface
+          - alibi-explain
+          - alibi-detect
+          - catboost
+        is-pr:
+          - ${{ github.event_name == 'pull_request' }}
+        # exclude:
+        #   # MacOS tests take a lot of time, so we will run them only on merge
+        #   # From https://github.com/orgs/community/discussions/26253
+        #   - is-pr: true
+        #     os: macos-13
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install OpenMP (in MacOS)
+        if: runner.os == 'macOS'
+        run: brew install libomp
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          auto-activate-base: false
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: false
+      - name: Install dependencies
+        run: poetry install --only test
+      - name: Test
+        run: |
+          if [[ ${{ runner.os == 'macOS' }} ]]; then
+            source ./hack/activate-libomp-macos.sh
+          fi
+          poetry run tox -c ./runtimes/${{ matrix.tox-environment }}
 
   # Ensure that having all the runtimes installed together works
   test-all-runtimes:
@@ -170,7 +167,7 @@ jobs:
           - ubuntu-22.04
           - macos-13
         python-version:
-          # - "3.9"
+          - "3.9"
           - "3.10"
     runs-on: ${{ matrix.os }}
     steps:
@@ -208,10 +205,7 @@ jobs:
         run: poetry install --only test
       - name: Test
         run: |
-          if [[ "$(uname)" == 'Darwin' ]]; then
+          if [[ ${{ runner.os == 'macOS' }} ]]; then
             source ./hack/activate-libomp-macos.sh
-            poetry run tox -e all-runtimes -- -- -n auto
-          else
-            poetry run tox -e all-runtimes -- -- -n auto
           fi
-          
+          poetry run tox -e all-runtimes
diff --git a/runtimes/alibi-detect/tox.ini b/runtimes/alibi-detect/tox.ini
@@ -7,4 +7,5 @@ commands_pre =
     poetry install --sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
-    python -m pytest {posargs} {toxinidir}/tests
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests
diff --git a/runtimes/alibi-explain/tox.ini b/runtimes/alibi-explain/tox.ini
@@ -7,4 +7,5 @@ commands_pre =
     poetry install --sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
-    python -m pytest {posargs} {toxinidir}/tests
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests
diff --git a/runtimes/catboost/tox.ini b/runtimes/catboost/tox.ini
@@ -7,4 +7,5 @@ commands_pre =
     poetry install --sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
-    python -m pytest {posargs} {toxinidir}/tests
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests
diff --git a/runtimes/huggingface/tox.ini b/runtimes/huggingface/tox.ini
@@ -7,4 +7,5 @@ commands_pre =
     poetry install --sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
-    python -m pytest {posargs} {toxinidir}/tests
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests
diff --git a/runtimes/lightgbm/tox.ini b/runtimes/lightgbm/tox.ini
@@ -7,4 +7,5 @@ commands_pre =
     poetry install --sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
-    python -m pytest {posargs} {toxinidir}/tests
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests
diff --git a/runtimes/mlflow/tox.ini b/runtimes/mlflow/tox.ini
@@ -7,4 +7,5 @@ commands_pre =
     poetry install --sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
-    python -m pytest {posargs} {toxinidir}/tests
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests
diff --git a/runtimes/mllib/tox.ini b/runtimes/mllib/tox.ini
@@ -7,4 +7,5 @@ commands_pre =
     poetry install --sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
-    python -m pytest {posargs} {toxinidir}/tests
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests
diff --git a/runtimes/sklearn/tox.ini b/runtimes/sklearn/tox.ini
@@ -7,4 +7,5 @@ commands_pre =
     poetry install --sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
-    python -m pytest {posargs} {toxinidir}/tests
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests
diff --git a/runtimes/xgboost/tox.ini b/runtimes/xgboost/tox.ini
@@ -7,4 +7,5 @@ commands_pre =
     poetry install --sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
-    python -m pytest {posargs} {toxinidir}/tests
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests
diff --git a/tox.ini b/tox.ini
@@ -13,13 +13,18 @@ allowlist_externals = poetry
 commands_pre =
     poetry install --sync --no-root
 commands =
-    python -m pytest {posargs}
+    python -m pytest {posargs} -n auto
 
 [testenv:mlserver]
 commands_pre =
     poetry install --sync --no-root
 commands =
-    python -m pytest {posargs} {toxinidir}/tests
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests --ignore={toxinidir}/tests/kafka
+    # kafka tests are failinig for macos when running in parallel
+    # with the entire test suite. So, we run them separately.
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests/kafka
 set_env =
     GITHUB_SERVER_URL = {env:GITHUB_SERVER_URL:https\://github.com}
     GITHUB_REPOSITORY = {env:GITHUB_REPOSITORY:SeldonIO/MLServer}
@@ -31,16 +36,29 @@ commands_pre =
         --with all-runtimes \
         --with all-runtimes-dev
 commands =
-    python -m pytest {posargs} \
-        {toxinidir}/tests \
-        {toxinidir}/runtimes/alibi-explain \
-        {toxinidir}/runtimes/alibi-detect \
-        {toxinidir}/runtimes/sklearn \
-        {toxinidir}/runtimes/xgboost \
-        {toxinidir}/runtimes/mllib \
-        {toxinidir}/runtimes/lightgbm \
-        {toxinidir}/runtimes/mlflow \
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests --ignore={toxinidir}/tests/kafka
+    # kafka tests are failinig for macos when running in parallel
+    # with the entire test suite. So, we run them separately.
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests/kafka
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/runtimes/alibi-explain
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/runtimes/alibi-detect
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/runtimes/sklearn
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/runtimes/xgboost
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/runtimes/mllib
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/runtimes/lightgbm
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/runtimes/mlflow
+    python -m pytest {posargs} -n auto \
         {toxinidir}/runtimes/huggingface
+
 set_env =
     GITHUB_SERVER_URL = {env:GITHUB_SERVER_URL:https\://github.com}
     GITHUB_REPOSITORY = {env:GITHUB_REPOSITORY:SeldonIO/MLServer}

diff --git a/tox.runtime.ini b/tox.runtime.ini
@@ -7,4 +7,5 @@ commands_pre =
     poetry install --sync --no-root
     poetry install -C {toxinidir}/../../
 commands =
-    python -m pytest {posargs} {toxinidir}/tests
+    python -m pytest {posargs} -n auto \
+        {toxinidir}/tests