GHA: Split macOS C++ / Python test jobs (#2260)

Currently, macOS jobs run for >1h, this can be rather annoying. Splitting current tests into two separate jobs. Now we have 33' + 23' instead of 1h20'. Also: * cleanup some bash scripts * manage dependencies via pyproject.toml * move h5py dependency to extras * parallelize model compilation on macos
AMICI-dev · Jan 6, 2024 · 66f4ba5 · 66f4ba5
1 parent 30ed2f0
commit 66f4ba5
Show file tree

Hide file tree

Showing 6 changed files with 109 additions and 35 deletions.
diff --git a/.github/actions/install-macos-dependencies/action.yml b/.github/actions/install-macos-dependencies/action.yml
@@ -0,0 +1,31 @@
+name: Install AMICI dependencies for MacOS
+description: Install AMICI dependencies for MacOS
+
+runs:
+  using: "composite"
+  steps:
+    # use all available cores
+    - run: echo "AMICI_PARALLEL_COMPILE=" >> $GITHUB_ENV
+      shell: bash
+
+    # AMICI repository root
+    - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV
+      shell: bash
+
+    # BioNetGen path
+    - run: echo "BNGPATH=${AMICI_DIR}/ThirdParty/BioNetGen-2.7.0" >> $GITHUB_ENV
+      shell: bash
+
+    # CMake hints
+    # Ensure CMake is using the python version that we will use for the python tests later on
+    - run: echo "PYTHON_EXECUTABLE=${Python3_ROOT_DIR}/bin/python3" >> $GITHUB_ENV
+      shell: bash
+    - run: echo "OpenMP_ROOT=$(brew --prefix)/opt/libomp" >> $GITHUB_ENV
+      shell: bash
+    - run: echo "BOOST_ROOT=$(brew --prefix)/opt/boost" >> $GITHUB_ENV
+      shell: bash
+
+    # install amici dependencies
+    - name: homebrew
+      run: brew install hdf5 swig gcc libomp boost
+      shell: bash
diff --git a/.github/workflows/test_python_cplusplus.yml b/.github/workflows/test_python_cplusplus.yml
@@ -218,8 +218,8 @@ jobs:
 
     # TODO: Include notebooks in coverage report
 
-  osx:
-    name: Tests OSX
+  macos_cpp_py:
+    name: Tests MacOS C++/Python
     runs-on: macos-latest
 
     steps:
@@ -231,16 +231,11 @@ jobs:
     - uses: actions/checkout@v3
     - run: git fetch --prune --unshallow
 
-    - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV
-    - run: echo "BNGPATH=${AMICI_DIR}/ThirdParty/BioNetGen-2.7.0" >> $GITHUB_ENV
-    # Ensure CMake is using the python version that we will use for the python tests later on
-    - run: echo "PYTHON_EXECUTABLE=${Python3_ROOT_DIR}/bin/python3" >> $GITHUB_ENV
-    - run: echo "OpenMP_ROOT=$(brew --prefix)/opt/libomp" >> $GITHUB_ENV
-    - run: echo "BOOST_ROOT=$(brew --prefix)/opt/boost" >> $GITHUB_ENV
+    - name: Install dependencies
+      uses: ./.github/actions/install-macos-dependencies
 
-    # install amici dependencies
     - name: homebrew
-      run: brew install hdf5 swig gcc cppcheck libomp boost
+      run: brew install cppcheck
 
     - name: Build AMICI
       run: scripts/buildAll.sh
@@ -254,8 +249,47 @@ jobs:
     - name: cppcheck
       run: scripts/run-cppcheck.sh
 
-    - name: Python tests
-      run: scripts/run-python-tests.sh
-
     - name: C++ tests
       run: scripts/run-cpp-tests.sh
+
+    - name: Python tests
+      run: |
+        scripts/run-python-tests.sh \
+          test_pregenerated_models.py \
+          test_splines_short.py \
+          test_misc.py
+
+
+  macos_python:
+    name: Tests MacOS Python
+    runs-on: macos-latest
+
+    steps:
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+
+    - uses: actions/checkout@v3
+    - run: git fetch --prune --unshallow
+
+    - name: Install dependencies
+      uses: ./.github/actions/install-macos-dependencies
+
+    - name: Install python package
+      run: |
+        pip show numpy > /dev/null || python3 -m pip install numpy
+        scripts/installAmiciSource.sh
+
+    - name: Check OpenMP support
+      run: source build/venv/bin/activate && python -c "import amici; import sys; sys.exit(not amici.compiledWithOpenMP())"
+
+    - name: Get BioNetGen
+      run: scripts/buildBNGL.sh
+
+    - name: Python tests
+      run: |
+        scripts/run-python-tests.sh \
+          --ignore=test_pregenerated_models.py \
+          --ignore=test_splines_short.py \
+          --ignore=test_misc.py
diff --git a/python/sdist/setup.cfg b/python/sdist/setup.cfg
@@ -34,7 +34,6 @@ install_requires =
     numpy>=1.23.2; python_version=='3.11'
     numpy; python_version>='3.12'
     python-libsbml
-    h5py
     pandas>=2.0.2
     wurlitzer
     toposort
@@ -47,6 +46,7 @@ zip_safe = False
 petab = petab>=0.2.1
 pysb = pysb>=1.13.1
 test =
+    h5py
     pytest
     pytest-cov
     pytest-rerunfailures
@@ -56,9 +56,13 @@ test =
     # see https://github.com/sys-bio/antimony/issues/92
     #  unsupported x86_64 / x86_64h
     antimony!=2.14; platform_system=='Darwin' and platform_machine in 'x86_64h'
+    scipy
 vis =
     matplotlib
     seaborn
+examples =
+    jupyter
+    scipy
 
 [options.package_data]
 amici =

diff --git a/scripts/installAmiciSource.sh b/scripts/installAmiciSource.sh
@@ -1,35 +1,37 @@
 #!/bin/bash
-#
-# Build libamici
-#
+# Create a virtual environment and perform an editable amici installation
 set -e
 
 SCRIPT_PATH=$(dirname $BASH_SOURCE)
-AMICI_PATH=$(cd $SCRIPT_PATH/.. && pwd)
+AMICI_PATH=$(cd "$SCRIPT_PATH/.." && pwd)
 
+venv_dir="${AMICI_PATH}/build/venv"
 # Disabled until cmake package is made compatible with updated setup.py
 #make python-wheel
 #pip3 install --user --prefix= `ls -t ${AMICI_PATH}/build/python/amici-*.whl | head -1`
 
 # test install from setup.py
 set +e
-python3 -m venv ${AMICI_PATH}/build/venv --clear
+mkdir -p "${venv_dir}"
+python3 -m venv "${venv_dir}" --clear
 # in case this fails (usually due to missing ensurepip, try getting pip
 # manually
 if [[ $? ]]; then
     set -e
-    python3 -m venv ${AMICI_PATH}/build/venv --clear --without-pip
-    source ${AMICI_PATH}/build/venv/bin/activate
-    curl https://bootstrap.pypa.io/get-pip.py -o ${AMICI_PATH}/build/get-pip.py
-    python3 ${AMICI_PATH}/build/get-pip.py
+    python3 -m venv "${venv_dir}" --clear --without-pip
+    source "${venv_dir}/bin/activate"
+    get_pip=${AMICI_PATH}/build/get-pip.py
+    curl "https://bootstrap.pypa.io/get-pip.py" -o "${get_pip}"
+    python3 "${get_pip}"
+    rm "${get_pip}"
 else
     set -e
-    source ${AMICI_PATH}/build/venv/bin/activate
+    source "${venv_dir}/bin/activate"
 fi
 
-pip install --upgrade pip wheel
-pip install --upgrade pip scipy matplotlib coverage pytest \
-  pytest-cov cmake_build_extension numpy
-pip install git+https://github.com/FFroehlich/pysb@fix_pattern_matching # pin to PR for SPM with compartments
-AMICI_BUILD_TEMP="${AMICI_PATH}/python/sdist/build/temp" pip install --verbose -e ${AMICI_PATH}/python/sdist[petab,test,vis] --no-build-isolation
+python -m pip install --upgrade pip wheel
+python -m pip install --upgrade pip setuptools cmake_build_extension numpy
+python -m pip install git+https://github.com/FFroehlich/pysb@fix_pattern_matching # pin to PR for SPM with compartments
+AMICI_BUILD_TEMP="${AMICI_PATH}/python/sdist/build/temp" \
+  python -m pip install --verbose -e "${AMICI_PATH}/python/sdist[petab,test,vis]" --no-build-isolation
 deactivate
diff --git a/scripts/run-python-tests.sh b/scripts/run-python-tests.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
-# Test python model wrapping inside virtual environment
+# Run Python test suite inside virtual environment
+# Usage: ./run-python-tests.sh [additional pytest arguments]
 
-script_path=$(dirname $BASH_SOURCE)
+script_path=$(dirname "${BASH_SOURCE[0]}")
 amici_path=$(cd "$script_path"/.. && pwd)
 
 set -e
@@ -12,7 +13,10 @@ fi
 
 cd "${amici_path}"/python/tests
 source "${amici_path}"/build/venv/bin/activate
-pip install scipy h5py pytest pytest-cov
 
 # PEtab tests are run separately
-pytest --ignore-glob=*petab* --ignore-glob=*test_splines.py
+pytest \
+  --ignore-glob=*petab* \
+  --ignore-glob=*test_splines.py \
+  --durations=10 \
+  $@
diff --git a/scripts/run-valgrind-py.sh b/scripts/run-valgrind-py.sh
@@ -2,7 +2,7 @@
 # Without arguments: run Python test suite under valgrind
 # With arguments: run whatever was passed as arguments under valgrind
 
-script_path=$(dirname $BASH_SOURCE)
+script_path=$(dirname "${BASH_SOURCE[0]}")
 amici_path=$(cd "$script_path"/.. && pwd)
 
 set -e
@@ -16,7 +16,6 @@ if [ $# -eq 0 ]
     # No arguments supplied, run all tests
     cd "${amici_path}"/python/tests
     source "${amici_path}"/build/venv/bin/activate
-    pip install scipy h5py pytest pytest-rerunfailures
     command=(python -m pytest -vv --ignore-glob=*petab* -W 'ignore:Signature ')
     #                                                    ^ ignores the following warning that occurs only under valgrind,
     # e.g. `valgrind python -c "import h5py"`: