diff --git a/.github/actions/setup-amici-cpp/action.yml b/.github/actions/setup-amici-cpp/action.yml new file mode 100644 index 0000000000..09ec9311bf --- /dev/null +++ b/.github/actions/setup-amici-cpp/action.yml @@ -0,0 +1,44 @@ +name: Set up AMICI C++ +description: | + Build the AMICI C++ interface and set things for for coverage analysis. + (Currently ubuntu-only). + +runs: + using: "composite" + steps: + # BioNetGen Path + - run: echo "BNGPATH=${GITHUB_WORKSPACE}/ThirdParty/BioNetGen-2.7.0" >> $GITHUB_ENV + shell: bash + + # use all available cores + - run: echo "AMICI_PARALLEL_COMPILE=" >> $GITHUB_ENV + shell: bash + + # enable coverage + - run: echo "ENABLE_GCOV_COVERAGE=TRUE" >> $GITHUB_ENV + shell: bash + + - name: Set up Sonar tools + uses: ./.github/actions/setup-sonar-tools + + - name: Install apt dependencies + uses: ./.github/actions/install-apt-dependencies + + - name: Install additional apt dependencies + run: | + sudo apt-get update \ + && sudo apt-get install -y \ + cmake \ + python3-venv \ + lcov + shell: bash + + - name: Build AMICI dependencies + run: scripts/buildDependencies.sh + shell: bash + + - name: Build AMICI + run: scripts/buildAmici.sh + shell: bash + env: + CI_SONARCLOUD: "TRUE" diff --git a/.github/workflows/test_python_cplusplus.yml b/.github/workflows/test_python_cplusplus.yml index acc8759a3c..a531d2db2c 100644 --- a/.github/workflows/test_python_cplusplus.yml +++ b/.github/workflows/test_python_cplusplus.yml @@ -8,20 +8,13 @@ on: - master jobs: - build: - name: Tests Ubuntu - - # TODO: prepare image with more deps preinstalled + ubuntu-cpp-python-tests: + name: C++/Python tests Ubuntu runs-on: ubuntu-22.04 - env: - AMICI_PARALLEL_COMPILE: "" - ENABLE_GCOV_COVERAGE: "TRUE" - CI_SONARCLOUD: "TRUE" - strategy: matrix: - python-version: [ 3.9 ] + python-version: [ "3.9" ] steps: - name: Set up Python ${{ matrix.python-version }} @@ -33,28 +26,9 @@ jobs: - run: git fetch --prune --unshallow - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV - - run: echo "BNGPATH=${GITHUB_WORKSPACE}/ThirdParty/BioNetGen-2.7.0" >> $GITHUB_ENV - - - name: Set up Sonar tools - uses: ./.github/actions/setup-sonar-tools - - - name: Install apt dependencies - uses: ./.github/actions/install-apt-dependencies - # install amici dependencies - - name: apt - run: | - sudo apt-get update \ - && sudo apt-get install -y \ - cmake \ - python3-venv \ - lcov - - - name: Build AMICI dependencies - run: scripts/buildDependencies.sh - - - name: Build AMICI - run: CI_SONARCLOUD=TRUE scripts/buildAmici.sh + - name: Set up AMICI C++ libraries + uses: ./.github/actions/setup-amici-cpp - name: C++ tests run: scripts/run-cpp-tests.sh @@ -62,19 +36,23 @@ jobs: - name: Install python package run: scripts/installAmiciSource.sh - - name: Python tests + - name: Check OpenMP support + run: source build/venv/bin/activate && python -c "import amici; import sys; sys.exit(not amici.compiledWithOpenMP())" + + - name: Python tests (part 1) run: | source build/venv/bin/activate \ && pytest \ --ignore-glob=*petab* \ --ignore-glob=*test_splines.py \ + --ignore-glob=*test_splines_short.py \ + --ignore-glob=*test_pysb.py \ --cov=amici \ --cov-report=xml:"${AMICI_DIR}/build/coverage_py.xml" \ --cov-append \ --durations=10 \ ${AMICI_DIR}/python/tests - - name: Python tests splines if: ${{ github.base_ref == 'master' || github.event.merge_group.base_ref == 'master'}} run: | @@ -86,16 +64,77 @@ jobs: --durations=10 \ ${AMICI_DIR}/python/tests/test_splines.py - - name: Install notebook dependencies + - name: Codecov Python + uses: codecov/codecov-action@v3.1.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: build/coverage_py.xml + flags: python + fail_ci_if_error: true + verbose: true + + - name: Capture coverage info (lcov) run: | - source build/venv/bin/activate \ - && pip install jax[cpu] + lcov --compat-libtool --no-external \ + -d ${AMICI_DIR}/build/CMakeFiles/amici.dir/src \ + -b ${AMICI_DIR} -c -o coverage_cpp.info \ + && lcov --compat-libtool --no-external \ + -d ${AMICI_DIR}/python/sdist/build/temp_amici/CMakeFiles/amici.dir/src \ + -b ${AMICI_DIR}/python/sdist -c -o coverage_py.info \ + && lcov -a coverage_cpp.info -a coverage_py.info -o coverage.info - - name: example notebooks - run: scripts/runNotebook.sh python/examples/example_*/ + - name: Codecov CPP + uses: codecov/codecov-action@v3.1.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: coverage.info + flags: cpp + fail_ci_if_error: true - - name: doc notebooks - run: scripts/runNotebook.sh documentation/GettingStarted.ipynb + - name: Run sonar-scanner + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + run: | + sonar-scanner \ + -Dsonar.cfamily.build-wrapper-output=bw-output \ + -Dsonar.projectVersion="$(git describe --abbrev=4 --dirty=-dirty --always --tags | tr -d '\n')" + + ubuntu-python-tests: + name: Python tests Ubuntu + runs-on: ubuntu-22.04 + + strategy: + matrix: + python-version: [ "3.9" ] + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - uses: actions/checkout@v3 + - run: git fetch --prune --unshallow + + - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV + + - name: Set up AMICI C++ libraries + uses: ./.github/actions/setup-amici-cpp + + - name: Install python package + run: scripts/installAmiciSource.sh + + - name: Python tests + run: | + source build/venv/bin/activate \ + && pytest \ + --cov=amici \ + --cov-report=xml:"${AMICI_DIR}/build/coverage_py.xml" \ + --cov-append \ + --durations=10 \ + ${AMICI_DIR}/python/tests/test_pysb.py \ + ${AMICI_DIR}/python/tests/test_splines_short.py - name: Codecov Python uses: codecov/codecov-action@v3.1.0 @@ -106,7 +145,7 @@ jobs: fail_ci_if_error: true verbose: true - - name: lcov + - name: Capture coverage info (lcov) run: | lcov --compat-libtool --no-external \ -d ${AMICI_DIR}/build/CMakeFiles/amici.dir/src \ @@ -121,7 +160,7 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: coverage.info - flags: cpp + flags: cpp_python fail_ci_if_error: true - name: Run sonar-scanner @@ -133,6 +172,45 @@ jobs: -Dsonar.cfamily.build-wrapper-output=bw-output \ -Dsonar.projectVersion="$(git describe --abbrev=4 --dirty=-dirty --always --tags | tr -d '\n')" + + ubuntu-notebook-tests: + name: Notebook tests Ubuntu + runs-on: ubuntu-22.04 + + strategy: + matrix: + python-version: [ "3.9" ] + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - uses: actions/checkout@v3 + - run: git fetch --prune --unshallow + + - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV + + - name: Set up AMICI C++ libraries + uses: ./.github/actions/setup-amici-cpp + + - name: Install python package + run: scripts/installAmiciSource.sh + + - name: Install notebook dependencies + run: | + source build/venv/bin/activate \ + && pip install jax[cpu] + + - name: example notebooks + run: scripts/runNotebook.sh python/examples/example_*/ + + - name: doc notebooks + run: scripts/runNotebook.sh documentation/GettingStarted.ipynb + + # TODO: Include notebooks in coverage report + osx: name: Tests OSX runs-on: macos-latest @@ -150,15 +228,12 @@ jobs: - run: echo "BNGPATH=${AMICI_DIR}/ThirdParty/BioNetGen-2.7.0" >> $GITHUB_ENV # Ensure CMake is using the python version that we will use for the python tests later on - run: echo "PYTHON_EXECUTABLE=${Python3_ROOT_DIR}/bin/python3" >> $GITHUB_ENV + - run: echo "OpenMP_ROOT=$(brew --prefix)/opt/libomp" >> $GITHUB_ENV + - run: echo "BOOST_ROOT=$(brew --prefix)/opt/boost" >> $GITHUB_ENV # install amici dependencies - name: homebrew - run: | - brew install hdf5 swig gcc cppcheck libomp boost \ - && brew ls -v boost \ - && brew ls -v libomp \ - && echo LDFLAGS="-L/usr/local/lib/ -L/usr/local/Cellar/boost/1.81.0_1/lib/" >> $GITHUB_ENV \ - && echo CPPFLAGS="-I /usr/local/Cellar/boost/1.81.0_1/include/" >> $GITHUB_ENV + run: brew install hdf5 swig gcc cppcheck libomp boost - name: Build AMICI run: scripts/buildAll.sh @@ -166,6 +241,9 @@ jobs: - name: Install python package run: scripts/installAmiciSource.sh + - name: Check OpenMP support + run: source build/venv/bin/activate && python -c "import amici; import sys; sys.exit(not amici.compiledWithOpenMP())" + - name: cppcheck run: scripts/run-cppcheck.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index aa46196e0e..4b511d92d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,46 @@ ## v0.X Series +### v0.20.0 (2023-11-23) + +**Fixes** + +* Fixed CMake cmake_minimum_required deprecation warning + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2183 +* Fixed misleading preequilibration failure messages + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2181 +* Removed setuptools<64 restriction + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2180 +* Fixed ExpData equality operator for Python + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2194 +* Enabled deepcopy for ExpData(View) + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2196 +* Allowed subsetting simulation conditions in simulate_petab + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2199 +* Set CMake CMP0144 to prevent warning + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2209 + +**Features** + +* Possibility to evaluate and plot symbolic expressions based on simulation results + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2152 +* Easier access to timepoints via ExpDataView + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2193 +* Nicer `__repr__` for ReturnDataView + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2192 + +**Documentation** + +* Added installation instructions for Arch Linux + by @stephanmg in https://github.com/AMICI-dev/AMICI/pull/2173 +* Updated reference list + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2172 +* Installation guide: optional requirements + by @dweindl in https://github.com/AMICI-dev/AMICI/pull/2207 + +**Full Changelog**: https://github.com/AMICI-dev/AMICI/compare/v0.19.0...v0.20.0 + + ### v0.19.0 (2023-08-26) **Features** diff --git a/CMakeLists.txt b/CMakeLists.txt index a31104b47e..d6f08a5097 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,11 @@ cmake_minimum_required(VERSION 3.15) cmake_policy(VERSION 3.15...3.27) +# cmake >=3.27 +if(POLICY CMP0144) + cmake_policy(SET CMP0144 NEW) +endif(POLICY CMP0144) + project(amici) # misc options diff --git a/documentation/amici_refs.bib b/documentation/amici_refs.bib index 8730883075..4c31869d87 100644 --- a/documentation/amici_refs.bib +++ b/documentation/amici_refs.bib @@ -1011,19 +1011,6 @@ @Article{MassonisVil2022 url = {https://doi.org/10.1093/bioinformatics/btac755}, } -@Article{RaimundezFed2022, - author = {Raimundez, Elba and Fedders, Michael and Hasenauer, Jan}, - journal = {bioRxiv}, - title = {Posterior marginalization accelerates Bayesian inference for dynamical systems}, - year = {2022}, - abstract = {Bayesian inference is an important method in the life and natural sciences for learning from data. It provides information about parameter uncertainties, and thereby the reliability of models and their predictions. Yet, generating representative samples from the Bayesian posterior distribution is often computationally challenging. Here, we present an approach that lowers the computational complexity of sample generation for problems with scaling, offset and noise parameters. The proposed method is based on the marginalization of the posterior distribution, which reduces the dimensionality of the sampling problem. We provide analytical results for a broad class of problems and show that the method is suitable for a large number of applications. Subsequently, we demonstrate the benefit of the approach for various application examples from the field of systems biology. We report a substantial improvement up to 50 times in the effective sample size per unit of time, in particular when applied to multi-modal posterior problems. As the scheme is broadly applicable, it will facilitate Bayesian inference in different research fields.Competing Interest StatementThe authors have declared no competing interest.}, - doi = {10.1101/2022.12.02.518841}, - elocation-id = {2022.12.02.518841}, - eprint = {https://www.biorxiv.org/content/early/2022/12/03/2022.12.02.518841.full.pdf}, - publisher = {Cold Spring Harbor Laboratory}, - url = {https://www.biorxiv.org/content/early/2022/12/03/2022.12.02.518841}, -} - @Article{AlbadryHoe2022, author = {Albadry, Mohamed and Höpfl, Sebastian and Ehteshamzad, Nadia and König, Matthias and Böttcher, Michael and Neumann, Jasna and Lupp, Amelie and Dirsch, Olaf and Radde, Nicole and Christ, Bruno and Christ, Madlen and Schwen, Lars Ole and Laue, Hendrik and Klopfleisch, Robert and Dahmen, Uta}, journal = {Scientific Reports}, @@ -1217,6 +1204,53 @@ @Article{TunedalVio2023 url = {https://physoc.onlinelibrary.wiley.com/doi/abs/10.1113/JP284652}, } +@Unknown{HasenauerMer2023, + author = {Hasenauer, Jan and Merkt, Simon and Ali, Solomon and Gudina, Esayas and Adissu, Wondimagegn and Münchhoff, Maximilian and Graf, Alexander and Krebs, Stefan and Elsbernd, Kira and Kisch, Rebecca and Sirgu, Sisay and Fantahun, Bereket and Bekele, Delayehu and Rubio-Acero, Raquel and Gashaw, Mulatu and Girma, Eyob and Yilma, Daniel and Zeynudin, Ahmed and Paunovic, Ivana and Wieser, Andreas}, + creationdate = {2023-09-19T09:21:01}, + doi = {10.21203/rs.3.rs-3307821/v1}, + modificationdate = {2023-09-19T09:21:01}, + month = {09}, + title = {Long-term monitoring of SARS-CoV-2 seroprevalence and variants in Ethiopia provides prediction for immunity and cross-immunity}, + year = {2023}, +} + +@Article{RaimundezFed2023, + author = {Elba Raim{\'{u}}ndez and Michael Fedders and Jan Hasenauer}, + journal = {{iScience}}, + title = {Posterior marginalization accelerates Bayesian inference for dynamical models of biological processes}, + year = {2023}, + month = {sep}, + pages = {108083}, + creationdate = {2023-10-04T14:12:00}, + doi = {10.1016/j.isci.2023.108083}, + modificationdate = {2023-10-04T14:12:00}, + publisher = {Elsevier {BV}}, +} + +@Article{Mendes2023, + author = {Mendes, Pedro}, + journal = {Frontiers in Cell and Developmental Biology}, + title = {Reproducibility and FAIR principles: the case of a segment polarity network model}, + year = {2023}, + issn = {2296-634X}, + volume = {11}, + abstract = {The issue of reproducibility of computational models and the related FAIR principles (findable, accessible, interoperable, and reusable) are examined in a specific test case. I analyze a computational model of the segment polarity network in Drosophila embryos published in 2000. Despite the high number of citations to this publication, 23 years later the model is barely accessible, and consequently not interoperable. Following the text of the original publication allowed successfully encoding the model for the open source software COPASI. Subsequently saving the model in the SBML format allowed it to be reused in other open source software packages. Submission of this SBML encoding of the model to the BioModels database enables its findability and accessibility. This demonstrates how the FAIR principles can be successfully enabled by using open source software, widely adopted standards, and public repositories, facilitating reproducibility and reuse of computational cell biology models that will outlive the specific software used.}, + creationdate = {2023-10-28T19:05:54}, + doi = {10.3389/fcell.2023.1201673}, + modificationdate = {2023-10-28T19:05:54}, + url = {https://www.frontiersin.org/articles/10.3389/fcell.2023.1201673}, +} + +@Misc{HuckBal2023, + author = {Wilhelm Huck and Mathieu Baltussen and Thijs de Jong and Quentin Duez and William Robinson}, + title = {Chemical reservoir computation in a self-organizing reaction network}, + year = {2023}, + creationdate = {2023-11-18T09:09:45}, + doi = {10.21203/rs.3.rs-3487081/v1}, + modificationdate = {2023-11-18T09:10:08}, + publisher = {Research Square Platform LLC}, +} + @Comment{jabref-meta: databaseType:bibtex;} @Comment{jabref-meta: grouping: diff --git a/documentation/python_installation.rst b/documentation/python_installation.rst index a0fcf0908b..6cc3402e61 100644 --- a/documentation/python_installation.rst +++ b/documentation/python_installation.rst @@ -13,7 +13,7 @@ Installation of the AMICI Python package has the following prerequisites: * CBLAS compatible BLAS library (e.g., OpenBLAS, CBLAS, Atlas, Accelerate, Intel MKL) * a C++17 compatible C++ compiler and a C compiler - (e.g., g++, clang, Intel C++ compiler, mingw) + (e.g., g++>=9.1, clang>=12, Intel C++ compiler, mingw) If these requirements are fulfilled and all relevant paths are setup properly, AMICI can be installed using: @@ -44,6 +44,9 @@ Install the AMICI dependencies via ``apt`` # optionally for HDF5 support: sudo apt install libhdf5-serial-dev + # optionally for boost support (thread-specific CPU times, extended math functions, serialization) + libboost-chrono-dev libboost-math-dev libboost-serialization-dev + Install AMICI: .. code-block:: bash @@ -88,13 +91,13 @@ Alternatively: .. code-block:: bash - sudo pacman -Si python swig openblas gcc hdf5 boost-libs + sudo pacman -Si python swig openblas gcc hdf5 boost-libs 2. Upgrade installed packages if required mininum versions are not satisfied for AMICI installation. .. code-block:: bash - sudo pacman -Su python swig openblas gcc hdf5 boost-libs + sudo pacman -Su python swig openblas gcc hdf5 boost-libs 3. Install AMICI: @@ -117,6 +120,13 @@ Install the AMICI dependencies using homebrew: # optionally for parallel simulations: brew install libomp + # followed by either `brew link openmp` once, + # or `export OpenMP_ROOT=$(brew --prefix)/opt/libomp"` where `OpenMP_ROOT` will have to be set during every re-installation of AMICI or any new model import + + # optionally for boost support (thread-specific CPU times, extended math functions, serialization) + brew install boost && export BOOST_ROOT=$(brew --prefix)/opt/boost + # followed by either `brew link boost` once, + # or `export BOOST_ROOT=$(brew --prefix)/opt/boost"` where `BOOST_ROOT` will have to be set during every re-installation of AMICI or any new model import Install AMICI: diff --git a/documentation/references.md b/documentation/references.md index 3f5833e26f..00c3f40cc8 100644 --- a/documentation/references.md +++ b/documentation/references.md @@ -1,6 +1,6 @@ # References -List of publications using AMICI. Total number is 79. +List of publications using AMICI. Total number is 82. If you applied AMICI in your work and your publication is missing, please let us know via a new GitHub issue. @@ -51,6 +51,20 @@ Rewiring Contribute to Drug Resistance.” Molecular Systems Biology 19 (2): e10988. https://doi.org/10.15252/msb.202210988. +