Release 0.4.8

Merge pull request #362 from ICB-DCM/develop

* Updated AMICI to v0.11.19 (#357)
* Documentation of mini-batch optimizers (#355)
* Docker image now comes with fides optimizer (#356)
* Docker image can now be imported by singularity (#289)
* Added option to specify maximum number of threads for multi-start optimization (#361)

New dependency: Boost.Thread

.github/workflows/benchmark_models.yml

@@ -5,7 +5,7 @@ on: [push, pull_request, workflow_dispatch]
 jobs:
   container:
     runs-on: ubuntu-latest
-    container: dweindl/parpeci:1902967
+    container: ghcr.io/icb-dcm/custom_ci_image:master
     name: benchmark models
     steps:
     - uses: actions/checkout@master

.github/workflows/parpe_tests.yml

@@ -5,7 +5,7 @@ on: [push, pull_request, workflow_dispatch]
 jobs:
   container:
     runs-on: ubuntu-latest
-    container: dweindl/parpeci:1902967
+    container: ghcr.io/icb-dcm/custom_ci_image:master
     name: parPE tests
     env:
       # mpiexec prefix for running tests
@@ -22,7 +22,7 @@ jobs:
     - run: echo "AMICI_PATH=${PARPE_BASE}/deps/AMICI/" >> $GITHUB_ENV
 
     # sonar cloud
-    - run: echo "SONAR_SCANNER_VERSION=4.6.0.2311" >> $GITHUB_ENV
+    - run: echo "SONAR_SCANNER_VERSION=4.6.1.2450" >> $GITHUB_ENV
     - run: echo "SONAR_SCANNER_HOME=/root/.sonar/sonar-scanner-$SONAR_SCANNER_VERSION-linux" >> $GITHUB_ENV
     - run: echo "SONAR_SCANNER_OPTS=-server" >> $GITHUB_ENV
     - run: echo "${SONAR_SCANNER_HOME}/bin" >> $GITHUB_PATH
@@ -37,13 +37,9 @@ jobs:
       run: |
         cd $AMICI_PATH \
           && scripts/buildSuiteSparse.sh \
-          && scripts/buildSundials.sh \
-          && scripts/buildCpputest.sh #&& scripts/buildAmici.sh
+          && scripts/buildSundials.sh
 
     - name: Install AMICI
-      # with tests:
-      # CPPUTEST_BUILD_DIR=${AMICI_PATH}/ThirdParty/cpputest-master/build/
-      # -DCppUTest_DIR=${CPPUTEST_BUILD_DIR}
       run: |
         cmake \
           -S ${AMICI_PATH} \

.github/workflows/petab_testsuite.yml

@@ -6,7 +6,7 @@ on: [push, pull_request, workflow_dispatch]
 jobs:
   container:
     runs-on: ubuntu-latest
-    container: dweindl/parpeci:1902967
+    container: ghcr.io/icb-dcm/custom_ci_image:master
     name: PEtab test suite
     steps:
     - uses: actions/checkout@master

CMakeLists.txt

@@ -8,13 +8,12 @@ endif(POLICY CMP0074)
 
 project(parpe)
 
-set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/CMakeModules/")
+set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/")
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 
 include(BuildType) # Ensure CMAKE_BUILD_TYPE is always set
-include(BuildOptimized)
 include(CTest)
 
 set(CMAKE_DEBUG_POSTFIX "-dbg")
@@ -36,7 +35,7 @@ add_custom_target(get_version ALL
     COMMAND "${CMAKE_COMMAND}"
     "-DINFILE=${PROJECT_SOURCE_DIR}/src/parpecommon/parpeVersion.h.in"
     "-DOUTFILE=${CMAKE_BINARY_DIR}/src/parpecommon/parpeVersion.h"
-    "-P" "${PROJECT_SOURCE_DIR}/CMakeModules/ConfigureVersion.cmake")
+    "-P" "${PROJECT_SOURCE_DIR}/cmake/ConfigureVersion.cmake")
 
 include("getVersion")
 include("split_version")
@@ -59,6 +58,12 @@ set(PARPE_ENABLE_TOMS611 FALSE CACHE BOOL "Enable toms611 optimizer?")
 set(PARPE_ENABLE_FSQP FALSE CACHE BOOL "Enable FSQP optimizer?")
 set(PARPE_EXPORT_PACKAGE FALSE CACHE BOOL
     "Export this build to CMake registry?")
+set(PARPE_BUILD_OPTIMIZED TRUE CACHE BOOL "Optimize for current CPU?")
+
+if(${PARPE_BUILD_OPTIMIZED})
+    include(BuildOptimized)
+endif(${PARPE_BUILD_OPTIMIZED})
+
 
 # OpenMP?
 find_package(OpenMP)
@@ -78,7 +83,7 @@ pkg_search_module(IPOPT REQUIRED IMPORTED_TARGET GLOBAL ipopt>=3.11.0 )
 
 if(${PARPE_ENABLE_DLIB})
     set(DLIB_DIR "${CMAKE_CURRENT_LIST_DIR}/ThirdParty/dlib-19.7"
-        CACHE FILEPATH "DLIB base directory")
+        CACHE PATH "DLIB base directory")
 endif(${PARPE_ENABLE_DLIB})
 
 # PThreads
@@ -229,7 +234,7 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/ParPETargets.cmake"
 set(LIB_INSTALL_DIR lib)
 include(CMakePackageConfigHelpers)
 configure_package_config_file(
-    "CMakeModules/ParPEConfig.cmake.in"
+    "cmake/ParPEConfig.cmake.in"
     "${CMAKE_CURRENT_BINARY_DIR}/ParPEConfig.cmake"
     INSTALL_DESTINATION "${LIB_INSTALL_DIR}/cmake/")
 

README.md

@@ -55,6 +55,7 @@ For full functionality, parPE requires the following libraries:
 * IPOPT (>= 1.2.7) (requires coinhsl)
 * CERES (>=1.13)
   ([requires Eigen](http://ceres-solver.org/installation.html#dependencies))
+* [Boost](https://www.boost.org/) (serialization, thread)
 * HDF5 (>= 1.10)
 * CBLAS compatible BLAS (libcblas, Intel MKL, ...)
 * [AMICI](https://github.com/ICB-DCM/AMICI) (included in this repository)
@@ -64,10 +65,21 @@ For full functionality, parPE requires the following libraries:
 
 On Debian-based systems, dependencies can be installed via:
 ```shell
-sudo apt-get install build-essential cmake cmake-curses-gui \
-    coinor-libipopt-dev curl gfortran \
-    libblas-dev libboost-serialization-dev libceres-dev libcpputest-dev \
-    libmpich-dev libhdf5-dev libpython3-dev python3-pip
+sudo apt-get install \
+  build-essential \
+  cmake \
+  cmake-curses-gui \
+  coinor-libipopt-dev \
+  curl \
+  gfortran \
+  libblas-dev \
+  libboost-serialization-dev \
+  libboost-thread-dev \
+  libceres-dev \
+  libmpich-dev \
+  libhdf5-dev \
+  libpython3-dev \
+  python3-pip
 ```
 
 Scripts to fetch and build the remaining dependencies are provided in

ThirdParty/installFides.sh

@@ -21,7 +21,7 @@ build_fides() {
     cd "${script_dir}"
     git clone https://github.com/dweindl/fides-cpp.git
     cd "${fides_dir}"
-    git checkout 9906bdac6a1966ddd4b37b96f98ad8f89770c128
+    git checkout 76e1ca57674a20a2821a6ae4987b85035bbad016
   fi
 
   cd "${fides_dir}"

buildSuperMUC.sh

@@ -9,17 +9,6 @@
 #   result in "ld: cannot find -lpkgcfg_lib_IPOPT_iomp5-NOTFOUND"
 #   -> use newer CMake
 
-build_cpputest() {
-  #cpputest
-  #CPPUTEST_PATH=${amici_path}/ThirdParty/cpputest-master
-  #cd ${CPPUTEST_PATH}
-  # -DC++11=ON breaks compilation of some `_override` for no obvious reason
-  #cmake -DC++11=OFF -DCMAKE_INSTALL_PREFIX=`pwd`
-  #make ${make_opts}
-  #make install
-  :
-}
-
 build_amici() {
   amici_path=${parpe_root}/deps/AMICI
 
@@ -69,7 +58,6 @@ build_3rd_party_deps() {
   # build_boost
   "${parpe_root}/ThirdParty/installIpopt.sh"
   #./installCeres.sh
-  #./installCpputest.sh
   #  ceres_base=${parpe_root}/ThirdParty/ceres-solver-1.13.0/
   #  ceres_install_dir=${ceres_base}/build/install/
   #  if [[ -d ${ceres_base} ]]; then
@@ -108,7 +96,6 @@ parpe_root=$(cd "${parpe_root}" && pwd)
 
 make_opts=${MAKEOPTS--j12}
 
-build_cpputest
 build_3rd_party_deps
 build_amici
 build_parpe

CMakeModules/ParPEConfig.cmake.in → cmake/ParPEConfig.cmake.in

@@ -19,13 +19,13 @@ endif()
 
 if(${PARPE_ENABLE_CERES})
 find_dependency(Ceres COMPONENTS
-    HINTS "${CMAKE_SOURCE_DIR}/ThirdParty/ceres-solver-2.0.0/build/install")
+    HINTS "@Ceres_DIR@")
 find_dependency(Eigen3 REQUIRED)
 endif()
 
 if(${PARPE_ENABLE_FIDES})
 find_dependency(Fides
-    HINTS "${CMAKE_SOURCE_DIR}/ThirdParty/fides-cpp/build/")
+    HINTS "@Fides_DIR@")
 endif()
 
 find_dependency(Amici REQUIRED)

container/charliecloud/parpe_base/Dockerfile

@@ -7,3 +7,7 @@ RUN /u18/install.sh && rm -rf /tmp && mkdir /tmp
 ENV BASH_ENV "/etc/drydock/.env"
 
 RUN /u18/install_parpe.sh
+
+ENV PARPE_DIR "/parPE"
+
+RUN chmod -R ugo+rwX $PARPE_DIR

container/charliecloud/parpe_base/install.sh

@@ -31,6 +31,7 @@ apt-get install -q -y \
   python3-dev \
   python3-pip \
   python3-venv \
+  libspdlog-dev \
   swig3.0 \
   unzip \
   wget

container/charliecloud/parpe_base/install_parpe.sh

@@ -3,35 +3,31 @@
 set -euo pipefail
 set -x
 
-cd
+export PARPE_BASE="${PARPE_DIR:-/parPE}"
 
 # unpack git archive
-mkdir parPE && cd parPE
+mkdir "$PARPE_BASE" && cd "$PARPE_BASE"
 tar -xzf /u18/parpe.tar.gz
 
-export PARPE_BASE=$(pwd)
-
 # Build dependencies
 
 # Install AMICI
 export AMICI_PATH=${PARPE_BASE}/deps/AMICI/
 cd "${AMICI_PATH}" \
   && scripts/buildSuiteSparse.sh \
-  && scripts/buildSundials.sh \
-  && scripts/buildCpputest.sh #&& scripts/buildAmici.sh
-mkdir -p "${AMICI_PATH}"/build && cd "${AMICI_PATH}"/build
-CPPUTEST_BUILD_DIR=${AMICI_PATH}/ThirdParty/cpputest-master/build/
+  && scripts/buildSundials.sh
+mkdir -p "${AMICI_PATH}/build" && cd "${AMICI_PATH}/build"
 cmake \
   -DCMAKE_BUILD_TYPE=Debug \
   -DENABLE_PYTHON=ON \
   -DBUILD_TESTS=OFF \
-  -DCppUTest_DIR="${CPPUTEST_BUILD_DIR}" \
   .. && make -j12
 
-#- cd $PARPE_BASE/ThirdParty && ./installCeres.sh
+# install fides optimizer
+cd "$PARPE_BASE/ThirdParty" && ./installFides.sh
 
 # install parPE python requirements
-pip install -r "${PARPE_BASE}"/python/requirements.txt
+pip3 install -r "${PARPE_BASE}"/python/requirements.txt
 
 # build parPE
 cd "${PARPE_BASE}"
@@ -50,12 +46,15 @@ mpi_cmd="$mpi_cmd;--mca;btl_tcp_if_include;lo;"
 mpi_cmd="$mpi_cmd;--mca;orte_base_help_aggregate;0"
 
 CC=mpicc CXX=mpiCC cmake \
+      -DPARPE_BUILD_OPTIMIZED=OFF \
+      -DPARPE_ENABLE_FIDES=ON \
       -DIPOPT_INCLUDE_DIRS=/usr/include/coin/ \
       -DIPOPT_LIBRARIES=/usr/lib/libipopt.so \
       -DMPI_INCLUDE_DIRS=/usr/include/openmpi-x86_64/ \
       -DBUILD_TESTING=ON \
       -DTESTS_MPIEXEC_COMMAND="$mpi_cmd" \
       ..
+
 make -j12 VERBOSE=1
 
 # MPI settings for python tests

container/singularity/README.md

@@ -0,0 +1,44 @@
+# Using parPE with singularity
+
+## Singularity
+
+> Singularity is an open source container platform designed to be simple, fast,
+> and secure. Singularity is optimized for compute focused enterprise and HPC
+>workloads, allowing untrusted users to run untrusted containers in a trusted
+>way.
+
+--- [https://github.com/hpcng/singularity](https://github.com/hpcng/singularity)
+
+Documentation: [https://sylabs.io/guides/3.0/user-guide/index.html](https://sylabs.io/guides/3.0/user-guide/index.html)
+
+
+## Using parPE with singularity
+
+Singularity images can be created from available docker containers using:
+
+```
+singularity pull docker://dweindl/parpe:develop
+```
+
+To create a custom docker containers, see 
+https://parpe.readthedocs.io/en/latest/parpe_with_charliecloud.html#generating-parpe-base-docker-image
+
+
+An example for parameter estimation for the model at https://github.com/Benchmarking-Initiative/Benchmark-Models-PEtab/tree/master/Benchmark-Models/Zheng_PNAS2012 :
+
+```shell
+AMICI_MODEL_DIR=Zheng_PNAS2012
+PARPE_MODEL_DIR=Zheng_PNAS2012_parpe
+PETAB_YAML_FILE=Benchmark-Models-PEtab/Benchmark-Models/Zheng_PNAS2012/Zheng_PNAS2012.yaml
+MODEL_NAME=Zheng_PNAS2012
+H5_PE_INPUT=$PARPE_MODEL_DIR/data.h5
+
+singularity exec parpe_develop.sif /parPE/misc/run_in_venv.sh /parPE/build/venv amici_import_petab -v -y $PETAB_YAML_FILE &
+singularity exec parpe_develop.sif /parPE/misc/setup_amici_model.sh ${AMICI_MODEL_DIR} ${PARPE_MODEL_DIR}
+singularity exec parpe_develop.sif /parPE/misc/run_in_venv.sh /parPE/build/venv parpe_petab_to_hdf5 \
+    -n ${MODEL_NAME} \
+    -y ${PETAB_YAML_FILE} \
+    -d ${AMICI_MODEL_DIR} \
+    -o ${H5_PE_INPUT}
+ singularity exec parpe_develop.sif ${PARPE_MODEL_DIR}/build/estimate_$MODEL_NAME -o ${MODEL_NAME}_results/ $H5_PE_INPUT
+```

deps/AMICI/.github/workflows/test_install.yml

@@ -33,10 +33,6 @@ jobs:
       run: |
         scripts/buildSundials.sh
 
-    - name: Build cpputest
-      run: |
-        scripts/buildCpputest.sh
-
     - name: Build AMICI
       run: |
         scripts/buildAmici.sh

deps/AMICI/.github/workflows/test_petab_test_suite.yml

@@ -44,7 +44,7 @@ jobs:
       - run: |
           echo "${HOME}/.local/bin/" >> $GITHUB_PATH
           echo "${GITHUB_WORKSPACE}/tests/performance/" >> $GITHUB_PATH
-          echo "BNGPATH=${GITHUB_WORKSPACE}/ThirdParty/BioNetGen-2.5.2" >> $GITHUB_ENV
+          echo "BNGPATH=${GITHUB_WORKSPACE}/ThirdParty/BioNetGen-2.7.0" >> $GITHUB_ENV
 
       # install AMICI
       - name: Install python package

deps/AMICI/.github/workflows/test_python_cplusplus.yml

@@ -17,7 +17,7 @@ jobs:
     - run: git fetch --prune --unshallow
 
     - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV
-    - run: echo "BNGPATH=${GITHUB_WORKSPACE}/ThirdParty/BioNetGen-2.5.2" >> $GITHUB_ENV
+    - run: echo "BNGPATH=${GITHUB_WORKSPACE}/ThirdParty/BioNetGen-2.7.0" >> $GITHUB_ENV
 
     # sonar cloud
     - run: echo "SONAR_SCANNER_VERSION=4.5.0.2216" >> $GITHUB_ENV
@@ -69,7 +69,7 @@ jobs:
 
     - name: C++ tests
       run: |
-        scripts/run-cpputest.sh
+        scripts/run-cpp-tests.sh
 
     - name: Install python package
       run: |
@@ -85,7 +85,7 @@ jobs:
           --cov-report=xml:"${AMICI_DIR}/build/coverage_py.xml" \
           --cov-append \
           ${AMICI_DIR}/python/tests
-          
+
     - name: example notebooks
       run: |
         scripts/runNotebook.sh python/examples/example_*/
@@ -96,7 +96,7 @@ jobs:
 
     - name: Codecov Python
       uses: codecov/codecov-action@v1
-      with: 
+      with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: ./build/coverage_py.xml
         flags: python
@@ -138,7 +138,7 @@ jobs:
     - run: git fetch --prune --unshallow
 
     - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV
-    - run: echo "BNGPATH=${AMICI_DIR}/ThirdParty/BioNetGen-2.5.2" >> $GITHUB_ENV
+    - run: echo "BNGPATH=${AMICI_DIR}/ThirdParty/BioNetGen-2.7.0" >> $GITHUB_ENV
 
     # install amici dependencies
     - name: homebrew
@@ -163,4 +163,4 @@ jobs:
 
     - name: C++ tests
       run: |
-        scripts/run-cpputest.sh
+        scripts/run-cpp-tests.sh

deps/AMICI/.github/workflows/test_python_ver_matrix.yml

@@ -31,7 +31,7 @@ jobs:
 
     steps:
     - run: echo "AMICI_DIR=$(pwd)" >> $GITHUB_ENV
-    - run: echo "BNGPATH=${AMICI_DIR}/ThirdParty/BioNetGen-2.5.2" >> $GITHUB_ENV
+    - run: echo "BNGPATH=${AMICI_DIR}/ThirdParty/BioNetGen-2.7.0" >> $GITHUB_ENV
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2