Fix paths / imports in sphinx configuration (#1196)

* fix paths * fix path * ) * set version / release after installation * Update documentation/conf.py * cblas * .. * .. * custom cblas * .. * .. * .. * .. * .. * .. * .. * .. * .. * .. * ? * ? * ? * mock before import? * ? * ? * .. * .. * .. * .. * .. * .. * .. * .. * .. * ... * ... * 3.8 * prio? * .. * .. * .. * .. * .. * dont use AMICI source installation for run-sphinx * fix fix_typehints and verbose output * readd checks * fixups * fix typo * fix GHA * newer swig version GHA * sphinx with python 3.8 * Update test_doc.yml * uninstall base swig3 for doc tests * fix swig * fix CI not running on forks * Revert "fix CI not running on forks" This reverts commit 6f00ddb. * Update test_doc.yml * fix phython ver, use 3.8 for sphinx * match sphinx script with RTD * Update run-sphinx.sh * Update run-sphinx.sh * update readthedocs configuration * ignore generated? * fixups and revert changes * fix sphinx path GHA, change exclude patterns * keep running GHA in ./documentation * revert exclude and pattern changes * use readthedocs builder * remove hdf5 from sphinx build * Update documentation/conf.py * Don't try to reinstall * should not be required... * it is... * remove extra rtd reqs * .. * remove autogen rtd code, keep ext * why? * try not to run test twice? * test CI * Revert "test CI" This reverts commit b3ce43f. Co-authored-by: Daniel Weindl <[email protected]> Co-authored-by: Daniel Weindl <[email protected]>
AMICI-dev · Aug 20, 2020 · 632f5ed · 632f5ed
1 parent 6dfa1f8
commit 632f5ed
Show file tree

Hide file tree

Showing 12 changed files with 121 additions and 82 deletions.
diff --git a/.github/workflows/deploy_branch.yml b/.github/workflows/deploy_branch.yml
@@ -1,10 +1,5 @@
 name: Deploy Branch
-on:
-    push:
-    pull_request:
-      branches:
-        - master
-        - develop
+on: [push, pull_request]
 
 jobs:
   sdist:
@@ -18,7 +13,7 @@ jobs:
         fetch-depth: 20
 
     - run: echo "::set-env name=AMICI_DIR::$(pwd)"
-    - run: echo ::set-env name=SWIG::${AMICI_DIR}/ThirdParty/swig-4.0.1/install/bin/swig
+    - run: echo "::set-env name=SWIG::${AMICI_DIR}/ThirdParty/swig-4.0.1/install/bin/swig"
 
     - name: Build swig4
       run: |

diff --git a/.github/workflows/deploy_release.yml b/.github/workflows/deploy_release.yml
@@ -51,7 +51,7 @@ jobs:
         fetch-depth: 20
 
     - run: echo "::set-env name=AMICI_DIR::$(pwd)"
-    - run: echo ::set-env name=SWIG::${AMICI_DIR}/ThirdParty/swig-4.0.1/install/bin/swig
+    - run: echo "::set-env name=SWIG::${AMICI_DIR}/ThirdParty/swig-4.0.1/install/bin/swig"
 
     - name: Build swig4
       run: |

diff --git a/.github/workflows/test_doc.yml b/.github/workflows/test_doc.yml
@@ -19,8 +19,6 @@ jobs:
     - uses: actions/checkout@master
     - run: git fetch --prune --unshallow
 
-    - run: echo "::set-env name=AMICI_DIR::$(pwd)"
-
     - name: apt
       run: |
         sudo apt-get update \
@@ -47,6 +45,15 @@ jobs:
       - uses: actions/checkout@master
       - run: git fetch --prune --unshallow
 
+      - run: echo "::set-env name=AMICI_DIR::$(pwd)"
+      - run: echo "::set-env name=SWIG::${AMICI_DIR}/ThirdParty/swig-4.0.1/install/bin/swig"
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          # Semantic version range syntax or exact version of a Python version
+          python-version: '3.8'
+
       # install amici dependencies
       - name: apt
         run: |
@@ -55,13 +62,15 @@ jobs:
               g++ \
               libatlas-base-dev \
               libboost-serialization-dev \
-              libhdf5-serial-dev \
               python3-venv \
-              swig \
-      
-      - name: Install python package
+
+      - name: Build swig
+        run: |
+          sudo scripts/downloadAndBuildSwig.sh
+              
+      - name: pip
         run: |
-          scripts/installAmiciSource.sh
+          pip3 install setuptools
       
       - name: sphinx
         run: |

diff --git a/.github/workflows/test_install.yml b/.github/workflows/test_install.yml
@@ -1,10 +1,5 @@
 name: Installation
-on:
-  push:
-  pull_request:
-    branches:
-      - master
-      - develop
+on: [push, pull_request]
 
 jobs:
   archive:

diff --git a/.github/workflows/test_python_cplusplus.yml b/.github/workflows/test_python_cplusplus.yml
@@ -1,10 +1,5 @@
 name: C++/Python Tests
-on:
-  push:
-  pull_request:
-    branches:
-      - master
-      - develop
+on: [push, pull_request]
 
 jobs:
   build:

diff --git a/.github/workflows/test_python_ver_matrix.yml b/.github/workflows/test_python_ver_matrix.yml
@@ -26,6 +26,11 @@ jobs:
     - run: echo "::set-env name=AMICI_DIR::$(pwd)"
     - run: echo "::set-env name=BNGPATH::${AMICI_DIR}/ThirdParty/BioNetGen-2.3.2"
 
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
     - uses: actions/checkout@v1
       with:
         fetch-depth: 20

diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,7 @@ tests/cpputest/build/*
 tests/cpputest/build_xcode/*
 tests/cpputest/Testing/*
 
+doc-venv/*
 doc/*
 fonts/*
 images/*

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -18,10 +18,10 @@ sphinx:
 # Optionally build your docs in additional formats such as PDF and ePub
 formats:
   - pdf
-  
+
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.7
+  version: 3.8
   install:
     - requirements: documentation/rtd_requirements.txt
 

diff --git a/documentation/conf.py b/documentation/conf.py
@@ -1,3 +1,11 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/stable/config
+
 import os
 import sys
 import re
@@ -6,45 +14,43 @@
 
 from sphinx.transforms.post_transforms import ReferencesResolver
 
-sys.path.insert(0, os.path.abspath('..'))
 
-import amici
+def install_amici_deps_rtd():
+    """Install AMICI dependencies and set up environment for use on RTD"""
 
-# The short X.Y version
-version = amici.__version__
-# The full version, including alpha/beta/rc tags
-release = version
+    # cblas -- manually install ubuntu deb package
+    cblas_root = os.path.join(amici_dir, 'ThirdParty', 'libatlas-base-dev',
+                              'usr')
 
-del amici
-
-
-# -*- coding: utf-8 -*-
-#
-# Configuration file for the Sphinx documentation builder.
-#
-# This file does only contain a selection of the most common options. For a
-# full list see the documentation:
-# http://www.sphinx-doc.org/en/stable/config
+    if os.path.isdir(cblas_root):
+        # If this exists, it means this has been run before. On RTD, sphinx is
+        #  being run several times and we don't want to reinstall dependencies
+        #  every time.
+        return
 
-# -- RTD custom build --------------------------------------------------------
+    cblas_inc_dir = os.path.join(cblas_root, "include", "x86_64-linux-gnu")
+    cblas_lib_dir = os.path.join(cblas_root, "lib", "x86_64-linux-gnu")
+    cmd = (f"cd '{os.path.join(amici_dir, 'ThirdParty')}' "
+           "&& apt download libatlas-base-dev && mkdir libatlas-base-dev "
+           "&& cd libatlas-base-dev "
+           "&& ar x ../libatlas-base-dev_3.10.3-5_amd64.deb "
+           "&& tar -xJf data.tar.xz "
+           f"&& ln -s {cblas_inc_dir}/cblas-atlas.h {cblas_inc_dir}/cblas.h "
+           )
+    subprocess.run(cmd, shell=True, check=True)
+    os.environ['BLAS_CFLAGS'] = f'-I{cblas_inc_dir}'
+    os.environ['BLAS_LIBS'] = (f'-L{cblas_lib_dir}/atlas -L{cblas_lib_dir} '
+                               '-lcblas -latlas -lblas -lm')
 
-# only execute those commands when running from RTD
-if 'READTHEDOCS' in os.environ and os.environ['READTHEDOCS']:
-    amici_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
     # build swig4.0
     subprocess.run(os.path.join(amici_dir, 'scripts',
-                                'downloadAndBuildSwig.sh'))
+                                'downloadAndBuildSwig.sh'), check=True)
 
     # add swig to path
     swig_dir = os.path.join(amici_dir, 'ThirdParty', 'swig-4.0.1', 'install',
                             'bin')
     os.environ['SWIG'] = os.path.join(swig_dir, 'swig')
-    # in source install, this fails to compile the c extensions but we don't
-    # care since we replace it by a mock import later on
-    subprocess.run([
-        'python', '-m', 'pip', 'install', '--verbose', '-e',
-        os.path.join(amici_dir, 'python', 'sdist')
-    ])
+
 
 # -- Path setup --------------------------------------------------------------
 
@@ -53,23 +59,49 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 
-sys.path.insert(0, os.path.abspath('../python/sdist'))
-sys.path.insert(0, os.path.abspath('../'))
+amici_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-# -- Mock out some problematic modules-------------------------------------
+# -- RTD custom build --------------------------------------------------------
 
-# Note that for sub-modules, all parent modules must be listed explicitly.
-autodoc_mock_imports = ['_amici', 'amici._amici']
-for mod_name in autodoc_mock_imports:
-    sys.modules[mod_name] = mock.MagicMock()
+# only execute those commands when running from RTD
+if 'READTHEDOCS' in os.environ and os.environ['READTHEDOCS']:
+    install_amici_deps_rtd()
+
+# Install AMICI if not already present
+try:
+    import amici
+except ModuleNotFoundError:
+    subprocess.run([
+        'python', '-m', 'pip', 'install', '--verbose', '-e',
+        os.path.join(amici_dir, 'python', 'sdist')
+    ], check=True)
+
+    from importlib import invalidate_caches
+    invalidate_caches()
+
+    sys.path.insert(0, amici_dir)
+    sys.path.insert(0, os.path.join(amici_dir, 'python', 'sdist'))
+
+    import amici
 
 # -- Project information -----------------------------------------------------
+# The short X.Y version
+version = amici.__version__
+# The full version, including alpha/beta/rc tags
+release = version
 
 project = 'AMICI'
 copyright = '2020, The AMICI developers'
 author = 'The AMICI developers'
 title = 'AMICI Documentation'
 
+# -- Mock out some problematic modules-------------------------------------
+
+# Note that for sub-modules, all parent modules must be listed explicitly.
+autodoc_mock_imports = ['_amici', 'amici._amici']
+for mod_name in autodoc_mock_imports:
+    sys.modules[mod_name] = mock.MagicMock()
+
 # -- General configuration ---------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -80,6 +112,7 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'readthedocs_ext.readthedocs',
     'sphinx.ext.autodoc',
     'sphinx.ext.doctest',
     'sphinx.ext.coverage',
@@ -260,7 +293,6 @@
 
 
 def process_docstring(app, what, name, obj, options, lines):
-
     # only apply in the amici.amici module
     if len(name.split('.')) < 2 or name.split('.')[1] != 'amici':
         return
@@ -290,7 +322,7 @@ def process_docstring(app, what, name, obj, options, lines):
 
     if name == 'amici.amici.ParameterScalingVector':
         lines.append(
-            'Swig-Generated class, which ,in contrast to other Vector '
+            'Swig-Generated class, which, in contrast to other Vector '
             'classes, does not allow for simple interoperability with common '
             'python types, but must be created using '
             ':func:`amici.amici.parameterScalingFromIntVector`'
@@ -306,7 +338,7 @@ def process_docstring(app, what, name, obj, options, lines):
         )
         return
 
-    # add linebreaks before argument/return defintions
+    # add linebreaks before argument/return definitions
     lines_clean = []
 
     while len(lines):
@@ -337,6 +369,9 @@ def process_docstring(app, what, name, obj, options, lines):
 
 def fix_typehints(sig: str) -> str:
     # cleanup types
+    if not isinstance(sig, str):
+        return sig
+
     for old, new in typemaps.items():
         sig = sig.replace(old, new)
     sig = sig.replace('void', 'None')
@@ -432,26 +467,26 @@ def skip_member(app, what, name, obj, skip, options):
     if name.startswith('_') and name != '__init__':
         return True
 
-    # igore various functions for std::vector<> types
+    # ignore various functions for std::vector<> types
     if re.match(r'^<function [\w]+Vector\.', str(obj)):
         return True
 
-    # igore various functions for smart pointer types
+    # ignore various functions for smart pointer types
     if re.match(r'^<function [\w]+Ptr\.', str(obj)):
         return True
 
-    # igore various functions for StringDoubleMap
+    # ignore various functions for StringDoubleMap
     if str(obj).startswith('<function StringDoubleMap'):
         return True
 
     return None
 
 
-def setup(app):
-    app.connect('autodoc-process-docstring', process_docstring)
-    app.connect('autodoc-process-signature', process_signature)
-    app.connect('missing-reference', process_missing_ref)
-    app.connect('autodoc-skip-member', skip_member)
+def setup(app: 'sphinx.application.Sphinx'):
+    app.connect('autodoc-process-docstring', process_docstring, priority=0)
+    app.connect('autodoc-process-signature', process_signature, priority=0)
+    app.connect('missing-reference', process_missing_ref, priority=0)
+    app.connect('autodoc-skip-member', skip_member, priority=0)
     app.config.intersphinx_mapping = intersphinx_mapping
     app.config.autosummary_generate = True
     app.config.autodoc_mock_imports = autodoc_mock_imports
diff --git a/documentation/rtd_requirements.txt b/documentation/rtd_requirements.txt
@@ -1,4 +1,4 @@
-sphinx>=2.4.3
+sphinx>=3.2.1
 mock>=4.0.1
 setuptools>=45.2.0
 pysb>=1.11.0

diff --git a/python/amici/pandas.py b/python/amici/pandas.py
@@ -419,13 +419,13 @@ def _get_names_or_ids(model: AmiciModel,
     """
     Obtains a unique list of identifiers for the specified variable.
     First tries model.getVariableNames and then uses model.getVariableIds.
-    
+
     :param model:
         Model instance.
-        
+
     :param variable:
         variable name.
-        
+
     :param by_id:
         If True, ids are used as identifiers, otherwise first the possibly
         more descriptive names are used.

diff --git a/scripts/run-sphinx.sh b/scripts/run-sphinx.sh
@@ -1,13 +1,17 @@
 #!/bin/bash
-# generate code documentation via sphinx and upload to rtd
+# generate code documentation via sphinx for upload to rtd
 
 SCRIPT_PATH=$(dirname $BASH_SOURCE)
 AMICI_PATH=$(cd $SCRIPT_PATH/.. && pwd)
 
 cd ${AMICI_PATH}/documentation
-source ../build/venv/bin/activate
-pip3 install -r ${AMICI_PATH}/documentation/rtd_requirements.txt
+python3 -m venv ${AMICI_PATH}/doc-venv --clear
+source ${AMICI_PATH}/doc-venv/bin/activate
+python -m pip install --upgrade --no-cache-dir pip
+python -m pip install git+https://github.com/readthedocs/readthedocs-sphinx-ext
+python -m pip install --exists-action=w --no-cache-dir -r ${AMICI_PATH}/documentation/rtd_requirements.txt
 
-sphinx-build -T -E -W -D language=en -b html . _build
+rm -rf ${AMICI_PATH}/documentation/generated
+sphinx-build -T -E -W --keep-going -b readthedocs -d _build/doctrees-readthedocs -D language=en . _build/html
 ret=$?
 if [[ $ret != 0 ]]; then exit $ret; fi