Run notebooks on RTD with sphinx-gallery and add WorkGraph tutorials

.github/workflows/ci.yml

@@ -1,6 +1,11 @@
 name: continuous-integration
 
-on: [push, pull_request]
+on:
+  push:
+    # only pushes to main trigger
+    branches: [main]
+  pull_request:
+    # always triggered
 
 jobs:
 
@@ -11,29 +16,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.12
+    - name: Set up Python 3.11
       uses: actions/setup-python@v2
       with:
-        python-version: 3.12
+        python-version: 3.11
     - uses: pre-commit/[email protected]
-
-  build:
-
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.12
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.12
-    - name: Install python dependencies
-      run: |
-        pip install --upgrade pip
-        pip install -r requirements.txt
-    - name: "Build HTML docs"
-      run: |
-        make -C docs html linkcheck
-      env:
-        SPHINXOPTS: -nW --keep-going

.readthedocs.yml

@@ -7,7 +7,21 @@ version: 2
 build:
   os: "ubuntu-22.04"
   tools:
-    python: "3.12"
+    python: "miniconda3-3.12-24.1" # note that libmamba-solver is available since 22.1
+  jobs:
+    post_create_environment:
+      # - python -m pip install --exists-action=w --no-cache-dir -r requirements.txt
+      - rabbitmq-server -detached
+      - sleep 10
+      - rabbitmq-diagnostics status
+      - verdi presto
+      - verdi daemon start
+      - verdi status
+      - aiida-pseudo install sssp -x PBEsol
+      - verdi group list
+      - cat /proc/cpuinfo | grep processor | wc -l
+      - python -m sphinx -T --keep-going -b linkcheck -d docs/_build/doctrees -D language=en docs $READTHEDOCS_OUTPUT/html
+
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
@@ -19,7 +33,7 @@ sphinx:
 # See https://docs.readthedocs.io/en/latest/yaml-config.html#formats
 formats: []
 
+conda:
+  environment: environment.yml
+
 # Optionally set the version of Python and requirements required to build your docs
-python:
-  install:
-    - requirements: requirements.txt

README.md

@@ -26,17 +26,19 @@ If you have a question, feel free to just [open an issue](https://github.com/aii
 
 ### Prerequisites
 
-* python 3.12 or greater
+* python 3.11
 
 ### Build instructions
 
 ```bash
 git clone https://github.com/aiidateam/aiida-tutorials.git
 cd aiida-tutorials
-pip install -r requirements.txt
+conda env create --quiet --name aiida-tutorials --file environment.yml
+conda activate aiida-tutorials
 pre-commit install   # enable pre-commit hooks (optional)
-cd docs/
-make
+make -C docs html # to build docs
+make -C docs linkcheck # to run link checks (only for dev)
+
 # open build/html/index.html
 ```
 

docs/conf.py

@@ -42,6 +42,7 @@
     "sphinx_copybutton",
     "sphinx_panels",
     "sphinx_tabs.tabs",
+    "sphinx_gallery.gen_gallery",
 ]
 
 myst_enable_extensions = [
@@ -68,7 +69,7 @@
 ipython_mplbackend = ""
 
 copybutton_selector = "div:not(.no-copy)>div.highlight pre"
-copybutton_prompt_text = ">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: "
+copybutton_prompt_text = ">>> |... |$ |In [d*]: | {2,5}...: | {5,8}: "
 copybutton_prompt_is_regexp = True
 
 todo_include_todos = True
@@ -391,13 +392,12 @@
 suppress_warnings = ["misc.highlighting_failure"]
 
 # Links we ignore, because they do not work temporary and we cannot fix it
-linkcheck_ignore = ["https://www.big-map.eu/"]
-
-
-def setup(app):
-    """Setup function called by sphinx."""
-    app.add_css_file("css/custom.css")
-
+linkcheck_ignore = [
+    "https://www.big-map.eu/",
+    r".*concept/index.html",
+    r".*howto/index.html",
+    "http://127.0.0.1:8000/workgraph",
+]
 
 # We are not installing a full aiida environment
 nb_execution_mode = "off"
@@ -408,6 +408,29 @@ def setup(app):
     "plumpy": ("https://plumpy.readthedocs.io/en/latest/", None),
 }
 
+gallery_src_relative_dir = "gallery"  # relative path of the gallery src wrt. sphinx src
+
+# we mimik the structure in the sphinx src directory in the gallery src directory
+sphinx_src_autogen_dirs = ["sections/writing_workflows_with_workgraph/autogen"]
+
+gallery_src_dirs = [
+    os.path.join(gallery_src_relative_dir, autogen_dir)
+    for autogen_dir in sphinx_src_autogen_dirs
+]  # path of the python scripts that should be executed
+sphinx_gallery_conf = {
+    "filename_pattern": "/*",
+    "examples_dirs": gallery_src_dirs,  # in sphinx-gallery doc referred as gallery source
+    "gallery_dirs": sphinx_src_autogen_dirs,  # path to where to gallery puts generated files
+}
+
+# ignore in the autogenerated ipynb files to surpress warning
+exclude_patterns.extend(
+    [
+        os.path.join(sphinx_src_autogen_dir, "*ipynb")
+        for sphinx_src_autogen_dir in sphinx_src_autogen_dirs
+    ]
+)
+
 # Compile all things needed before building the docs
 # For instance, convert the notebook templates to actual tutorial and solution versions
 print(
@@ -416,3 +439,57 @@ def setup(app):
         universal_newlines=True,
     )
 )
+
+import shutil
+from pathlib import Path
+
+
+def copy_html_files(app, exception):
+    """
+    Copy all .html files from source to build directory, maintaining the directory structure.
+    """
+    print("Copying HTML files to build directory")
+    copy_print_info = "Copying HTML files to build directory"
+    print()
+    print(copy_print_info)
+    print(len(copy_print_info) * "=")
+    if exception is not None:  # Only copy files if the build succeeded
+        print(
+            "Build failed, but we still try to copy the HTML files to the build directory"
+        )
+    try:
+        src_path = Path(app.builder.srcdir)
+        build_path = Path(app.builder.outdir)
+
+        copy_print_info = f"Copying html files from sphinx src directory {src_path}"
+        print()
+        print(copy_print_info)
+        print(len(copy_print_info) * "-")
+        for html_file in src_path.rglob("*.html"):
+            relative_path = html_file.relative_to(src_path)
+            destination_file = build_path / relative_path
+            destination_file.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy(html_file, destination_file)
+            print(f"Copy {html_file} to {destination_file}")
+
+        gallery_src_path = Path(app.builder.srcdir / Path(gallery_src_relative_dir))
+
+        copy_print_info = (
+            f"Copying html files from gallery src directory {gallery_src_path} to build"
+        )
+        print()
+        print(copy_print_info)
+        print(len(copy_print_info) * "-")
+        for html_file in gallery_src_path.rglob("*.html"):
+            relative_path = html_file.relative_to(gallery_src_path)
+            destination_file = build_path / relative_path
+            destination_file.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy(html_file, destination_file)
+            print(f"Copy {html_file} to {destination_file}")
+    except Exception as e:
+        print(f"Failed to copy HTML files: {e}")
+
+
+def setup(app):
+    app.add_css_file("css/custom.css")
+    app.connect("build-finished", copy_html_files)

docs/gallery/sections/writing_workflows_with_workgraph/autogen/GALLERY_HEADER.rst

@@ -0,0 +1,5 @@
+:orphan:
+
+================================
+Running workflows with WorkGraph
+================================