Create pre-commit hooks

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-files: '.py'
+files: '.py|.ipynb'
 exclude: '.git'
 default_stages: [pre-commit]
 
@@ -23,4 +23,17 @@ repos:
   - repo: https://github.com/christopher-hacker/enforce-notebook-run-order
     rev: 2.0.1
     hooks:
-      - id: enforce-notebook-run-order
+      - id: enforce-notebook-run-order
+
+  - repo: local
+    hooks:
+      - id: check_notebooks
+        name: check notebooks
+        entry: python ./pre_commit_hooks/check_notebooks.py
+        language: system
+        files: '.ipynb'
+      - id: check_badges
+        name: check badges
+        entry: python ./pre_commit_hooks/check_badges.py
+        language: system
+        files: '.ipynb'

pre_commit_hooks/check_badges.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+import argparse
+from collections.abc import Sequence
+
+import nbformat
+
+from utils import relative_path, repo_path
+
+COLAB_HEADER = f"""import sys
+if 'google.colab' in sys.modules:
+    !pip --quiet install open-atmos-jupyter-utils
+    from open_atmos_jupyter_utils import pip_install_on_colab
+    pip_install_on_colab('{repo_path().name}-examples')"""
+
+
+def _preview_badge_markdown(absolute_path):
+    svg_badge_url = (
+        "https://img.shields.io/static/v1?"
+        + "label=render%20on&logo=github&color=87ce3e&message=GitHub"
+    )
+    link = (
+        f"https://github.com/open-atmos/{repo_path().name}/blob/main/"
+        + f"{relative_path(absolute_path)}"
+    )
+    return f"[![preview notebook]({svg_badge_url})]({link})"
+
+
+def _mybinder_badge_markdown(absolute_path):
+    svg_badge_url = "https://mybinder.org/badge_logo.svg"
+    link = (
+        f"https://mybinder.org/v2/gh/open-atmos/{repo_path().name}.git/main?urlpath=lab/tree/"
+        + f"{relative_path(absolute_path)}"
+    )
+    return f"[![launch on mybinder.org]({svg_badge_url})]({link})"
+
+
+def _colab_badge_markdown(absolute_path):
+    svg_badge_url = "https://colab.research.google.com/assets/colab-badge.svg"
+    link = (
+        f"https://colab.research.google.com/github/open-atmos/{repo_path().name}/blob/main/"
+        + f"{relative_path(absolute_path)}"
+    )
+    return f"[![launch on Colab]({svg_badge_url})]({link})"
+
+
+def test_first_cell_contains_three_badges(notebook_filename):
+    """checks if all notebooks feature Github preview, mybinder and Colab badges
+    (in the first cell)"""
+    with open(notebook_filename, encoding="utf8") as fp:
+        nb = nbformat.read(fp, nbformat.NO_CONVERT)
+        assert len(nb.cells) > 0
+        assert nb.cells[0].cell_type == "markdown"
+        lines = nb.cells[0].source.split("\n")
+        assert len(lines) == 3
+        assert lines[0] == _preview_badge_markdown(notebook_filename)
+        assert lines[1] == _mybinder_badge_markdown(notebook_filename)
+        assert lines[2] == _colab_badge_markdown(notebook_filename)
+
+
+def test_second_cell_is_a_markdown_cell(notebook_filename):
+    """checks if all notebooks have their second cell with some markdown
+    (hopefully clarifying what the example is about)"""
+    with open(notebook_filename, encoding="utf8") as fp:
+        nb = nbformat.read(fp, nbformat.NO_CONVERT)
+        assert len(nb.cells) > 1
+        assert nb.cells[1].cell_type == "markdown"
+
+
+def test_third_cell_contains_colab_header(notebook_filename):
+    """checks if all notebooks feature a Colab-magic cell"""
+    with open(notebook_filename, encoding="utf8") as fp:
+        nb = nbformat.read(fp, nbformat.NO_CONVERT)
+        assert len(nb.cells) > 2
+        assert nb.cells[2].cell_type == "code"
+        assert nb.cells[2].source == COLAB_HEADER
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("filenames", nargs="*", help="Filenames to check.")
+    args = parser.parse_args(argv)
+
+    retval = 0
+    for filename in args.filenames:
+        try:
+            test_first_cell_contains_three_badges(filename)
+            test_second_cell_is_a_markdown_cell(filename)
+            test_third_cell_contains_colab_header(filename)
+        except ValueError as exc:
+            retval = 1
+    return retval
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pre_commit_hooks/check_notebooks.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import argparse
+from collections.abc import Sequence
+
+import nbformat
+
+
+def test_cell_contains_output(notebook):
+    """checks if all notebook cells have an output present"""
+    for cell in notebook.cells:
+        if cell.cell_type == "code" and cell.source != "":
+            assert cell.execution_count is not None
+
+
+def test_no_errors_or_warnings_in_output(notebook):
+    """checks if all example Jupyter notebooks have clear std-err output
+    (i.e., no errors or warnings) visible; except acceptable
+    diagnostics from the joblib package"""
+    for cell in notebook.cells:
+        if cell.cell_type == "code":
+            for output in cell.outputs:
+                if "name" in output and output["name"] == "stderr":
+                    if not output["text"].startswith("[Parallel(n_jobs="):
+                        raise AssertionError(output["text"])
+
+
+def test_show_plot_used_instead_of_matplotlib(notebook):
+    """checks if plotting is done with open_atmos_jupyter_utils show_plot()"""
+    matplot_used = False
+    show_plot_used = False
+    for cell in notebook.cells:
+        if cell.cell_type == "code":
+            if (
+                "pyplot.show()" in cell.source
+                or "plt.show()" in cell.source
+                or "from matplotlib import pyplot" in cell.source
+            ):
+                matplot_used = True
+            if "show_plot()" in cell.source:
+                show_plot_used = True
+    if matplot_used and not show_plot_used:
+        raise AssertionError(
+            "if using matplotlib, please use open_atmos_jupyter_utils.show_plot()"
+        )
+
+
+def test_show_anim_used_instead_of_matplotlib(notebook):
+    """checks if animation generation is done with open_atmos_jupyter_utils show_anim()"""
+    matplot_used = False
+    show_anim_used = False
+    for cell in notebook.cells:
+        if cell.cell_type == "code":
+            if (
+                "funcAnimation" in cell.source
+                or "matplotlib.animation" in cell.source
+                or "from matplotlib import animation" in cell.source
+            ):
+                matplot_used = True
+            if "show_anim()" in cell.source:
+                show_anim_used = True
+    if matplot_used and not show_anim_used:
+        raise AssertionError(
+            """if using matplotlib for animations,
+            please use open_atmos_jupyter_utils.show_anim()"""
+        )
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("filenames", nargs="*", help="Filenames to check.")
+    args = parser.parse_args(argv)
+
+    retval = 0
+    for filename in args.filenames:
+        with open(filename, encoding="utf8") as notebook_file:
+            notebook = nbformat.read(notebook_file, nbformat.NO_CONVERT)
+            try:
+                test_cell_contains_output(notebook)
+                test_no_errors_or_warnings_in_output(notebook)
+            except ValueError as exc:
+                retval = 1
+    return retval
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pre_commit_hooks/utils.py

@@ -0,0 +1,45 @@
+"""
+Utils functions to reuse in different parts of the codebase
+"""
+
+import os
+import pathlib
+
+from git.cmd import Git
+
+
+def find_files(path_to_folder_from_project_root=".", file_extension=None):
+    """
+    Returns all files in a current git repo.
+    The list of returned files may be filtered with `file_extension` param.
+    """
+    all_files = [
+        path
+        for path in Git(
+            Git(path_to_folder_from_project_root).rev_parse("--show-toplevel")
+        )
+        .ls_files()
+        .split("\n")
+        if os.path.isfile(path)
+    ]
+    if file_extension is not None:
+        return list(filter(lambda path: path.endswith(file_extension), all_files))
+
+    return all_files
+
+
+def relative_path(absolute_path):
+    """returns a path relative to the repo base (converting backslashes to slashes on Windows)"""
+    relpath = os.path.relpath(absolute_path, repo_path().absolute())
+    posixpath_to_make_it_usable_in_urls_even_on_windows = pathlib.Path(
+        relpath
+    ).as_posix()
+    return posixpath_to_make_it_usable_in_urls_even_on_windows
+
+
+def repo_path():
+    """returns absolute path to the repo base (ignoring .git location if in a submodule)"""
+    path = pathlib.Path(__file__)
+    while not (path.is_dir() and Git(path).rev_parse("--git-dir") == ".git"):
+        path = path.parent
+    return path