Write out QSIRecon pipeline-wise reports (#7)

* Add report config files.

* Remove viz submodule.

* Work on nireports.

* Add viz utils back in.

* More work.

* Fix import.

* Remove reportlets_dir.

* Try to fix things.

* Add qsirecon_suffix info.

* Update core.py

* Update run.py

* Update expected outputs.

* Update run.py

* Update.

* Update test_cli.py

* Update run.py

* Update expected outputs.

* Update workflow.py

* Update.

* Fix.

pyproject.toml

@@ -28,6 +28,7 @@ dependencies = [
     "nibabel <= 5.2.0",
     "nilearn == 0.10.1",
     "nipype == 1.8.6",
+    "nireports",
     "niworkflows >=1.9,<= 1.10",
     "numpy <= 1.26.3",
     "pandas < 2.0.0",
@@ -177,6 +178,7 @@ markers = [
     "pyafq_recon_full: test 19",
     "mrtrix3_recon: test 20",
     "tortoise_recon: test 21",
+    "autotrack: test 22",
 ]
 env = [
     "RUNNING_PYTEST = 1",

qsirecon/cli/parser.py

@@ -450,9 +450,6 @@ def parse_args(args=None, namespace=None):
     work_dir = config.execution.work_dir
     version = config.environment.version
 
-    if config.execution.reportlets_dir is None:
-        config.execution.reportlets_dir = work_dir / "reportlets"
-
     # Update the config with an empty dict to trigger initialization of all config
     # sections (we used `init=False` above).
     # This must be done after cleaning the work directory, or we could delete an
@@ -479,7 +476,6 @@ def parse_args(args=None, namespace=None):
     config.execution.log_dir = config.execution.output_dir / "logs"
     # Check and create output and working directories
     config.execution.log_dir.mkdir(exist_ok=True, parents=True)
-    config.execution.reportlets_dir.mkdir(exist_ok=True, parents=True)
     work_dir.mkdir(exist_ok=True, parents=True)
 
     # Force initialization of the BIDSLayout

qsirecon/cli/run.py

@@ -171,39 +171,53 @@ def main():
         errno = 0
     finally:
 
-        from ..viz.reports import generate_reports
+        from ..reports.core import generate_reports
+        from ..workflows.base import _load_recon_spec
+        from .workflow import copy_boilerplate
 
         # Generate reports phase
-        # session_list = (
-        #     config.execution.get().get('bids_filters', {}).get('dwi', {}).get('session')
-        # )
+        session_list = config.execution.get().get("bids_filters", {}).get("dwi", {}).get("session")
 
-        failed_reports = generate_reports(
-            config.execution.participant_label,
-            # session_list=session_list,
-        )
         write_derivative_description(
             config.execution.bids_dir,
             config.execution.output_dir,
             # dataset_links=config.execution.dataset_links,
         )
         write_bidsignore(config.execution.output_dir)
 
+        workflow_spec = _load_recon_spec()
+        # Compile list of output folders
+        # TODO: Retain QSIRecon pipeline names in the config object
+        qsirecon_suffixes = []
+        for node_spec in workflow_spec["nodes"]:
+            qsirecon_suffix = node_spec.get("qsirecon_suffix", None)
+            qsirecon_suffixes += [qsirecon_suffix] if qsirecon_suffix else []
+
+        qsirecon_suffixes = sorted(list(set(qsirecon_suffixes)))
+        config.loggers.cli.info(f"QSIRecon pipeline suffixes: {qsirecon_suffixes}")
+        failed_reports = []
+        for qsirecon_suffix in qsirecon_suffixes:
+            suffix_dir = str(config.execution.output_dir / f"qsirecon-{qsirecon_suffix}")
+
+            # Copy the boilerplate files
+            copy_boilerplate(config.execution.output_dir, suffix_dir)
+
+            suffix_failed_reports = generate_reports(
+                config.execution.participant_label,
+                suffix_dir,
+                config.execution.run_uuid,
+                session_list=session_list,
+                qsirecon_suffix=qsirecon_suffix,
+            )
+            failed_reports += suffix_failed_reports
+            write_derivative_description(
+                config.execution.bids_dir,
+                suffix_dir,
+                # dataset_links=config.execution.dataset_links,
+            )
+            write_bidsignore(suffix_dir)
+
         if failed_reports:
             print(failed_reports)
-            # msg = (
-            #     'Report generation was not successful for the following participants '
-            #     f': {", ".join(failed_reports)}.'
-            # )
-            # config.loggers.cli.error(msg)
-            # if sentry_sdk is not None:
-            #     sentry_sdk.capture_message(msg, level='error')
-
-        # If preprocessing and recon are requested in the same call, start the recon workflow now.
-        if errno > 0:
-            if config.nipype.stop_on_first_crash:
-                config.loggers.workflow.critical(
-                    "Errors occurred during preprocessing - Recon will not run."
-                )
 
-        sys.exit(int((errno + failed_reports) > 0))
+        sys.exit(int(errno + len(failed_reports)) > 0)

qsirecon/cli/workflow.py

@@ -45,8 +45,8 @@ def build_workflow(config_file, retval):
     # from niworkflows.utils.misc import check_valid_fs_license
     # from ..utils.bids import check_pipeline_version
     from .. import config
+    from ..reports.core import generate_reports
     from ..utils.misc import check_deps
-    from ..viz.reports import generate_reports
     from ..workflows.base import init_qsirecon_wf
 
     config.load(config_file)
@@ -99,6 +99,7 @@ def build_workflow(config_file, retval):
             config.execution.output_dir,
             config.execution.run_uuid,
             session_list=session_list,
+            qsirecon_suffix="",
         )
         if failed_reports:
             config.loggers.cli.error(
@@ -224,3 +225,17 @@ def build_boilerplate(config_file, workflow):
             check_call(cmd, timeout=10)
         except (FileNotFoundError, CalledProcessError, TimeoutExpired):
             config.loggers.cli.warning("Could not generate CITATION.tex file:\n%s", " ".join(cmd))
+
+
+def copy_boilerplate(in_dir, out_dir):
+    import shutil
+
+    in_logs_path = Path(in_dir) / "logs"
+    out_logs_path = Path(out_dir) / "logs"
+    out_logs_path.mkdir(exist_ok=True, parents=True)
+    citation_files = {
+        ext: in_logs_path / ("CITATION.%s" % ext) for ext in ("bib", "tex", "md", "html")
+    }
+    for ext, citation_file in citation_files.items():
+        if citation_file.exists():
+            shutil.copy(citation_file, out_logs_path / f"CITATION.{ext}")

qsirecon/config.py

@@ -426,8 +426,6 @@ class execution(_Config):
     """List of participant identifiers that are to be preprocessed."""
     freesurfer_input = None
     """Directory containing FreeSurfer directories to use for recon workflows."""
-    reportlets_dir = None
-    """Path where reportlets are written."""
     templateflow_home = _templateflow_home
     """The root folder of the TemplateFlow client."""
     work_dir = Path("work").absolute()
@@ -436,6 +434,8 @@ class execution(_Config):
     """Write out the computational graph corresponding to the planned postprocessing."""
     dataset_links = {}
     """A dictionary of dataset links to be used to track Sources in sidecars."""
+    aggr_ses_reports = 4  # TODO: Change to None when implemented on command line
+    """Maximum number of sessions aggregated in one subject's visual report."""
 
     _layout = None
 
@@ -450,7 +450,6 @@ class execution(_Config):
         "layout",
         "log_dir",
         "output_dir",
-        "reportlets_dir",
         "templateflow_home",
         "work_dir",
     )

qsirecon/data/__init__.py

@@ -1,5 +1,182 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
-# vi: set ft=python sts=4 ts=4 sw=4 et:
-""" Data fetchers module """
+"""QSIRecon data files
+
+.. autofunction:: load
+
+.. automethod:: load.readable
+
+.. automethod:: load.as_path
+
+.. automethod:: load.cached
+
+.. autoclass:: Loader
+"""
+
+from __future__ import annotations
+
+import atexit
+import os
+from contextlib import AbstractContextManager, ExitStack
+from functools import cached_property
+from pathlib import Path
+from types import ModuleType
+
+try:
+    from functools import cache
+except ImportError:  # PY38
+    from functools import lru_cache as cache
+
+try:  # Prefer backport to leave consistency to dependency spec
+    from importlib_resources import as_file, files
+except ImportError:
+    from importlib.resources import as_file, files  # type: ignore
+
+try:  # Prefer stdlib so Sphinx can link to authoritative documentation
+    from importlib.resources.abc import Traversable
+except ImportError:
+    from importlib_resources.abc import Traversable
+
+__all__ = ["load"]
+
+
+class Loader:
+    """A loader for package files relative to a module
+
+    This class wraps :mod:`importlib.resources` to provide a getter
+    function with an interpreter-lifetime scope. For typical packages
+    it simply passes through filesystem paths as :class:`~pathlib.Path`
+    objects. For zipped distributions, it will unpack the files into
+    a temporary directory that is cleaned up on interpreter exit.
+
+    This loader accepts a fully-qualified module name or a module
+    object.
+
+    Expected usage::
+
+        '''Data package
+
+        .. autofunction:: load_data
+
+        .. automethod:: load_data.readable
+
+        .. automethod:: load_data.as_path
+
+        .. automethod:: load_data.cached
+        '''
+
+        from qsirecon.data import Loader
+
+        load_data = Loader(__package__)
+
+    :class:`~Loader` objects implement the :func:`callable` interface
+    and generate a docstring, and are intended to be treated and documented
+    as functions.
+
+    For greater flexibility and improved readability over the ``importlib.resources``
+    interface, explicit methods are provided to access resources.
+
+    +---------------+----------------+------------------+
+    | On-filesystem | Lifetime       | Method           |
+    +---------------+----------------+------------------+
+    | `True`        | Interpreter    | :meth:`cached`   |
+    +---------------+----------------+------------------+
+    | `True`        | `with` context | :meth:`as_path`  |
+    +---------------+----------------+------------------+
+    | `False`       | n/a            | :meth:`readable` |
+    +---------------+----------------+------------------+
+
+    It is also possible to use ``Loader`` directly::
+
+        from qsirecon.data import Loader
+
+        Loader(other_package).readable('data/resource.ext').read_text()
+
+        with Loader(other_package).as_path('data') as pkgdata:
+            # Call function that requires full Path implementation
+            func(pkgdata)
+
+        # contrast to
+
+        from importlib_resources import files, as_file
+
+        files(other_package).joinpath('data/resource.ext').read_text()
+
+        with as_file(files(other_package) / 'data') as pkgdata:
+            func(pkgdata)
+
+    .. automethod:: readable
+
+    .. automethod:: as_path
+
+    .. automethod:: cached
+    """
+
+    def __init__(self, anchor: str | ModuleType):
+        self._anchor = anchor
+        self.files = files(anchor)
+        self.exit_stack = ExitStack()
+        atexit.register(self.exit_stack.close)
+        # Allow class to have a different docstring from instances
+        self.__doc__ = self._doc
+
+    @cached_property
+    def _doc(self):
+        """Construct docstring for instances
+
+        Lists the public top-level paths inside the location, where
+        non-public means has a `.` or `_` prefix or is a 'tests'
+        directory.
+        """
+        top_level = sorted(
+            os.path.relpath(p, self.files) + "/"[: p.is_dir()]
+            for p in self.files.iterdir()
+            if p.name[0] not in (".", "_") and p.name != "tests"
+        )
+        doclines = [
+            f"Load package files relative to ``{self._anchor}``.",
+            "",
+            "This package contains the following (top-level) files/directories:",
+            "",
+            *(f"* ``{path}``" for path in top_level),
+        ]
+
+        return "\n".join(doclines)
+
+    def readable(self, *segments) -> Traversable:
+        """Provide read access to a resource through a Path-like interface.
+
+        This file may or may not exist on the filesystem, and may be
+        efficiently used for read operations, including directory traversal.
+
+        This result is not cached or copied to the filesystem in cases where
+        that would be necessary.
+        """
+        return self.files.joinpath(*segments)
+
+    def as_path(self, *segments) -> AbstractContextManager[Path]:
+        """Ensure data is available as a :class:`~pathlib.Path`.
+
+        This method generates a context manager that yields a Path when
+        entered.
+
+        This result is not cached, and any temporary files that are created
+        are deleted when the context is exited.
+        """
+        return as_file(self.files.joinpath(*segments))
+
+    @cache  # noqa: B019
+    def cached(self, *segments) -> Path:
+        """Ensure data is available as a :class:`~pathlib.Path`.
+
+        Any temporary files that are created remain available throughout
+        the duration of the program, and are deleted when Python exits.
+
+        Results are cached so that multiple calls do not unpack the same
+        data multiple times, but the cache is sensitive to the specific
+        argument(s) passed.
+        """
+        return self.exit_stack.enter_context(as_file(self.files.joinpath(*segments)))
+
+    __call__ = cached
+
+
+load = Loader(__package__)