Merge pull request #86 from kaitj/maint/dwi_dir

Update handling of external `dwi_dir`

docs/outputs/output_files.md

@@ -1,9 +1,9 @@
 # Output Files
 
-After running the workflow, the `/path/to/output/dir` folder will contain a 
-hidden `.logs` folder for troubleshooting, as well as additional folders 
-associated with the tools that were used to generate files (e.g. `freesurfer`, 
-`labelmerge`), but for most purposes, all the primary outputs of interest will 
+After running the workflow, the `/path/to/output/dir` folder will contain a
+hidden `.logs` folder for troubleshooting, as well as additional folders
+associated with the tools that were used to generate files (e.g. `freesurfer`,
+`labelmerge`), but for most purposes, all the primary outputs of interest will
 be in the `mrtrix` directory with the following structure:
 
 ```
@@ -15,16 +15,16 @@ mrtrix/
     └── tractography
 ```
 
-Briefly, `dti` contains processed diffusion tensor images along with 
+Briefly, `dti` contains processed diffusion tensor images along with
 quantitative maps (e.g. `FA`, `MD`, etc.), `dwi` contains images that were
-converted from Nifti (`.nii.gz`) to the MRtrix imaging format (`.mif`), 
+converted from Nifti (`.nii.gz`) to the MRtrix imaging format (`.mif`),
 `response` contains subject-specific response functions, and `tractography`
 contains tractogram files.
 
 ## dti
 
-This folder contains processed diffusion tensor images (DTI) that were used to 
-compute quantitative maps (stored in the MRtrix imaging format - `.mif`), also 
+This folder contains processed diffusion tensor images (DTI) that were used to
+compute quantitative maps (stored in the MRtrix imaging format - `.mif`), also
 found within the folder:
 
 ```
@@ -38,20 +38,20 @@ sub-{subject}
 ```
 
 As per the BIDS extension proposal, `model-dti` denotes to the DTI model used to
-process the data, and the suffix (`ad`, `fa`, `md`, `rd`, `tensor`) describes 
+process the data, and the suffix (`ad`, `fa`, `md`, `rd`, `tensor`) describes
 the imaging data.
 
 ## dwi
 
-This folder contains the input diffusion weighted image (dwi) and brain mask 
+This folder contains the input diffusion weighted image (dwi) and brain mask
 converted from Nifti (`.nii.gz`) to the MRtrix imaging format (`.mif`) used in
 downstream processing. It also contains an intensity normalized dwi image used
 for DTI:
 
 ```
 sub-{subject}
 └── dwi
-    ├── sub-{subject}_brainmask.mif
+    ├── sub-{subject}_desc-brain_mask.mif
     ├── sub-{subject}_desc-normalized_dwi.mif
     └── sub-{subject}_dwi.mif
 ```
@@ -60,7 +60,7 @@ sub-{subject}
 
 This folder contains the subject-specific response functions that were estimated
 from the input diffusion data, as well as both the normalized and unnormalized
-versions of the estimated fibre orientation distribution (FOD) maps. Both 
+versions of the estimated fibre orientation distribution (FOD) maps. Both
 response functions and FOD maps are estimated for the three different tissue
 types: white matter (wm), grey matter (gm), and corticospinal fluid (csf):
 
@@ -96,19 +96,20 @@ sub-{subject}
     ├── sub-{subject}_desc-subcortical_nodeWeights.csv
 ```
 
-The `desc-filteredsubcortical` entity pair describes associated tractogram 
-files that have been filtered to only pass through WM and connect two GM 
+The `desc-filteredsubcortical` entity pair describes associated tractogram
+files that have been filtered to only pass through WM and connect two GM
 structures, while the `desc-subcortical` entity pair denotes files associated
 with the unfiltered subcortical connectome. Whole-brain tractogram associated
 files are denoted by `desc-iFOD2`, which describes the tractography algorithm
 used to perform tractograpy.
 
-_Note: As the BIDS extension proposal has not been finalized, the naming of 
+_Note: As the BIDS extension proposal has not been finalized, the naming of
 such files may be subject to change_
 
 ## Additional Files
 
 The top-level `/path/to/output/dir` contains additional files / folders:
+
 ```
 /path/to/output/dir
 ├── ...
@@ -118,13 +119,13 @@ The top-level `/path/to/output/dir` contains additional files / folders:
 └── .snakemake
 ```
 
-The `config` folder, along with the hidden `.snakebids` and `.snakemake` folders 
+The `config` folder, along with the hidden `.snakebids` and `.snakemake` folders
 contain a record of the code and parameters used, and paths to the inputs.
 
-Workflow steps that write logs to file are stored in the hidden `.logs` 
-subfolder, with the file names based on the tools used (e.g. `mrtrix`) and rule 
-wildcards (e.g. `subject`). 
+Workflow steps that write logs to file are stored in the hidden `.logs`
+subfolder, with the file names based on the tools used (e.g. `mrtrix`) and rule
+wildcards (e.g. `subject`).
 
-If the app is run in workflow mode (`--workflow-mode` / `-W`), which enables 
+If the app is run in workflow mode (`--workflow-mode` / `-W`), which enables
 direct use of the Snakemake CLI to run scattr, output folders (e.g. `work`) will
-be placed in a `results` folder.
+be placed in a `results` folder.

pyproject.toml

@@ -100,6 +100,13 @@ python ./scattr/run.py ./test/data/bids_nodwi test/data/derivatives/ \
     --fs-license ./test/.fs_license -np --force-output
 """
 
+[tool.poe.tasks.test_snakedwi]
+shell = """
+python ./scattr/run.py ./test/data/bids_nodwi test/data/derivatives/ \
+    participant --dwi_dir ./test/data/derivatives/snakedwi \
+    --fs-license ./test/.fs_license -np --force-output
+"""
+
 [tool.poe.tasks.test_labelmerge]
 shell = """
 python ./scattr/run.py ./test/data/bids test/data/derivatives/ participant \

scattr/config/snakebids.yml

@@ -30,36 +30,27 @@ pybids_inputs:
     wildcards:
       - subject
       - session
-      - acquisition
-      - part
-      - reconstruction
       - run
+pybids_inputs_dwi:
   dwi:
     filters:
       suffix: "dwi"
       extension: ".nii.gz"
       datatype: "dwi"
       space: "T1w"
-      part: ["mag", false]
     wildcards:
       - subject
       - session
-      - acquisition
-      - direction
-      - part
-      - reconstruction
       - run
   mask:
     filters:
       suffix: "mask"
       extension: ".nii.gz"
       datatype: "dwi"
-      space: ["orig", "scanner", "individual", "T1w"]
+      space: "T1w"
     wildcards:
       - subject
       - session
-      - acquisition
-      - res
       - run
 
 # Configuration for the command-line parameters to make available
@@ -118,6 +109,11 @@ parse_args:
     nargs: "?"
     type: Path
 
+  --pybidsdb_dwi_dir:
+    help: "The path to the pybids database associated with provided dwi_dir"
+    nargs: "?"
+    type: Path
+
   --responsemean_dir:
     help: "The path to the directory containing average response functions. If
       not provided, one will be computed from the subjects in the input

scattr/workflow/Snakefile

@@ -12,10 +12,24 @@ configfile: "config/snakebids.yml"
 
 
 # writes inputs_config.yml and updates config dict
-inputs = generate_inputs(
+inputs_t1w = generate_inputs(
     bids_dir=config["bids_dir"],
     pybids_inputs=config["pybids_inputs"],
     pybids_config=["bids", "derivatives"],
+    pybidsdb_dir=config.get("pybidsdb_dir"),
+    pybidsdb_reset=config.get("pybidsdb_reset"),
+    derivatives=config["derivatives"],
+    participant_label=config["participant_label"],
+    exclude_participant_label=config["exclude_participant_label"],
+)
+inputs_dwi = generate_inputs(
+    bids_dir=config["dwi_dir"] if config["dwi_dir"] else config["bids_dir"],
+    pybids_inputs=config["pybids_inputs_dwi"],
+    pybids_config=["bids", "derivatives"],
+    pybidsdb_dir=config.get("pybidsdb_dwi_dir")
+    if config["dwi_dir"]
+    else config.get("pybidsdb_dir"),
+    pybidsdb_reset=config.get("pybidsdb_reset"),
     derivatives=config["derivatives"],
     participant_label=config["participant_label"],
     exclude_participant_label=config["exclude_participant_label"],
@@ -33,13 +47,6 @@ import os
 from pathlib import Path
 from functools import partial
 
-# Relevant wildcards (upgrade for filtering)
-subj_zip_list = {
-    k: v
-    for k, v in inputs["T1w"].zip_lists.items()
-    if k in ["subject", "session"]
-}
-
 # Warnings
 if config.get("labelmerge_base_dir") or config.get("labelmerge_overlay_dir"):
     print(
@@ -49,7 +56,7 @@ if config.get("labelmerge_base_dir") or config.get("labelmerge_overlay_dir"):
     """
     )
 
-if len(inputs.sessions) > 1 and not config.get("responsemean_ses"):
+if len(inputs_dwi.sessions) > 1 and not config.get("responsemean_ses"):
     print(
         """
     WARNING: Multiple sessions detected - average response function will be 
@@ -68,10 +75,10 @@ include: "rules/qc.smk"
 
 rule all:
     input:
-        tck_files=inputs["T1w"].expand(
+        tck_files=inputs_t1w["T1w"].expand(
             rules.filtered_tck2connectome.output.sl_assignment
         ),
-        dti_files=inputs["T1w"].expand(rules.dwi2tensor.output.dti),
+        dti_files=inputs_t1w["T1w"].expand(rules.dwi2tensor.output.dti),
         qc_files=rules.gather_qc.input,
     params:
         mrtrix_dir=mrtrix_dir,

scattr/workflow/rules/freesurfer.smk

@@ -19,13 +19,13 @@ bids_fs_out = partial(
     bids,
     root=freesurfer_dir,
     datatype="anat",
-    **inputs.subj_wildcards,
+    **inputs_t1w.subj_wildcards,
 )
 
 bids_log = partial(
     bids,
     root=log_dir,
-    **inputs.subj_wildcards,
+    **inputs_t1w.subj_wildcards,
 )
 
 """Freesurfer references (with additional in rules as necessary)
@@ -73,10 +73,12 @@ rule thalamic_segmentation:
         freesurfer_dir=freesurfer_dir,
     params:
         fs_license=fs_license,
-        subj_dir=str(Path(bids(**inputs.subj_wildcards)).parent),
+        subj_dir=str(Path(bids(**inputs_t1w.subj_wildcards)).parent),
     output:
         thal_seg=str(
-            Path(bids(root=freesurfer_dir, **inputs.subj_wildcards)).parent
+            Path(
+                bids(root=freesurfer_dir, **inputs_t1w.subj_wildcards)
+            ).parent
             / "mri"
             / "ThalamicNuclei.v12.T1.mgz"
         ),
@@ -113,7 +115,9 @@ rule mgz2nii:
         if not config.get("skip_thal_seg")
         else [],
         aparcaseg=str(
-            Path(bids(root=freesurfer_dir, **inputs.subj_wildcards)).parent
+            Path(
+                bids(root=freesurfer_dir, **inputs_t1w.subj_wildcards)
+            ).parent
             / "mri"
             / "aparc+aseg.mgz"
         ),
@@ -158,7 +162,7 @@ rule fs_xfm_to_native:
     input:
         thal=rules.mgz2nii.output.thal,
         aparcaseg=rules.mgz2nii.output.aparcaseg,
-        ref=lambda wildcards: inputs["T1w"].filter(**wildcards).expand()[0],
+        ref=lambda wildcards: inputs_t1w["T1w"].filter(**wildcards).expand()[0],
     output:
         thal=bids_fs_out(
             space="T1w",

scattr/workflow/rules/mrtpipelines.smk

@@ -4,7 +4,6 @@ import numpy as np
 
 # Directories
 responsemean_dir = config.get("responsemean_dir")
-dwi_dir = config.get("dwi_dir")
 mrtrix_dir = str(Path(config["output_dir"]) / "mrtrix")
 labelmerge_dir = str(Path(config["output_dir"]) / "labelmerge")
 zona_dir = str(Path(config["output_dir"]) / "zona_bb_subcortex")
@@ -19,15 +18,6 @@ lmax = config.get("lmax")
 
 
 # BIDS partials
-bids_dwi = partial(
-    bids,
-    root=dwi_dir,
-    datatype="dwi",
-    space="T1w",
-    desc="preproc",
-    **inputs.subj_wildcards,
-)
-
 bids_response_out = partial(
     bids,
     root=mrtrix_dir,
@@ -40,35 +30,35 @@ bids_dti_out = partial(
     root=mrtrix_dir,
     datatype="dti",
     model="dti",
-    **inputs.subj_wildcards,
+    **inputs_dwi.subj_wildcards,
 )
 
 bids_tractography_out = partial(
     bids,
     root=mrtrix_dir,
     datatype="tractography",
-    **inputs.subj_wildcards,
+    **inputs_dwi.subj_wildcards,
 )
 
 bids_anat_out = partial(
     bids,
     root=mrtrix_dir,
     datatype="anat",
-    **inputs.subj_wildcards,
+    **inputs_t1w.subj_wildcards,
 )
 
 bids_labelmerge = partial(
     bids,
     root=str(Path(labelmerge_dir) / "combined")
     if not config.get("skip_labelmerge")
     else config.get("labelmerge_base_dir") or zona_dir,
-    **inputs.subj_wildcards,
+    **inputs_t1w.subj_wildcards,
 )
 
 bids_log = partial(
     bids,
     root=log_dir,
-    **inputs.subj_wildcards,
+    **inputs_dwi.subj_wildcards,
 )
 
 """Mrtrix3 reference (additional citations are included per rule as necessary):