From d3eae83d792cafe7a44af698473a2e5e84d3a2b5 Mon Sep 17 00:00:00 2001 From: Jason Kai Date: Tue, 12 Sep 2023 09:59:06 -0400 Subject: [PATCH 1/5] split generate_inputs for T1w and dwi Previously, if dwi_dir was passed, a naming scheme as assumed. Rather than hardcode the naming scheme, this change passes dwi_dir through to generate_inputs, which should enable different name schemes to be picked up. Bonus, it suppresses the message of not being able to find files, unless it actually is unable to find data in the provided paths. By default, if dwi_dir is not provided, it falls back on bids_dir. --- scattr/config/snakebids.yml | 1 + scattr/workflow/Snakefile | 23 ++++---- scattr/workflow/rules/freesurfer.smk | 12 ++--- scattr/workflow/rules/mrtpipelines.smk | 20 ++----- .../workflow/rules/mrtpipelines/preproc.smk | 54 +++++++------------ .../rules/mrtpipelines/tractography.smk | 4 +- scattr/workflow/rules/qc.smk | 16 +++--- scattr/workflow/rules/zona_bb_subcortex.smk | 16 +++--- 8 files changed, 62 insertions(+), 84 deletions(-) diff --git a/scattr/config/snakebids.yml b/scattr/config/snakebids.yml index 1277aa7e..0fcd3044 100644 --- a/scattr/config/snakebids.yml +++ b/scattr/config/snakebids.yml @@ -34,6 +34,7 @@ pybids_inputs: - part - reconstruction - run +pybids_inputs_dwi: dwi: filters: suffix: "dwi" diff --git a/scattr/workflow/Snakefile b/scattr/workflow/Snakefile index 4ff5fe7d..4c726648 100644 --- a/scattr/workflow/Snakefile +++ b/scattr/workflow/Snakefile @@ -12,7 +12,7 @@ configfile: "config/snakebids.yml" # writes inputs_config.yml and updates config dict -inputs = generate_inputs( +inputs_t1w = generate_inputs( bids_dir=config["bids_dir"], pybids_inputs=config["pybids_inputs"], pybids_config=["bids", "derivatives"], @@ -20,6 +20,14 @@ inputs = generate_inputs( participant_label=config["participant_label"], exclude_participant_label=config["exclude_participant_label"], ) +inputs_dwi = generate_inputs( + bids_dir=config["dwi_dir"] if config["dwi_dir"] else config["bids_dir"], + pybids_inputs=config["pybids_inputs_dwi"], + pybids_config=["bids", "derivatives"], + derivatives=config["derivatives"], + participant_label=config["participant_label"], + exclude_participant_label=config["exclude_participant_label"], +) # this adds constraints to the bids naming @@ -33,13 +41,6 @@ import os from pathlib import Path from functools import partial -# Relevant wildcards (upgrade for filtering) -subj_zip_list = { - k: v - for k, v in inputs["T1w"].zip_lists.items() - if k in ["subject", "session"] -} - # Warnings if config.get("labelmerge_base_dir") or config.get("labelmerge_overlay_dir"): print( @@ -49,7 +50,7 @@ if config.get("labelmerge_base_dir") or config.get("labelmerge_overlay_dir"): """ ) -if len(inputs.sessions) > 1 and not config.get("responsemean_ses"): +if len(inputs_dwi.sessions) > 1 and not config.get("responsemean_ses"): print( """ WARNING: Multiple sessions detected - average response function will be @@ -68,10 +69,10 @@ include: "rules/qc.smk" rule all: input: - tck_files=inputs["T1w"].expand( + tck_files=inputs_t1w["T1w"].expand( rules.filtered_tck2connectome.output.sl_assignment ), - dti_files=inputs["T1w"].expand(rules.dwi2tensor.output.dti), + dti_files=inputs_t1w["T1w"].expand(rules.dwi2tensor.output.dti), qc_files=rules.gather_qc.input, params: mrtrix_dir=mrtrix_dir, diff --git a/scattr/workflow/rules/freesurfer.smk b/scattr/workflow/rules/freesurfer.smk index 721ac171..92f8c7bf 100644 --- a/scattr/workflow/rules/freesurfer.smk +++ b/scattr/workflow/rules/freesurfer.smk @@ -19,13 +19,13 @@ bids_fs_out = partial( bids, root=freesurfer_dir, datatype="anat", - **inputs.subj_wildcards, + **inputs_t1w.subj_wildcards, ) bids_log = partial( bids, root=log_dir, - **inputs.subj_wildcards, + **inputs_t1w.subj_wildcards, ) """Freesurfer references (with additional in rules as necessary) @@ -73,10 +73,10 @@ rule thalamic_segmentation: freesurfer_dir=freesurfer_dir, params: fs_license=fs_license, - subj_dir=str(Path(bids(**inputs.subj_wildcards)).parent), + subj_dir=str(Path(bids(**inputs_t1w.subj_wildcards)).parent), output: thal_seg=str( - Path(bids(root=freesurfer_dir, **inputs.subj_wildcards)).parent + Path(bids(root=freesurfer_dir, **inputs_t1w.subj_wildcards)).parent / "mri" / "ThalamicNuclei.v12.T1.mgz" ), @@ -113,7 +113,7 @@ rule mgz2nii: if not config.get("skip_thal_seg") else [], aparcaseg=str( - Path(bids(root=freesurfer_dir, **inputs.subj_wildcards)).parent + Path(bids(root=freesurfer_dir, **inputs_t1w.subj_wildcards)).parent / "mri" / "aparc+aseg.mgz" ), @@ -158,7 +158,7 @@ rule fs_xfm_to_native: input: thal=rules.mgz2nii.output.thal, aparcaseg=rules.mgz2nii.output.aparcaseg, - ref=lambda wildcards: inputs["T1w"].filter(**wildcards).expand()[0], + ref=lambda wildcards: inputs_t1w["T1w"].filter(**wildcards).expand()[0], output: thal=bids_fs_out( space="T1w", diff --git a/scattr/workflow/rules/mrtpipelines.smk b/scattr/workflow/rules/mrtpipelines.smk index b4c60ac9..42d3b39f 100644 --- a/scattr/workflow/rules/mrtpipelines.smk +++ b/scattr/workflow/rules/mrtpipelines.smk @@ -4,7 +4,6 @@ import numpy as np # Directories responsemean_dir = config.get("responsemean_dir") -dwi_dir = config.get("dwi_dir") mrtrix_dir = str(Path(config["output_dir"]) / "mrtrix") labelmerge_dir = str(Path(config["output_dir"]) / "labelmerge") zona_dir = str(Path(config["output_dir"]) / "zona_bb_subcortex") @@ -19,15 +18,6 @@ lmax = config.get("lmax") # BIDS partials -bids_dwi = partial( - bids, - root=dwi_dir, - datatype="dwi", - space="T1w", - desc="preproc", - **inputs.subj_wildcards, -) - bids_response_out = partial( bids, root=mrtrix_dir, @@ -40,21 +30,21 @@ bids_dti_out = partial( root=mrtrix_dir, datatype="dti", model="dti", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ) bids_tractography_out = partial( bids, root=mrtrix_dir, datatype="tractography", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ) bids_anat_out = partial( bids, root=mrtrix_dir, datatype="anat", - **inputs.subj_wildcards, + **inputs_t1w.subj_wildcards, ) bids_labelmerge = partial( @@ -62,13 +52,13 @@ bids_labelmerge = partial( root=str(Path(labelmerge_dir) / "combined") if not config.get("skip_labelmerge") else config.get("labelmerge_base_dir") or zona_dir, - **inputs.subj_wildcards, + **inputs_t1w.subj_wildcards, ) bids_log = partial( bids, root=log_dir, - **inputs.subj_wildcards, + **inputs_t1w.subj_wildcards, ) """Mrtrix3 reference (additional citations are included per rule as necessary): diff --git a/scattr/workflow/rules/mrtpipelines/preproc.smk b/scattr/workflow/rules/mrtpipelines/preproc.smk index 0d6257db..4bd344a9 100644 --- a/scattr/workflow/rules/mrtpipelines/preproc.smk +++ b/scattr/workflow/rules/mrtpipelines/preproc.smk @@ -1,35 +1,21 @@ -if dwi_dir: - print(f"Searching {dwi_dir} for dwi and mask images...") - - rule nii2mif: input: - dwi=(bids_dwi(suffix="dwi.nii.gz") if dwi_dir else inputs["dwi"].path), - bval=( - bids_dwi(suffix="dwi.bval") - if dwi_dir - else re.sub(".nii.gz", ".bval", inputs["dwi"].path) - ), - bvec=( - bids_dwi(suffix="dwi.bvec") - if dwi_dir - else re.sub(".nii.gz", ".bvec", inputs["dwi"].path) - ), - mask=( - bids_dwi(suffix="mask.nii.gz") if dwi_dir else inputs["mask"].path - ), + dwi=inputs_dwi["dwi"].path, + bval=re.sub(".nii.gz", ".bval", inputs_dwi["dwi"].path), + bvec=re.sub(".nii.gz", ".bvec", inputs_dwi["dwi"].path), + mask=inputs_dwi["mask"].path, output: dwi=bids( root=mrtrix_dir, datatype="dwi", suffix="dwi.mif", - **inputs.subj_wildcards + **inputs_dwi.subj_wildcards ), mask=bids( root=mrtrix_dir, datatype="dwi", suffix="brainmask.mif", - **inputs.subj_wildcards + **inputs_dwi.subj_wildcards ), threads: 4 resources: @@ -69,15 +55,15 @@ rule dwi2response: output: wm_rf=bids_response_out( desc="wm", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ), gm_rf=bids_response_out( desc="gm", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ), csf_rf=bids_response_out( desc="csf", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ), threads: 4 resources: @@ -102,13 +88,13 @@ rule dwi2response: def get_subject_rf(wildcards): """Get appropriate subject response path""" - if not inputs.sessions: + if not inputs_dwi.sessions: return expand( bids_response_out( subject="{subject}", desc="{tissue}", ), - subject=inputs.subjects, + subject=inputs_dwi.subjects, allow_missing=True, ) else: @@ -118,11 +104,11 @@ def get_subject_rf(wildcards): session="{session}", desc="{tissue}", ), - subject=inputs.subjects, + subject=inputs_dwi.subjects, session=( config.get("responsemean_ses") if config.get("responsemean_ses") - else inputs.sessions + else inputs_dwi.sessions ), allow_missing=True, ) @@ -197,19 +183,19 @@ rule dwi2fod: model="csd", desc="wm", suffix="fod.mif", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ), gm_fod=bids_response_out( model="csd", desc="gm", suffix="fod.mif", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ), csf_fod=bids_response_out( model="csd", desc="csf", suffix="fod.mif", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ), threads: 4 resources: @@ -252,19 +238,19 @@ rule mtnormalise: model="csd", desc="wm", suffix="fodNormalized.mif", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ), gm_fod=bids_response_out( model="csd", desc="gm", suffix="fodNormalized.mif", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ), csf_fod=bids_response_out( model="csd", desc="csf", suffix="fodNormalized.mif", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ), threads: 4 resources: @@ -299,7 +285,7 @@ rule dwinormalise: datatype="dwi", desc="normalized", suffix="dwi.mif", - **inputs.subj_wildcards, + **inputs_dwi.subj_wildcards, ), threads: 4 resources: diff --git a/scattr/workflow/rules/mrtpipelines/tractography.smk b/scattr/workflow/rules/mrtpipelines/tractography.smk index da99b24e..4e6612a7 100644 --- a/scattr/workflow/rules/mrtpipelines/tractography.smk +++ b/scattr/workflow/rules/mrtpipelines/tractography.smk @@ -89,7 +89,7 @@ checkpoint create_roi_mask: num_labels=rules.get_num_nodes.output.num_labels, params: base_dir=mrtrix_dir, - subj_wildcards=inputs.subj_wildcards, + subj_wildcards=inputs_dwi.subj_wildcards, output: out_dir=directory(bids_anat_out(datatype="roi_masks")), threads: 4 @@ -151,7 +151,7 @@ checkpoint create_exclude_mask: mask_dir=bids_anat_out( datatype="roi_masks", ), - subj_wildcards=inputs.subj_wildcards, + subj_wildcards=inputs_dwi.subj_wildcards, output: out_dir=directory(bids_anat_out(datatype="exclude_mask")), threads: 4 diff --git a/scattr/workflow/rules/qc.smk b/scattr/workflow/rules/qc.smk index 46feae66..fd638de6 100644 --- a/scattr/workflow/rules/qc.smk +++ b/scattr/workflow/rules/qc.smk @@ -4,7 +4,7 @@ bids_labelmerge = partial( root=str(Path(labelmerge_dir) / "combined") if not config.get("skip_labelmerge") else config.get("labelmerge_base_dir") or zona_dir, - **inputs.subj_wildcards, + **inputs_t1w.subj_wildcards, ) bids_qc = partial( @@ -12,7 +12,7 @@ bids_qc = partial( root=str(Path(config["output_dir"]) / "qc"), datatype="anat", space="T1w", - **inputs.subj_wildcards, + **inputs_t1w.subj_wildcards, ) @@ -26,7 +26,7 @@ rule segment_qc: else config.get("labelmerge_base_desc"), suffix="dseg.nii.gz", ), - t1w_image=lambda wildcards: inputs["T1w"] + t1w_image=lambda wildcards: inputs_t1w["T1w"] .filter(**wildcards) .expand()[0], output: @@ -66,7 +66,7 @@ rule segment_qc: rule registration_qc: input: moving_nii=rules.reg2native.output.t1w_nativespace, - fixed_nii=lambda wildcards: inputs["T1w"] + fixed_nii=lambda wildcards: inputs_t1w["T1w"] .filter(**wildcards) .expand()[0], params: @@ -103,7 +103,7 @@ rule registration_qc: rule gather_qc: input: - dseg_png=inputs["T1w"].expand(rules.segment_qc.output.qc_png), - dseg_html=inputs["T1w"].expand(rules.segment_qc.output.qc_html), - reg_svg=inputs["T1w"].expand(rules.registration_qc.output.qc_svg), - reg_html=inputs["T1w"].expand(rules.registration_qc.output.qc_html), + dseg_png=inputs_t1w["T1w"].expand(rules.segment_qc.output.qc_png), + dseg_html=inputs_t1w["T1w"].expand(rules.segment_qc.output.qc_html), + reg_svg=inputs_t1w["T1w"].expand(rules.registration_qc.output.qc_svg), + reg_html=inputs_t1w["T1w"].expand(rules.registration_qc.output.qc_html), diff --git a/scattr/workflow/rules/zona_bb_subcortex.smk b/scattr/workflow/rules/zona_bb_subcortex.smk index 68a7b487..52282937 100644 --- a/scattr/workflow/rules/zona_bb_subcortex.smk +++ b/scattr/workflow/rules/zona_bb_subcortex.smk @@ -12,7 +12,7 @@ bids_anat = partial( bids, root=zona_dir, datatype="anat", - **inputs.subj_wildcards, + **inputs_t1w.subj_wildcards, ) bids_labelmerge = partial( @@ -20,13 +20,13 @@ bids_labelmerge = partial( root=str(Path(labelmerge_dir) / "combined") if not config.get("skip_labelmerge") else config.get("labelmerge_base_dir") or zona_dir, - **inputs.subj_wildcards, + **inputs_t1w.subj_wildcards, ) bids_log = partial( bids, root=log_dir, - **inputs.subj_wildcards, + **inputs_t1w.subj_wildcards, ) """ References: @@ -71,7 +71,7 @@ rule reg2native: / Path(config["zona_bb_subcortex"][config["Space"]]["dir"]) / Path(config["zona_bb_subcortex"][config["Space"]]["T1w"]) ), - target=lambda wildcards: inputs["T1w"].filter(**wildcards).expand()[0], + target=lambda wildcards: inputs_t1w["T1w"].filter(**wildcards).expand()[0], params: out_dir=directory(str(Path(bids_anat()).parent)), out_prefix=bids_anat( @@ -153,12 +153,12 @@ rule warp2native: rule labelmerge: input: - zona_seg=inputs["T1w"].expand( + zona_seg=inputs_t1w["T1w"].expand( rules.warp2native.output.nii, allow_missing=True ) if not config.get("labelmerge_base_dir") else [], - fs_seg=inputs["T1w"].expand( + fs_seg=inputs_t1w["T1w"].expand( rules.fs_xfm_to_native.output.thal, allow_missing=True ) if not config.get("labelmerge_overlay_dir") @@ -198,7 +198,7 @@ rule labelmerge: else "" ), output: - seg=inputs["T1w"].expand( + seg=inputs_t1w["T1w"].expand( bids_labelmerge( space="T1w", desc="combined", @@ -206,7 +206,7 @@ rule labelmerge: ), allow_missing=True, ), - tsv=inputs["T1w"].expand( + tsv=inputs_t1w["T1w"].expand( bids_labelmerge( space="T1w", desc="combined", From c2b3eb21b6bb294979029577ce937158fef046ba Mon Sep 17 00:00:00 2001 From: Jason Kai Date: Tue, 12 Sep 2023 10:15:48 -0400 Subject: [PATCH 2/5] add snakedwi naming scheme for testing --- pyproject.toml | 7 +++++++ .../dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.bval | 0 .../dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.bvec | 0 .../dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.nii.gz | 0 .../dwi/sub-001_space-T1w_desc-eddy_res-orig_mask.nii.gz | 0 5 files changed, 7 insertions(+) create mode 100644 test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.bval create mode 100644 test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.bvec create mode 100644 test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.nii.gz create mode 100644 test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_mask.nii.gz diff --git a/pyproject.toml b/pyproject.toml index f6b9d92c..bf4d376e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,6 +100,13 @@ python ./scattr/run.py ./test/data/bids_nodwi test/data/derivatives/ \ --fs-license ./test/.fs_license -np --force-output """ +[tool.poe.tasks.test_snakedwi] +shell = """ +python ./scattr/run.py ./test/data/bids_nodwi test/data/derivatives/ \ + participant --dwi_dir ./test/data/derivatives/snakedwi \ + --fs-license ./test/.fs_license -np --force-output +""" + [tool.poe.tasks.test_labelmerge] shell = """ python ./scattr/run.py ./test/data/bids test/data/derivatives/ participant \ diff --git a/test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.bval b/test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.bval new file mode 100644 index 00000000..e69de29b diff --git a/test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.bvec b/test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.bvec new file mode 100644 index 00000000..e69de29b diff --git a/test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.nii.gz b/test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_dwi.nii.gz new file mode 100644 index 00000000..e69de29b diff --git a/test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_mask.nii.gz b/test/data/derivatives/snakedwi/sub-001/dwi/sub-001_space-T1w_desc-eddy_res-orig_mask.nii.gz new file mode 100644 index 00000000..e69de29b From 0998cc76ffead029b1130295469dba707686a3c5 Mon Sep 17 00:00:00 2001 From: Jason Kai Date: Tue, 12 Sep 2023 10:42:30 -0400 Subject: [PATCH 3/5] update docs and add pybidsdb options --- docs/outputs/output_files.md | 43 ++++++++++++++++++------------------ scattr/config/snakebids.yml | 6 +++++ scattr/workflow/Snakefile | 4 ++++ 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/docs/outputs/output_files.md b/docs/outputs/output_files.md index 57375b3e..e4da6f09 100644 --- a/docs/outputs/output_files.md +++ b/docs/outputs/output_files.md @@ -1,9 +1,9 @@ # Output Files -After running the workflow, the `/path/to/output/dir` folder will contain a -hidden `.logs` folder for troubleshooting, as well as additional folders -associated with the tools that were used to generate files (e.g. `freesurfer`, -`labelmerge`), but for most purposes, all the primary outputs of interest will +After running the workflow, the `/path/to/output/dir` folder will contain a +hidden `.logs` folder for troubleshooting, as well as additional folders +associated with the tools that were used to generate files (e.g. `freesurfer`, +`labelmerge`), but for most purposes, all the primary outputs of interest will be in the `mrtrix` directory with the following structure: ``` @@ -15,16 +15,16 @@ mrtrix/ └── tractography ``` -Briefly, `dti` contains processed diffusion tensor images along with +Briefly, `dti` contains processed diffusion tensor images along with quantitative maps (e.g. `FA`, `MD`, etc.), `dwi` contains images that were -converted from Nifti (`.nii.gz`) to the MRtrix imaging format (`.mif`), +converted from Nifti (`.nii.gz`) to the MRtrix imaging format (`.mif`), `response` contains subject-specific response functions, and `tractography` contains tractogram files. ## dti -This folder contains processed diffusion tensor images (DTI) that were used to -compute quantitative maps (stored in the MRtrix imaging format - `.mif`), also +This folder contains processed diffusion tensor images (DTI) that were used to +compute quantitative maps (stored in the MRtrix imaging format - `.mif`), also found within the folder: ``` @@ -38,12 +38,12 @@ sub-{subject} ``` As per the BIDS extension proposal, `model-dti` denotes to the DTI model used to -process the data, and the suffix (`ad`, `fa`, `md`, `rd`, `tensor`) describes +process the data, and the suffix (`ad`, `fa`, `md`, `rd`, `tensor`) describes the imaging data. ## dwi -This folder contains the input diffusion weighted image (dwi) and brain mask +This folder contains the input diffusion weighted image (dwi) and brain mask converted from Nifti (`.nii.gz`) to the MRtrix imaging format (`.mif`) used in downstream processing. It also contains an intensity normalized dwi image used for DTI: @@ -51,7 +51,7 @@ for DTI: ``` sub-{subject} └── dwi - ├── sub-{subject}_brainmask.mif + ├── sub-{subject}_desc-brain_mask.mif ├── sub-{subject}_desc-normalized_dwi.mif └── sub-{subject}_dwi.mif ``` @@ -60,7 +60,7 @@ sub-{subject} This folder contains the subject-specific response functions that were estimated from the input diffusion data, as well as both the normalized and unnormalized -versions of the estimated fibre orientation distribution (FOD) maps. Both +versions of the estimated fibre orientation distribution (FOD) maps. Both response functions and FOD maps are estimated for the three different tissue types: white matter (wm), grey matter (gm), and corticospinal fluid (csf): @@ -96,19 +96,20 @@ sub-{subject} ├── sub-{subject}_desc-subcortical_nodeWeights.csv ``` -The `desc-filteredsubcortical` entity pair describes associated tractogram -files that have been filtered to only pass through WM and connect two GM +The `desc-filteredsubcortical` entity pair describes associated tractogram +files that have been filtered to only pass through WM and connect two GM structures, while the `desc-subcortical` entity pair denotes files associated with the unfiltered subcortical connectome. Whole-brain tractogram associated files are denoted by `desc-iFOD2`, which describes the tractography algorithm used to perform tractograpy. -_Note: As the BIDS extension proposal has not been finalized, the naming of +_Note: As the BIDS extension proposal has not been finalized, the naming of such files may be subject to change_ ## Additional Files The top-level `/path/to/output/dir` contains additional files / folders: + ``` /path/to/output/dir ├── ... @@ -118,13 +119,13 @@ The top-level `/path/to/output/dir` contains additional files / folders: └── .snakemake ``` -The `config` folder, along with the hidden `.snakebids` and `.snakemake` folders +The `config` folder, along with the hidden `.snakebids` and `.snakemake` folders contain a record of the code and parameters used, and paths to the inputs. -Workflow steps that write logs to file are stored in the hidden `.logs` -subfolder, with the file names based on the tools used (e.g. `mrtrix`) and rule -wildcards (e.g. `subject`). +Workflow steps that write logs to file are stored in the hidden `.logs` +subfolder, with the file names based on the tools used (e.g. `mrtrix`) and rule +wildcards (e.g. `subject`). -If the app is run in workflow mode (`--workflow-mode` / `-W`), which enables +If the app is run in workflow mode (`--workflow-mode` / `-W`), which enables direct use of the Snakemake CLI to run scattr, output folders (e.g. `work`) will -be placed in a `results` folder. \ No newline at end of file +be placed in a `results` folder. diff --git a/scattr/config/snakebids.yml b/scattr/config/snakebids.yml index 0fcd3044..7f4241fd 100644 --- a/scattr/config/snakebids.yml +++ b/scattr/config/snakebids.yml @@ -50,6 +50,7 @@ pybids_inputs_dwi: - part - reconstruction - run + - res mask: filters: suffix: "mask" @@ -119,6 +120,11 @@ parse_args: nargs: "?" type: Path + --pybidsdb_dwi_dir: + help: "The path to the pybids database associated with provided dwi_dir" + nargs: "?" + type: Path + --responsemean_dir: help: "The path to the directory containing average response functions. If not provided, one will be computed from the subjects in the input diff --git a/scattr/workflow/Snakefile b/scattr/workflow/Snakefile index 4c726648..a53b06bb 100644 --- a/scattr/workflow/Snakefile +++ b/scattr/workflow/Snakefile @@ -16,6 +16,8 @@ inputs_t1w = generate_inputs( bids_dir=config["bids_dir"], pybids_inputs=config["pybids_inputs"], pybids_config=["bids", "derivatives"], + pybidsdb_dir=config.get("pybidsdb_dir"), + pybidsdb_reset=config.get("pybidsdb_reset"), derivatives=config["derivatives"], participant_label=config["participant_label"], exclude_participant_label=config["exclude_participant_label"], @@ -24,6 +26,8 @@ inputs_dwi = generate_inputs( bids_dir=config["dwi_dir"] if config["dwi_dir"] else config["bids_dir"], pybids_inputs=config["pybids_inputs_dwi"], pybids_config=["bids", "derivatives"], + pybidsdb_dir=config.get("pybidsdb_dwi_dir") if config["pybidsdb_dwi_dir"] else config.get("pybidsdb_dir"), + pybidsdb_reset=config.get("pybidsdb_reset"), derivatives=config["derivatives"], participant_label=config["participant_label"], exclude_participant_label=config["exclude_participant_label"], From 57b43b6c2547e4a97ddb7e772844a4b9565486a9 Mon Sep 17 00:00:00 2001 From: Jason Kai Date: Tue, 12 Sep 2023 11:23:40 -0400 Subject: [PATCH 4/5] update config wildcards and filtering Previously, the usage of `wildcards` was vast in the config file, allowing for wildcards for all sorts of entities. This PR constrains the wildcards to just subject, session, and run (can consider others in the future, but these make the most sense at the moment). Dwi filters are constrained to be in the "T1w" space, as we assume dwi processing should be performed in the same space as T1w. Alternatively, can consider "orig" space as well, but will need to think about to handle transformations to the T1w in that case. If multiple entities are found (e.g. "res"), Snakebids will throw an error reflecting the files found and suggest to the user to make use of the `--filter` option to constrain the files. --- scattr/config/snakebids.yml | 13 +------------ scattr/workflow/Snakefile | 4 +++- scattr/workflow/rules/freesurfer.smk | 8 ++++++-- scattr/workflow/rules/mrtpipelines.smk | 2 +- scattr/workflow/rules/mrtpipelines/preproc.smk | 5 +++-- scattr/workflow/rules/zona_bb_subcortex.smk | 4 +++- 6 files changed, 17 insertions(+), 19 deletions(-) diff --git a/scattr/config/snakebids.yml b/scattr/config/snakebids.yml index 7f4241fd..90132051 100644 --- a/scattr/config/snakebids.yml +++ b/scattr/config/snakebids.yml @@ -30,9 +30,6 @@ pybids_inputs: wildcards: - subject - session - - acquisition - - part - - reconstruction - run pybids_inputs_dwi: dwi: @@ -41,27 +38,19 @@ pybids_inputs_dwi: extension: ".nii.gz" datatype: "dwi" space: "T1w" - part: ["mag", false] wildcards: - subject - session - - acquisition - - direction - - part - - reconstruction - run - - res mask: filters: suffix: "mask" extension: ".nii.gz" datatype: "dwi" - space: ["orig", "scanner", "individual", "T1w"] + space: "T1w" wildcards: - subject - session - - acquisition - - res - run # Configuration for the command-line parameters to make available diff --git a/scattr/workflow/Snakefile b/scattr/workflow/Snakefile index a53b06bb..d00fca75 100644 --- a/scattr/workflow/Snakefile +++ b/scattr/workflow/Snakefile @@ -26,7 +26,9 @@ inputs_dwi = generate_inputs( bids_dir=config["dwi_dir"] if config["dwi_dir"] else config["bids_dir"], pybids_inputs=config["pybids_inputs_dwi"], pybids_config=["bids", "derivatives"], - pybidsdb_dir=config.get("pybidsdb_dwi_dir") if config["pybidsdb_dwi_dir"] else config.get("pybidsdb_dir"), + pybidsdb_dir=config.get("pybidsdb_dwi_dir") + if config["pybidsdb_dwi_dir"] + else config.get("pybidsdb_dir"), pybidsdb_reset=config.get("pybidsdb_reset"), derivatives=config["derivatives"], participant_label=config["participant_label"], diff --git a/scattr/workflow/rules/freesurfer.smk b/scattr/workflow/rules/freesurfer.smk index 92f8c7bf..84162863 100644 --- a/scattr/workflow/rules/freesurfer.smk +++ b/scattr/workflow/rules/freesurfer.smk @@ -76,7 +76,9 @@ rule thalamic_segmentation: subj_dir=str(Path(bids(**inputs_t1w.subj_wildcards)).parent), output: thal_seg=str( - Path(bids(root=freesurfer_dir, **inputs_t1w.subj_wildcards)).parent + Path( + bids(root=freesurfer_dir, **inputs_t1w.subj_wildcards) + ).parent / "mri" / "ThalamicNuclei.v12.T1.mgz" ), @@ -113,7 +115,9 @@ rule mgz2nii: if not config.get("skip_thal_seg") else [], aparcaseg=str( - Path(bids(root=freesurfer_dir, **inputs_t1w.subj_wildcards)).parent + Path( + bids(root=freesurfer_dir, **inputs_t1w.subj_wildcards) + ).parent / "mri" / "aparc+aseg.mgz" ), diff --git a/scattr/workflow/rules/mrtpipelines.smk b/scattr/workflow/rules/mrtpipelines.smk index 42d3b39f..181d34b7 100644 --- a/scattr/workflow/rules/mrtpipelines.smk +++ b/scattr/workflow/rules/mrtpipelines.smk @@ -58,7 +58,7 @@ bids_labelmerge = partial( bids_log = partial( bids, root=log_dir, - **inputs_t1w.subj_wildcards, + **inputs_dwi.subj_wildcards, ) """Mrtrix3 reference (additional citations are included per rule as necessary): diff --git a/scattr/workflow/rules/mrtpipelines/preproc.smk b/scattr/workflow/rules/mrtpipelines/preproc.smk index 4bd344a9..b0f717d2 100644 --- a/scattr/workflow/rules/mrtpipelines/preproc.smk +++ b/scattr/workflow/rules/mrtpipelines/preproc.smk @@ -14,7 +14,8 @@ rule nii2mif: mask=bids( root=mrtrix_dir, datatype="dwi", - suffix="brainmask.mif", + desc="brain", + suffix="mask.mif", **inputs_dwi.subj_wildcards ), threads: 4 @@ -22,7 +23,7 @@ rule nii2mif: mem_mb=16000, time=10, log: - bids_log(suffix="nii2mif.log"), + bids_log(suffix="nii2mif.log", **inputs_dwi.subj_wildcards), group: "dwiproc" container: diff --git a/scattr/workflow/rules/zona_bb_subcortex.smk b/scattr/workflow/rules/zona_bb_subcortex.smk index 52282937..681c6d4f 100644 --- a/scattr/workflow/rules/zona_bb_subcortex.smk +++ b/scattr/workflow/rules/zona_bb_subcortex.smk @@ -71,7 +71,9 @@ rule reg2native: / Path(config["zona_bb_subcortex"][config["Space"]]["dir"]) / Path(config["zona_bb_subcortex"][config["Space"]]["T1w"]) ), - target=lambda wildcards: inputs_t1w["T1w"].filter(**wildcards).expand()[0], + target=lambda wildcards: inputs_t1w["T1w"] + .filter(**wildcards) + .expand()[0], params: out_dir=directory(str(Path(bids_anat()).parent)), out_prefix=bids_anat( From d546c63812160c8f825699c7852b9a001df964b0 Mon Sep 17 00:00:00 2001 From: Jason Kai Date: Fri, 15 Sep 2023 10:23:09 -0400 Subject: [PATCH 5/5] address tkkuehn's review --- scattr/workflow/Snakefile | 2 +- .../derivatives_sessions/config/snakebids.yml | 69 +++++++++++-------- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/scattr/workflow/Snakefile b/scattr/workflow/Snakefile index d00fca75..3ba72411 100644 --- a/scattr/workflow/Snakefile +++ b/scattr/workflow/Snakefile @@ -27,7 +27,7 @@ inputs_dwi = generate_inputs( pybids_inputs=config["pybids_inputs_dwi"], pybids_config=["bids", "derivatives"], pybidsdb_dir=config.get("pybidsdb_dwi_dir") - if config["pybidsdb_dwi_dir"] + if config["dwi_dir"] else config.get("pybidsdb_dir"), pybidsdb_reset=config.get("pybidsdb_reset"), derivatives=config["derivatives"], diff --git a/test/data/derivatives_sessions/config/snakebids.yml b/test/data/derivatives_sessions/config/snakebids.yml index 6c30f9e9..606d5557 100644 --- a/test/data/derivatives_sessions/config/snakebids.yml +++ b/test/data/derivatives_sessions/config/snakebids.yml @@ -1,5 +1,5 @@ -bids_dir: /scratch/tkai/0_dev/scattr/test/data/bids_sessions -output_dir: /scratch/tkai/0_dev/scattr/test/data/derivatives_sessions +bids_dir: /home/jkai/git/scattr/test/data/bids_sessions +output_dir: /home/jkai/git/scattr/test/data/derivatives_sessions snakebids_dir: . debug: false derivatives: false @@ -15,12 +15,14 @@ pybids_inputs: extension: .nii.gz datatype: anat desc: brain + part: + - mag + - false wildcards: - subject - session - - acquisition - - reconstruction - run +pybids_inputs_dwi: dwi: filters: suffix: dwi @@ -30,25 +32,16 @@ pybids_inputs: wildcards: - subject - session - - acquisition - - direction - - reconstruction - run mask: filters: suffix: mask extension: .nii.gz datatype: dwi - space: - - orig - - scanner - - individual - - T1w + space: T1w wildcards: - subject - session - - acquisition - - res - run parse_args: bids_dir: @@ -82,31 +75,41 @@ parse_args: help: The path to the freesurfer directory. If not provided, workflow assumes the directory exists at /freesurfer. nargs: '?' - type: &id002 !!python/name:pathlib.Path '' + type: Path --dwi_dir: help: The path to the directory containing pre-processed dwi data transformed to subject T1w space. If not provided, workflow assumes this data exists in //dwi. nargs: '?' - type: *id002 + type: Path + --pybidsdb_dwi_dir: + help: The path to the pybids database associated with provided dwi_dir + nargs: '?' + type: Path --responsemean_dir: help: The path to the directory containing average response functions. If not provided, one will be computed from the subjects in the input directory. nargs: '?' - type: *id002 + type: Path + --responsemean_ses: + help: The session used to compute the average response function. If multiple are + available in the dataset, it is HIGHLY RECOMMENDED to set this to a specific + session. If not provided, the average response function will be computed with + all possible sessions. + nargs: '?' --fs_license: help: Path to Freesurfer license file. If not provided, workflow will check FS_LICENSE environment variable for one. nargs: '?' - type: *id002 + type: Path --labelmerge_base_dir: help: BIDS directory containing base input labelmaps. nargs: '?' - type: *id002 + type: Path --labelmerge_overlay_dir: help: BIDS directory containing overlay input labelmaps. nargs: '?' - type: *id002 + type: Path --labelmerge_base_desc: help: 'Description entity for base labelmaps used in labelmerge. By default, uses ZonaBB entity as intended in the subcortical connectome workflow (default: ZonaBB).' @@ -151,6 +154,11 @@ parse_args: step. action: store_true default: false + --bzero_thresh: + help: Set the threshold for a shell to be considered b=0. By default, this value + is set to 10 + nargs: '?' + default: 10 --shells: help: '(Mrtrix3) specify one or more b-values to use during processing, as a space-separated list of the desired approximate b-values (b-values are clustered to allow for @@ -201,21 +209,14 @@ zona_bb_subcortex: freesurfer: tsv: resources/freesurfer/desc-FreesurferThal_dseg.tsv singularity: - freesurfer: docker://pwighton/freesurfer:7.2.0 - neuroglia-core: docker://khanlab/neuroglia-core:latest - ants: docker://kaczmarj/ants:2.3.4 - mrtrix: docker://brainlife/mrtrix3:3.0.3 - labelmerge: docker://khanlab/labelmerge:v0.4.3 - scattr: docker://khanlab/scattr:v0.1.2 -fs_license: /scratch/tkai/0_dev/scattr/test.fs_license -snakemake_dir: /scratch/tkai/0_dev/scattr/scattr -snakefile: /scratch/tkai/0_dev/scattr/scattr/workflow/Snakefile -pybids_db_reset: false + scattr: docker://khanlab/scattr:v0.2.1 +fs_license: /home/jkai/git/scattr/test.fs_license snakemake_args: - -np workflow_mode: false force_conversion: false pybidsdb_dir: null +pybidsdb_reset: false reset_db: false force_output: true retrofit: false @@ -226,7 +227,9 @@ exclude_participant_label: null slurm_tmpdir: false freesurfer_dir: null dwi_dir: null +pybidsdb_dwi_dir: null responsemean_dir: null +responsemean_ses: null labelmerge_base_dir: null labelmerge_overlay_dir: null labelmerge_base_desc: ZonaBB @@ -238,9 +241,15 @@ labelmerge_overlay_exceptions: null skip_labelmerge: false skip_brainstem: false skip_thal_seg: false +bzero_thresh: 10 shells: null lmax: null step: 0.35 sl_count: 20000000 radial_search: 1.5 +snakemake_dir: /home/jkai/git/scattr/scattr +snakefile: /home/jkai/git/scattr/scattr/workflow/Snakefile root: '' +snakemake_version: 7.32.3 +snakebids_version: 0.9.2 +app_version: 0.3.0rc9