diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..df656a8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +.idea +*/__pycache__/ +venv/ diff --git a/README.md b/README.md index 68b0f28..9b8d9c5 100644 --- a/README.md +++ b/README.md @@ -1 +1,44 @@ -# manual-correction \ No newline at end of file +# Manual correction + +This repository contains scripts for the manual correction of spinal cord labels. Currently supported labels are: +- spinal cord segmentation +- gray matter segmentation +- disc labels +- ponto-medullary junction (PMJ) label + +## Installation + +```console +git clone https://github.com/spinalcordtoolbox/manual-correction.git +cd manual-correction +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +## Usage + +> **Note** All scripts currently assume BIDS-compliant data. For more information about the BIDS standard, please visit http://bids.neuroimaging.io. + +### `package_for_correction.py` + +The `package_for_correction.py` script is used to create a zip file containing the processed images and labels +(segmentations, disc labels, etc.). The zip file is then sent to the user for manual correction. + +This is useful when you need to correct the labels of a large dataset processed on a remote server. In this case, you do +not need to copy the whole dataset. Instead, only the images and labels that need to be corrected are zipped. The yaml +list of images to correct can be obtained from the [SCT QC html report](https://spinalcordtoolbox.com/overview/concepts/inspecting-results-qc-fsleyes.html#how-do-i-use-the-qc-report). + +### `manual_correction.py` + +The `manual_correction.py` script is used to correct the labels (segmentations, disc labels, etc.) of a dataset. +The script takes as input a processed dataset and outputs the corrected labels to `derivatives/labels` folder. + +For the correction of spinal cord and gray matter segmentation, you can choose a viewer ([FSLeyes](https://open.win.ox.ac.uk/pages/fsl/fsleyes/fsleyes/userdoc/#), [ITK-SNAP](http://www.itksnap.org/pmwiki/pmwiki.php), [3D Slicer](https://www.slicer.org)). + +For the correction of vertebral labeling and ponto-medullary junction (PMJ), [sct_label_utils](https://github.com/spinalcordtoolbox/spinalcordtoolbox/blob/master/spinalcordtoolbox/scripts/sct_label_utils.py) is used. + +### `copy_files_to_derivatives.py` + +The `copy_files_to_derivatives.py` script is used to copy manually corrected labels (segmentations, disc labels, etc.) +from your local `derivatives/labels` folder to the already existing git-annex BIDS dataset's `derivatives/labels` folder. \ No newline at end of file diff --git a/copy_files_to_derivatives.py b/copy_files_to_derivatives.py new file mode 100644 index 0000000..2f95249 --- /dev/null +++ b/copy_files_to_derivatives.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# +# Copy manually corrected labels (segmentations, vertebral labeling, etc.) from the preprocessed dataset to the +# git-annex BIDS dataset's derivatives folder +# +# Authors: Jan Valosek +# + +import argparse +import glob +import os +import shutil +import utils + + +def get_parser(): + """ + parser function + """ + + parser = argparse.ArgumentParser( + description='Copy manually corrected files (segmentations, vertebral labeling, etc.) from the source ' + 'preprocessed dataset to the git-annex BIDS derivatives folder', + formatter_class=utils.SmartFormatter, + prog=os.path.basename(__file__).strip('.py') + ) + parser.add_argument( + '-path-in', + metavar="", + required=True, + type=str, + help='Path to the folder with manually corrected files (usually derivatives). The script assumes that labels ' + 'folder is located in the provided folder.' + ) + parser.add_argument( + '-path-out', + metavar="", + required=True, + type=str, + help='Path to the BIDS dataset where manually corrected files will be copied. Include also derivatives folder ' + 'in the path. Files will be copied to the derivatives/label folder.' + ) + + return parser + + +def main(): + + # Parse the command line arguments + parser = get_parser() + args = parser.parse_args() + + # Check if path_in exists + if os.path.isdir(args.path_in): + path_in = os.path.join(os.path.abspath(args.path_in), 'labels') + else: + raise NotADirectoryError(f'{args.path_in} does not exist.') + + # Check if path_out exists + if os.path.isdir(args.path_out): + path_out = os.path.join(os.path.abspath(args.path_out), 'labels') + else: + raise NotADirectoryError(f'{args.path_out} does not exist.') + + # Loop across files in input dataset + for path_file_in in sorted(glob.glob(path_in + '/**/*.nii.gz', recursive=True)): + sub, ses, filename, contrast = utils.fetch_subject_and_session(path_file_in) + # Construct path for the output file + path_file_out = os.path.join(path_out, sub, ses, contrast, filename) + # Check if subject's folder exists in the output dataset, if not, create it + path_subject_folder_out = os.path.join(path_out, sub, ses, contrast) + if not os.path.isdir(path_subject_folder_out): + os.makedirs(path_subject_folder_out) + print(f'Creating directory: {path_subject_folder_out}') + # Copy nii and json files to the output dataset + # TODO - consider rsync instead of shutil.copy + shutil.copy(path_file_in, path_file_out) + print(f'Copying: {path_file_in} to {path_file_out}') + path_file_json_in = path_file_in.replace('nii.gz', 'json') + path_file_json_out = path_file_out.replace('nii.gz', 'json') + shutil.copy(path_file_json_in, path_file_json_out) + print(f'Copying: {path_file_json_in} to {path_file_json_out}') + + +if __name__ == '__main__': + main() diff --git a/manual_correction.py b/manual_correction.py index 71aa581..2ca6005 100644 --- a/manual_correction.py +++ b/manual_correction.py @@ -1,11 +1,12 @@ #!/usr/bin/env python # -# Script to perform manual correction of segmentations and vertebral labeling. +# Script to perform manual correction of spinal cord segmentation, gray matter segmentation, vertebral labeling, and +# pontomedullary junction labeling. # # For usage, type: python manual_correction.py -h # -# Authors: Jan Valosek, Julien Cohen-Adad -# Adapted by Sandrine Bédard for cord CSA project UK Biobank +# Authors: Jan Valosek, Sandrine Bédard, Naga Karthik, Julien Cohen-Adad +# import argparse import coloredlogs @@ -16,12 +17,7 @@ import shutil from textwrap import dedent import time -import yaml -import pipeline_ukbiobank.utils as utils - -# Folder where to output manual labels, at the root of a BIDS dataset. -# TODO: make it an input argument (with default value) -FOLDER_DERIVATIVES = os.path.join('derivatives', 'labels') +import utils def get_parser(): @@ -29,8 +25,9 @@ def get_parser(): parser function """ parser = argparse.ArgumentParser( - description='Manual correction of spinal cord segmentation, vertebral and pontomedullary junction labeling. ' - 'Manually corrected files are saved under derivatives/ folder (BIDS standard).', + description='Manual correction of spinal cord segmentation, gray matter segmentation, multiple sclerosis ' + 'lesion segmentation, vertebral labeling, and pontomedullary junction labeling. ' + 'Manually corrected files are saved under derivatives/ folder (according to BIDS standard).', formatter_class=utils.SmartFormatter, prog=os.path.basename(__file__).strip('.py') ) @@ -40,38 +37,116 @@ def get_parser(): required=True, help= "R|Config yaml file listing images that require manual corrections for segmentation and vertebral " - "labeling. 'FILES_SEG' lists images associated with spinal cord segmentation " - ",'FILES_LABEL' lists images associated with vertebral labeling " - "and 'FILES_PMJ' lists images associated with pontomedullary junction labeling" + "labeling. " + "'FILES_SEG' lists images associated with spinal cord segmentation, " + "'FILES_GMSEG' lists images associated with gray matter segmentation, " + "'FILES_LESION' lists images associated with multiple sclerosis lesion segmentation, " + "'FILES_LABEL' lists images associated with vertebral labeling, " + "and 'FILES_PMJ' lists images associated with pontomedullary junction labeling. " "You can validate your .yml file at this website: http://www.yamllint.com/." - " If you want to correct segmentation only, ommit 'FILES_LABEL' in the list. Below is an example .yml file:\n" + "Below is an example .yml file:\n" + dedent( """ FILES_SEG: - - sub-1000032_T1w.nii.gz - - sub-1000083_T2w.nii.gz + - sub-001_T1w.nii.gz + - sub-002_T2w.nii.gz + FILES_GMSEG: + - sub-001_T1w.nii.gz + - sub-002_T2w.nii.gz + FILES_LESION: + - sub-001_T1w.nii.gz + - sub-002_T2w.nii.gz FILES_LABEL: - - sub-1000032_T1w.nii.gz - - sub-1000710_T1w.nii.gz + - sub-001_T1w.nii.gz + - sub-002_T1w.nii.gz FILES_PMJ: - - sub-1000032_T1w.nii.gz - - sub-1000710_T1w.nii.gz\n + - sub-001_T1w.nii.gz + - sub-002_T1w.nii.gz\n """) ) parser.add_argument( '-path-in', metavar="", - help='Path to the processed data. Example: ~/ukbiobank_results/data_processed', - default='./' + required=True, + help='Path to the processed data. Example: ~//data_processed', ) parser.add_argument( '-path-out', metavar="", - help="Path to the BIDS dataset where the corrected labels will be generated. Note: if the derivatives/ folder " - "does not already exist, it will be created." - "Example: ~/data-ukbiobank", + help= + "R|Path to the output folder where the corrected labels will be saved. Example: ~//" + "Note: The path provided within this flag will be combined with the path provided within the " + "'-path-derivatives' flag. ", default='./' ) + parser.add_argument( + '-path-derivatives', + metavar="", + help= + "R|Path to the 'derivatives' BIDS-complaint folder where the corrected labels will be saved. " + "Default: derivatives/labels" + "Note: if the provided folder (e.g., 'derivatives/labels') does not already exist, it will be created." + "Note: if segmentation or labels files already exist and you would like to correct them, provide path to them " + "within this flag.", + default=os.path.join('derivatives', 'labels') + ) + parser.add_argument( + '-suffix-files-in', + help= + "R|Suffix of the input files. For example: '_RPI_r'." + "Note: this flag is useful in cases when the input files have been processed and thus contains a specific " + "suffix.", + default='' + ) + parser.add_argument( + '-suffix-files-seg', + help="FILES-SEG suffix. Available options: '_seg' (default), '_label-SC_mask'.", + choices=['_seg', '_label-SC_mask'], + default='_seg' + ) + parser.add_argument( + '-suffix-files-gmseg', + help="FILES-GMSEG suffix. Available options: '_gmseg' (default), '_label-GM_mask'.", + choices=['_gmseg', '_label-GM_mask'], + default='_gmseg' + ) + parser.add_argument( + '-suffix-files-lesion', + help="FILES-LESION suffix. Available options: '_lesion' (default).", + choices=['_lesion'], + default='_lesion' + ) + parser.add_argument( + '-suffix-files-label', + help="FILES-LABEL suffix. Available options: '_labels' (default), '_labels-disc'.", + choices=['_labels', '_labels-disc'], + default='_labels' + ) + parser.add_argument( + '-suffix-files-pmj', + help="FILES-PMJ suffix. Available options: '_pmj' (default), '_label-pmj'.", + choices=['_pmj', '_label-pmj'], + default='_pmj' + ) + parser.add_argument( + '-label-disc-list', + help="Comma-separated list containing individual values and/or intervals for disc labeling. Example: '1:4,6,8' " + "or 1:20 (default)", + default='1:20' + ) + parser.add_argument( + '-viewer', + help="Viewer used for manual correction. Available options: 'itksnap' (default), 'fsleyes', 'slicer'.", + choices=['fsleyes', 'itksnap', 'slicer'], + default='itksnap' + ) + parser.add_argument( + '-fsl-color', + help="Color to be used for loading the label file on FSLeyes (default: red). `fsleyes -h` gives all the " + "available color options. If using a combination of colors, specify them with '-', e.g. 'red-yellow'.", + type=str, + default='red' + ) parser.add_argument( '-qc-only', help="Only output QC report based on the manually-corrected files already present in the derivatives folder. " @@ -93,9 +168,17 @@ def get_parser(): return parser -def get_function(task): +# TODO: add also sct_get_centerline +def get_function_for_qc(task): + """ + Get the function to use for QC based on the task. + :param task: + :return: + """ if task == 'FILES_SEG': return 'sct_deepseg_sc' + elif task == "FILES_GMSEG": + return "sct_deepseg_gm" elif task == 'FILES_LABEL': return 'sct_label_utils' elif task == 'FILES_PMJ': @@ -104,60 +187,84 @@ def get_function(task): raise ValueError("This task is not recognized: {}".format(task)) -def get_suffix(task, suffix=''): - if task == 'FILES_SEG': - return '_seg'+suffix - elif task == 'FILES_LABEL': - return '_labels'+suffix - elif task == 'FILES_PMJ': - return '_pmj'+suffix - - else: - raise ValueError("This task is not recognized: {}".format(task)) - - -def correct_segmentation(fname, fname_seg_out): +def correct_segmentation(fname, fname_seg_out, viewer, viewer_color): """ - Copy fname_seg in fname_seg_out, then open ITK-SNAP with fname and fname_seg_out. + Open viewer (ITK-SNAP, FSLeyes, or 3D Slicer) with fname and fname_seg_out. :param fname: - :param fname_seg: :param fname_seg_out: - :param name_rater: + :param viewer: + :param viewer_color: color to be used for the label. Only valid for on FSLeyes (default: red). :return: """ # launch ITK-SNAP - # Note: command line differs for macOs/Linux and Windows - print("In ITK-SNAP, correct the segmentation, then save it with the same name (overwrite).") - if shutil.which('itksnap') is not None: # Check if command 'itksnap' exists - os.system('itksnap -g ' + fname + ' -s ' + fname_seg_out) # for macOS and Linux - elif shutil.which('ITK-SNAP') is not None: # Check if command 'ITK-SNAP' exists - os.system('ITK-SNAP -g ' + fname + ' -s ' + fname_seg_out) # For windows - else: - sys.exit("ITK-SNAP not found. Please install it before using this program or check if it was added to PATH variable. Exit program.") + if viewer == 'itksnap': + print("In ITK-SNAP, correct the segmentation, then save it with the same name (overwrite).") + # Note: command line differs for macOs/Linux and Windows + if shutil.which('itksnap') is not None: # Check if command 'itksnap' exists + # macOS and Linux + os.system('itksnap -g {} -s {}'.format(fname, fname_seg_out)) + elif shutil.which('ITK-SNAP') is not None: # Check if command 'ITK-SNAP' exists + # Windows + os.system('ITK-SNAP -g {} -s {}'.format(fname, fname_seg_out)) + else: + viewer_not_found(viewer) + # launch FSLeyes + elif viewer == 'fsleyes': + if shutil.which('fsleyes') is not None: # Check if command 'fsleyes' exists + print("In FSLeyes, click on 'Edit mode', correct the segmentation, and then save it with the same name " + "(overwrite).") + os.system('fsleyes {} {} -cm {}'.format(fname, fname_seg_out, viewer_color)) + else: + viewer_not_found(viewer) + # launch 3D Slicer + elif viewer == 'slicer': + if shutil.which('slicer') is not None: + # TODO: Add instructions for 3D Slicer + pass + else: + viewer_not_found(viewer) + + +def viewer_not_found(viewer): + """ + Print that viewer is not installed and exit the program. + :param viewer: + :return: + """ + sys.exit("{} not found. Please install it before using this program or check if it was added to PATH variable. " + "You can also use another viewer by using the flag -viewer.".format(viewer)) -def correct_vertebral_labeling(fname, fname_label): +def correct_vertebral_labeling(fname, fname_label, label_list, viewer='sct_label_utils'): """ Open sct_label_utils to manually label vertebral levels. :param fname: :param fname_label: - :param name_rater: + :param label_list: Comma-separated list containing individual values and/or intervals. Example: '1:4,6,8' or 1:20 :return: """ - message = "Click at the posterior tip of the disc between C1-C2, C2-C3 and C3-C4 vertebral levels, then click 'Save and Quit'." - os.system('sct_label_utils -i {} -create-viewer 2,3,4 -o {} -msg "{}"'.format(fname, fname_label, message)) + if shutil.which(viewer) is not None: # Check if command 'sct_label_utils' exists + message = "Click at the posterior tip of the disc(s). Then click 'Save and Quit'." + if os.path.exists(fname_label): + os.system('sct_label_utils -i {} -create-viewer {} -o {} -ilabel {} -msg "{}"'.format(fname, label_list, fname_label, fname_label, message)) + else: + os.system('sct_label_utils -i {} -create-viewer {} -o {} -msg "{}"'.format(fname, label_list, fname_label, message)) + else: + viewer_not_found(viewer) -def correct_pmj_label(fname, fname_label): +def correct_pmj_label(fname, fname_label, viewer='sct_label_utils'): """ Open sct_label_utils to manually label PMJ. :param fname: :param fname_label: - :param name_rater: :return: """ - message = "Click at the posterior tip of the pontomedullary junction (PMJ) then click 'Save and Quit'." - os.system('sct_label_utils -i {} -create-viewer 50 -o {} -msg "{}"'.format(fname, fname_label, message)) + if shutil.which(viewer) is not None: # Check if command 'sct_label_utils' exists + message = "Click at the posterior tip of the pontomedullary junction (PMJ). Then click 'Save and Quit'." + os.system('sct_label_utils -i {} -create-viewer 50 -o {} -msg "{}"'.format(fname, fname_label, message)) + else: + viewer_not_found(viewer) def create_json(fname_nifti, name_rater): @@ -171,6 +278,54 @@ def create_json(fname_nifti, name_rater): fname_json = fname_nifti.rstrip('.nii').rstrip('.nii.gz') + '.json' with open(fname_json, 'w') as outfile: json.dump(metadata, outfile, indent=4) + # Add last newline + outfile.write("\n") + + +def ask_if_modify(fname_label): + """ + Check if file under derivatives already exists. If so, asks user if they want to modify it. + :param fname_label: file under derivatives + :return: + """ + # Check if file under derivatives already exists + if os.path.isfile(fname_label): + answer = None + while answer not in ("y", "n"): + answer = input("WARNING! The file {} already exists. " + "Would you like to modify it? [y/n] ".format(fname_label)) + if answer == "y": + do_labeling = True + elif answer == "n": + do_labeling = False + else: + print("Please answer with 'y' or 'n'") + # We don't want to copy because we want to modify the existing file + copy = False + # If the file under derivatives does not exist, copy it from processed data + else: + do_labeling = True + copy = True + + return do_labeling, copy + + +def generate_qc(fname, fname_label, task, fname_qc, subject, config_file): + """ + Generate QC report. + :param fname: + :param fname_seg: + :param fname_label: + :param fname_pmj: + :param qc_folder: + :return: + """ + os.system('sct_qc -i {} -s {} -p {} -qc {} -qc-subject {}'.format( + fname, fname_label, get_function_for_qc(task), fname_qc, subject)) + # Archive QC folder + shutil.copy(utils.get_full_path(config_file), fname_qc) + shutil.make_archive(fname_qc, 'zip', fname_qc) + print("Archive created:\n--> {}".format(fname_qc + '.zip')) def main(): @@ -180,32 +335,33 @@ def main(): args = parser.parse_args() # Logging level + # TODO: how is this actually used? if args.verbose: coloredlogs.install(fmt='%(message)s', level='DEBUG') else: coloredlogs.install(fmt='%(message)s', level='INFO') - # check if input yml file exists - if os.path.isfile(args.config): - fname_yml = args.config - else: - sys.exit("ERROR: Input yml file {} does not exist or path is wrong.".format(args.config)) - - # fetch input yml file as dict - with open(fname_yml, 'r') as stream: - try: - dict_yml = yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) + # Fetch configuration from YAML file + dict_yml = utils.fetch_yaml_config(args.config) # Curate dict_yml to only have filenames instead of absolute path dict_yml = utils.curate_dict_yml(dict_yml) - # check for missing files before starting the whole process - utils.check_files_exist(dict_yml, args.path_in) + # Check for missing files before starting the whole process + if not args.add_seg_only: + utils.check_files_exist(dict_yml, utils.get_full_path(args.path_in)) + suffix_dict = { + 'FILES_SEG': args.suffix_files_seg, # e.g., _seg or _label-SC_mask + 'FILES_GMSEG': args.suffix_files_gmseg, # e.g., _gmseg or _label-GM_mask + 'FILES_LESION': args.suffix_files_lesion, # e.g., _lesion + 'FILES_LABEL': args.suffix_files_label, # e.g., _labels or _labels-disc + 'FILES_PMJ': args.suffix_files_pmj # e.g., _pmj or _label-pmj + } + + path_out = utils.get_full_path(args.path_out) # check that output folder exists and has write permission - path_out_deriv = utils.check_output_folder(args.path_out, FOLDER_DERIVATIVES) + path_out_deriv = utils.check_output_folder(path_out, args.path_derivatives) # Get name of expert rater (skip if -qc-only is true) if not args.qc_only: @@ -213,79 +369,80 @@ def main(): "corrected file: ") # Build QC report folder name - fname_qc = 'qc_corr_' + time.strftime('%Y%m%d%H%M%S') + fname_qc = os.path.join(path_out, 'qc_corr_' + time.strftime('%Y%m%d%H%M%S')) # Get list of segmentations files for all subjects in -path-in (if -add-seg-only) if args.add_seg_only: - path_list = glob.glob(args.path_in + "/**/*_seg.nii.gz", recursive=True) # TODO: add other extension + path_list = glob.glob(args.path_in + "/**/*" + args.suffix_files_seg + ".nii.gz", recursive=True) # Get only filenames without suffix _seg to match files in -config .yml list - file_list = [utils.remove_suffix(os.path.split(path)[-1], '_seg') for path in path_list] + file_list = [utils.remove_suffix(os.path.split(path)[-1], args.suffix_files_seg) for path in path_list] # TODO: address "none" issue if no file present under a key # Perform manual corrections for task, files in dict_yml.items(): - # Get the list of segmentation files to add to derivatives, excluding the manually corrrected files in -config. + # Get the list of segmentation files to add to derivatives, excluding the manually corrected files in -config. + # TODO: probably extend also for other tasks (such as FILES_GMSEG) if args.add_seg_only and task == 'FILES_SEG': # Remove the files in the -config list for file in files: + # Remove the file suffix (e.g., '_RPI_r') to match the list of files in -path-in + file = utils.remove_suffix(file, args.suffix_files_in) if file in file_list: file_list.remove(file) files = file_list # Rename to use those files instead of the ones to exclude if files is not None: for file in files: # build file names - subject = file.split('_')[0] - contrast = utils.get_contrast(file) - fname = os.path.join(args.path_in, subject, contrast, file) - fname_label = os.path.join( - path_out_deriv, subject, contrast, utils.add_suffix(file, get_suffix(task, '-manual'))) - os.makedirs(os.path.join(path_out_deriv, subject, contrast), exist_ok=True) + subject, ses, filename, contrast = utils.fetch_subject_and_session(file) + # Construct absolute path to the input file + # For example: '/Users/user/dataset/data_processed/sub-001/anat/sub-001_T2w.nii.gz' + fname = os.path.join(utils.get_full_path(args.path_in), subject, ses, contrast, filename) + # Construct absolute path to the input label (segmentation, labeling etc.) file + # For example: '/Users/user/dataset/data_processed/sub-001/anat/sub-001_T2w_seg.nii.gz' + fname_seg = utils.add_suffix(fname, suffix_dict[task]) + # Construct absolute path to the derivative file (i.e., path where manually corrected file will be saved) + # For example: '/Users/user/dataset/derivatives/labels/sub-001/anat/sub-001_T2w_seg-manual.nii.gz' + fname_label = os.path.join(path_out_deriv, subject, ses, contrast, + utils.add_suffix(utils.remove_suffix(filename, args.suffix_files_in), + suffix_dict[task] + '-manual')) + # Create output folders under derivative if they do not exist + os.makedirs(os.path.join(path_out_deriv, subject, ses, contrast), exist_ok=True) if not args.qc_only: - if os.path.isfile(fname_label): - # if corrected file already exists, asks user if they want to overwrite it - answer = None - while answer not in ("y", "n"): - answer = input("WARNING! The file {} already exists. " - "Would you like to modify it? [y/n] ".format(fname_label)) - if answer == "y": - do_labeling = True - overwrite = False - elif answer == "n": - do_labeling = False - else: - print("Please answer with 'y' or 'n'") - else: - do_labeling = True - overwrite = True - # Perform labeling for the specific task + # Check if file under derivatives already exists. If so, asks user if they want to modify it. + do_labeling, copy = ask_if_modify(fname_label) + # Perform labeling (i.e., segmentation correction, labeling correction etc.) for the specific task if do_labeling: - if task in ['FILES_SEG']: - fname_seg = utils.add_suffix(fname, get_suffix(task)) - if overwrite: - shutil.copyfile(fname_seg, fname_label) + if copy: + # Copy file to derivatives folder + shutil.copyfile(fname_seg, fname_label) + print(f'Copying: {fname_seg} to {fname_label}') + if task in ['FILES_SEG', 'FILES_GMSEG']: if not args.add_seg_only: - correct_segmentation(fname, fname_label) + correct_segmentation(fname, fname_label, args.viewer, args.fsl_color) + elif task == 'FILES_LESION': + correct_segmentation(fname, fname_label, args.viewer, args.fsl_color) elif task == 'FILES_LABEL': - if not utils.check_software_installed(): - sys.exit("Some required software are not installed. Exit program.") - correct_vertebral_labeling(fname, fname_label) + correct_vertebral_labeling(fname, fname_label, args.label_disc_list) elif task == 'FILES_PMJ': - if not utils.check_software_installed(): - sys.exit("Some required software are not installed. Exit program.") correct_pmj_label(fname, fname_label) else: sys.exit('Task not recognized from yml file: {}'.format(task)) - # create json sidecar with the name of the expert rater - create_json(fname_label, name_rater) - - # generate QC report (only for vertebral labeling or for qc only) - if args.qc_only or task != 'FILES_SEG': - os.system('sct_qc -i {} -s {} -p {} -qc {} -qc-subject {}'.format( - fname, fname_label, get_function(task), fname_qc, subject)) - # Archive QC folder - shutil.copy(fname_yml, fname_qc) - shutil.make_archive(fname_qc, 'zip', fname_qc) - print("Archive created:\n--> {}".format(fname_qc+'.zip')) + + if task == 'FILES_LESION': + # create json sidecar with the name of the expert rater + create_json(fname_label, name_rater) + # NOTE: QC for lesion segmentation does not exist or not implemented yet + else: + # create json sidecar with the name of the expert rater + create_json(fname_label, name_rater) + # Generate QC report + generate_qc(fname, fname_label, task, fname_qc, subject, args.config) + + # Generate QC report only + if args.qc_only: + # Note: QC for lesion segmentation is not implemented yet + if task != "FILES_LESION": + generate_qc(fname, fname_label, task, fname_qc, subject, args.config) if __name__ == '__main__': diff --git a/package_for_correction.py b/package_for_correction.py index ed7b12d..ca8fc36 100644 --- a/package_for_correction.py +++ b/package_for_correction.py @@ -1,10 +1,11 @@ #!/usr/bin/env python # -# Script to package data for manual correction from SpineGeneric adapted for ukbiobank cord CSA project. +# Script to package data for manual correction. # # For usage, type: python package_for_correction.py -h # -# Author: Julien Cohen-Adad +# Authors: Jan Valosek, Sandrine Bédard, Julien Cohen-Adad +# import os @@ -13,10 +14,8 @@ import tempfile from textwrap import dedent import argparse -import yaml import coloredlogs - -import pipeline_ukbiobank.utils as utils +import utils def get_parser(): @@ -35,26 +34,34 @@ def get_parser(): metavar="", required=True, help= - "R|Config .yml file listing images that require manual corrections for segmentation and vertebral " - "labeling. 'FILES_SEG' lists images associated with spinal cord segmentation," - "and 'FILES_LABEL' lists images associated with vertebral labeling. " - "You can validate your .yml file at this website: http://www.yamllint.com/. Below is an example .yml file:\n" + "R|Config yaml file listing images that require manual corrections for segmentation and vertebral " + "labeling. 'FILES_SEG' lists images associated with spinal cord segmentation " + ",'FILES_GMSEG' lists images associated with gray matter segmentation " + ",'FILES_LABEL' lists images associated with vertebral labeling " + "and 'FILES_PMJ' lists images associated with pontomedullary junction labeling" + "You can validate your .yml file at this website: http://www.yamllint.com/." + "Below is an example .yml file:\n" + dedent( """ FILES_SEG: - - sub-1000032_T1w.nii.gz - - sub-1000083_T2w.nii.gz + - sub-001_T1w.nii.gz + - sub-002_T2w.nii.gz + FILES_GMSEG: + - sub-001_T1w.nii.gz + - sub-002_T2w.nii.gz FILES_LABEL: - - sub-1000032_T1w.nii.gz - - sub-1000710_T1w.nii.gz\n + - sub-001_T1w.nii.gz + - sub-002_T1w.nii.gz + FILES_PMJ: + - sub-001_T1w.nii.gz + - sub-002_T1w.nii.gz\n """) ) parser.add_argument( '-path-in', metavar="", required=True, - help='Path to the processed data. Example: ~/ukbiobank_results/data_processed', - default='./' + help='Path to the processed data. Example: ~//data_processed', ) parser.add_argument( '-o', @@ -62,6 +69,36 @@ def get_parser(): help="Zip file that contains the packaged data, without the extension. Default: data_to_correct", default='data_to_correct' ) + parser.add_argument( + '-suffix-files-seg', + help="FILES-SEG suffix. Available options: '_seg' (default), '_label-SC_mask'.", + choices=['_seg', '_label-SC_mask'], + default='_seg' + ) + parser.add_argument( + '-suffix-files-gmseg', + help="FILES-GMSEG suffix. Available options: '_gmseg' (default), '_label-GM_mask'.", + choices=['_gmseg', '_label-GM_mask'], + default='_gmseg' + ) + parser.add_argument( + '-suffix-files-lesion', + help="FILES-LESION suffix. Available options: '_lesion' (default).", + choices=['_lesion'], + default='_lesion' + ) + parser.add_argument( + '-suffix-files-label', + help="FILES-LABEL suffix. Available options: '_labels' (default), '_labels-disc'.", + choices=['_labels', '_labels-disc'], + default='_labels' + ) + parser.add_argument( + '-suffix-files-pmj', + help="FILES-PMJ suffix. Available options: '_pmj' (default), '_label-pmj'.", + choices=['_pmj', '_label-pmj'], + default='_pmj' + ) parser.add_argument( '-v', '--verbose', help="Full verbose (for debugging)", @@ -76,7 +113,7 @@ def copy_file(fname_in, path_out): os.makedirs(path_out, exist_ok=True) # copy file fname_out = shutil.copy(fname_in, path_out) - print("-> {}".format(fname_out)) + print(f'Copying: {fname_in} to {fname_out}') def main(): @@ -90,24 +127,22 @@ def main(): else: coloredlogs.install(fmt='%(message)s', level='INFO') - # Check if input yml file exists - if os.path.isfile(args.config): - fname_yml = args.config - else: - sys.exit("ERROR: Input yml file {} does not exist or path is wrong.".format(args.config)) - - # Fetch input yml file as dict - with open(fname_yml, 'r') as stream: - try: - dict_yml = yaml.safe_load(stream) - except yaml.YAMLError as exc: - print(exc) - + # Fetch configuration from YAML file + dict_yml = utils.fetch_yaml_config(args.config) + # Curate dict_yml to only have filenames instead of absolute path dict_yml = utils.curate_dict_yml(dict_yml) # Check for missing files before starting the whole process - utils.check_files_exist(dict_yml, args.path_in) + utils.check_files_exist(dict_yml, utils.get_full_path(args.path_in)) + + suffix_dict = { + 'FILES_SEG': args.suffix_files_seg, # e.g., _seg or _label-SC_mask + 'FILES_GMSEG': args.suffix_files_gmseg, # e.g., _gmseg or _label-GM_mask + 'FILES_LESION': args.suffix_files_lesion, # e.g., _lesion + 'FILES_LABEL': args.suffix_files_label, # e.g., _labels or _labels-disc + 'FILES_PMJ': args.suffix_files_pmj # e.g., _pmj or _label-pmj + } # Create temp folder path_tmp = tempfile.mkdtemp() @@ -116,22 +151,24 @@ def main(): # Note: in case the file is listed twice, we just overwrite it in the destination dir. for task, files in dict_yml.items(): for file in files: - if task == 'FILES_SEG': - suffix_label = '_seg' - elif task == 'FILES_LABEL': - suffix_label = None - elif task == 'FILES_PMJ': - suffix_label = None + if task in suffix_dict.keys(): + suffix_label = suffix_dict[task] else: sys.exit('Task not recognized from yml file: {}'.format(task)) + subject, ses, filename, contrast = utils.fetch_subject_and_session(file) + # Construct absolute path to the input file + # For example: '/Users/user/dataset/data_processed/sub-001/anat/sub-001_T2w.nii.gz' + fname = os.path.join(utils.get_full_path(args.path_in), subject, ses, contrast, filename) + # Construct absolute path to the temp folder + path_out = os.path.join(path_tmp, subject, ses, contrast) # Copy image - copy_file(os.path.join(args.path_in, utils.get_subject(file), utils.get_contrast(file), file), - os.path.join(path_tmp, utils.get_subject(file), utils.get_contrast(file))) + copy_file(fname, path_out) # Copy label if exists if suffix_label is not None: - copy_file(os.path.join(args.path_in, utils.get_subject(file), utils.get_contrast(file), - utils.add_suffix(file, suffix_label)), - os.path.join(path_tmp, utils.get_subject(file), utils.get_contrast(file))) + # Construct absolute path to the input label (segmentation, labeling etc.) file + # For example: '/Users/user/dataset/data_processed/sub-001/anat/sub-001_T2w_seg.nii.gz' + fname_seg = utils.add_suffix(fname, suffix_dict[task]) + copy_file(fname_seg, path_out) # Package to zip file print("Creating archive...") @@ -141,7 +178,7 @@ def main(): if os.path.isdir(new_path_tmp): shutil.rmtree(new_path_tmp) shutil.move(path_tmp, new_path_tmp) - fname_archive = shutil.make_archive(args.o, 'zip', root_dir_tmp, base_dir_name) + fname_archive = shutil.make_archive(utils.get_full_path(args.o), 'zip', root_dir_tmp, base_dir_name) print("-> {}".format(fname_archive)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..61ac723 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +coloredlogs==15.0.1 +humanfriendly==10.0 +PyYAML==6.0 diff --git a/utils.py b/utils.py index 7a669c0..823150a 100644 --- a/utils.py +++ b/utils.py @@ -1,34 +1,46 @@ #!/usr/bin/env python # -*- coding: utf-8 -# Collection of useful functions from spine_generic, used for manual segmentation +# +# Collection of useful functions used by other scripts +# +# Authors: Jan Valosek, Sandrine Bédard, Julien Cohen-Adad +# import os import re import logging +import sys import textwrap import argparse import subprocess import shutil -from pathlib import Path -from enum import Enum +import yaml + # BIDS utility tool -def get_subject(file): +def fetch_subject_and_session(filename_path): """ - Get subject from BIDS file name - :param file: - :return: subject + Get subject ID, session ID and filename from the input BIDS-compatible filename or file path + The function works both on absolute file path as well as filename + :param filename_path: input nifti filename (e.g., sub-001_ses-01_T1w.nii.gz) or file path + (e.g., /home/user/MRI/bids/derivatives/labels/sub-001/ses-01/anat/sub-001_ses-01_T1w.nii.gz + :return: subjectID: subject ID (e.g., sub-001) + :return: sessionID: session ID (e.g., ses-01) + :return: filename: nii filename (e.g., sub-001_ses-01_T1w.nii.gz) """ - return file.split('_')[0] + _, filename = os.path.split(filename_path) # Get just the filename (i.e., remove the path) + subject = re.search('sub-(.*?)[_/]', filename_path) + subjectID = subject.group(0)[:-1] if subject else "" # [:-1] removes the last underscore or slash + session = re.findall(r'ses-..', filename_path) + sessionID = session[0] if session else "" # Return None if there is no session + contrast = 'dwi' if 'dwi' in filename_path else 'anat' # Return contrast (dwi or anat) + # REGEX explanation + # \d - digit + # \d? - no or one occurrence of digit + # *? - match the previous element as few times as possible (zero or more times) -def get_contrast(file): - """ - Get contrast from BIDS file name - :param file: - :return: - """ - return 'dwi' if (file.split('_')[-1]).split('.')[0] == 'dwi' else 'anat' + return subjectID, sessionID, filename, contrast class SmartFormatter(argparse.HelpFormatter): @@ -136,6 +148,29 @@ def remove_suffix(fname, suffix): return os.path.join(stem.replace(suffix, '') + ext) +def fetch_yaml_config(config_file): + """ + Fetch configuration from YAML file + :param config_file: YAML file + :return: dictionary with configuration + """ + config_file = get_full_path(config_file) + # Check if input yml file exists + if os.path.isfile(config_file): + fname_yml = config_file + else: + sys.exit("ERROR: Input yml file {} does not exist or path is wrong.".format(config_file)) + + # Fetch input yml file as dict + with open(fname_yml, 'r') as stream: + try: + dict_yml = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + return dict_yml + + def curate_dict_yml(dict_yml): """ Curate dict_yml to only have filenames instead of absolute path @@ -148,6 +183,15 @@ def curate_dict_yml(dict_yml): return dict_yml_curate +def get_full_path(path): + """ + Return full path. If ~ is passed, expand it to home directory. + :param path: str: Input path + :return: str: Full path + """ + return os.path.abspath(os.path.expanduser(path)) + + def check_files_exist(dict_files, path_data): """ Check if all files listed in the input dictionary exist @@ -159,7 +203,8 @@ def check_files_exist(dict_files, path_data): for task, files in dict_files.items(): if files is not None: for file in files: - fname = os.path.join(path_data, get_subject(file), get_contrast(file), file) + subject, ses, filename, contrast = fetch_subject_and_session(file) + fname = os.path.join(path_data, subject, ses, contrast, filename) if not os.path.exists(fname): missing_files.append(fname) if missing_files: @@ -171,7 +216,7 @@ def check_output_folder(path_bids, folder_derivatives): """ Make sure path exists, has writing permissions, and create derivatives folder if it does not exist. :param path_bids: - :return: path_bids_derivatives + :return: folder_derivatives: """ if path_bids is None: logging.error("-path-out should be provided.")