From cb54cc58662e08e2b070f89ba8df785a5ca18cb5 Mon Sep 17 00:00:00 2001 From: Sydney Covitz Date: Thu, 4 May 2023 07:50:32 -0400 Subject: [PATCH 1/8] printing erroneous jsons and only rounding float parameters --- cubids/cubids.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/cubids/cubids.py b/cubids/cubids.py index e9b24d70..a7e8bd40 100644 --- a/cubids/cubids.py +++ b/cubids/cubids.py @@ -1464,8 +1464,9 @@ def round_params(param_group_df, config, modality): if column_name not in param_group_df: continue if 'precision' in column_fmt: - param_group_df[column_name] = \ - param_group_df[column_name].round(column_fmt['precision']) + if isinstance(param_group_df[column_name], float): + param_group_df[column_name] = \ + param_group_df[column_name].round(column_fmt['precision']) return param_group_df @@ -1473,9 +1474,13 @@ def round_params(param_group_df, config, modality): def get_sidecar_metadata(json_file): # get all metadata values in a file's sidecar # transform json dictionary to python dictionary - with open(json_file) as json_file: - data = json.load(json_file) - return data + try: + with open(json_file) as json_file: + data = json.load(json_file) + return data + except Exception: + print("Error loading sidecar: ", json_file) + return None def format_params(param_group_df, config, modality): From 57e7535cba493bca761528ad7385ec4da60c560e Mon Sep 17 00:00:00 2001 From: Sydney Covitz Date: Thu, 4 May 2023 08:27:28 -0400 Subject: [PATCH 2/8] tried changing conda version in circle config --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a6b26aaf..837f69e1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,7 @@ jobs: cd /tmp export MINICONDA=/tmp/miniconda export PATH="$MINICONDA/bin:$PATH" - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-s390x.sh -O miniconda.sh bash miniconda.sh -b -f -p $MINICONDA conda config --set always_yes yes conda update conda From 2eebe13a6abddf02d02c4bc93161f3e630cc710b Mon Sep 17 00:00:00 2001 From: Sydney Covitz Date: Thu, 4 May 2023 08:29:23 -0400 Subject: [PATCH 3/8] tried changing conda version in circle config --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 837f69e1..ef2757ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,7 @@ jobs: cd /tmp export MINICONDA=/tmp/miniconda export PATH="$MINICONDA/bin:$PATH" - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-s390x.sh -O miniconda.sh + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh -O miniconda.sh bash miniconda.sh -b -f -p $MINICONDA conda config --set always_yes yes conda update conda From 4585c6afd36c60b1e5fe7c9ac837e4f860789d2e Mon Sep 17 00:00:00 2001 From: Sydney Covitz Date: Thu, 4 May 2023 08:34:37 -0400 Subject: [PATCH 4/8] tried changing conda version in circle config --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ef2757ff..5753e9fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,12 +70,12 @@ jobs: cd /tmp export MINICONDA=/tmp/miniconda export PATH="$MINICONDA/bin:$PATH" - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh -O miniconda.sh + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh bash miniconda.sh -b -f -p $MINICONDA conda config --set always_yes yes conda update conda conda info -a - conda create -n cubids python=3.7 + conda create -n cubids python=3.8 source activate cubids conda install -c conda-forge -y datalad conda install pip From 5dfb1898df8bff721635d2faf3685601259e7b9f Mon Sep 17 00:00:00 2001 From: Sydney Covitz Date: Thu, 4 May 2023 11:08:57 -0400 Subject: [PATCH 5/8] moved return outside in get sidecar metadata --- cubids/cubids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cubids/cubids.py b/cubids/cubids.py index a7e8bd40..9b7b51c5 100644 --- a/cubids/cubids.py +++ b/cubids/cubids.py @@ -1477,10 +1477,10 @@ def get_sidecar_metadata(json_file): try: with open(json_file) as json_file: data = json.load(json_file) - return data except Exception: print("Error loading sidecar: ", json_file) return None + return data def format_params(param_group_df, config, modality): From 1714074ba10d135eebc500029e591f4a6886071e Mon Sep 17 00:00:00 2001 From: Sydney Covitz Date: Thu, 4 May 2023 17:27:00 -0400 Subject: [PATCH 6/8] now only printing sidecars that are erroneous --- cubids/cubids.py | 126 ++++++++++++++++++++++++++++------------------- 1 file changed, 74 insertions(+), 52 deletions(-) diff --git a/cubids/cubids.py b/cubids/cubids.py index 9b7b51c5..e6dec26d 100644 --- a/cubids/cubids.py +++ b/cubids/cubids.py @@ -499,6 +499,9 @@ def change_filename(self, filepath, entities): # json_file = self.layout.get_file(str(path)) # data = json_file.get_dict() data = get_sidecar_metadata(str(path)) + if data == "Erroneous sidecar": + print('Error parsing sidecar: ', str(path)) + continue if 'IntendedFor' in data.keys(): # check if IntendedFor field is a str or list @@ -630,9 +633,11 @@ def _purge_associations(self, scans): # json_file = self.layout.get_file(str(path)) # data = json_file.get_dict() data = get_sidecar_metadata(str(path)) + if data == "Erroneous sidecar": + print('Error parsing sidecar: ', str(path)) + continue # remove scan references in the IntendedFor - if 'IntendedFor' in data.keys(): # check if IntendedFor field value is a list or a string if isinstance(data['IntendedFor'], str): @@ -757,7 +762,11 @@ def _cache_fieldmaps(self): for fmap_file in tqdm(fmap_files): # intentions = listify(fmap_file.get_metadata().get("IntendedFor")) fmap_json = img_to_new_ext(fmap_file.path, '.json') - if_list = get_sidecar_metadata(fmap_json).get("IntendedFor") + metadata = get_sidecar_metadata(fmap_json) + if metadata == "Erroneous sidecar": + print('Error parsing sidecar: ', str(fmap_json)) + continue + if_list = metadata.get("IntendedFor") intentions = listify(if_list) subject_prefix = "sub-%s" % fmap_file.entities['subject'] @@ -824,6 +833,9 @@ def get_param_groups_from_key_group(self, key_group): to_include, self.layout, self.fieldmap_lookup, key_group, self.grouping_config, modality, self.keys_files) + if ret == "erroneous sidecar found": + return "erroneous sidecar found" + # add modality to the retun tuple l_ret = list(ret) l_ret.append(modality) @@ -946,8 +958,11 @@ def get_param_groups_dataframes(self): labeled_files = [] param_group_summaries = [] for key_group in key_groups: - labeled_file_params, param_summary, modality = \ - self.get_param_groups_from_key_group(key_group) + try: + labeled_file_params, param_summary, modality = \ + self.get_param_groups_from_key_group(key_group) + except Exception: + continue if labeled_file_params is None: continue param_group_summaries.append(param_summary) @@ -1338,58 +1353,62 @@ def _get_param_groups(files, layout, fieldmap_lookup, key_group_name, dfs = [] # path needs to be relative to the root with no leading prefix + for path in files: # metadata = layout.get_metadata(path) metadata = get_sidecar_metadata(img_to_new_ext(path, '.json')) - intentions = metadata.get("IntendedFor", []) - slice_times = metadata.get("SliceTiming", []) - - wanted_keys = metadata.keys() & imaging_params - example_data = {key: metadata[key] for key in wanted_keys} - example_data["KeyGroup"] = key_group_name - - # Get the fieldmaps out and add their types - if 'FieldmapKey' in relational_params: - fieldmap_types = sorted([_file_to_key_group(fmap.path) for - fmap in fieldmap_lookup[path]]) - - # check if config says columns or bool - if relational_params['FieldmapKey']['display_mode'] == \ - 'bool': - if len(fieldmap_types) > 0: - example_data['HasFieldmap'] = True + if metadata == "Erroneous sidecar": + print('Error parsing sidecar: ', img_to_new_ext(path, '.json')) + else: + intentions = metadata.get("IntendedFor", []) + slice_times = metadata.get("SliceTiming", []) + + wanted_keys = metadata.keys() & imaging_params + example_data = {key: metadata[key] for key in wanted_keys} + example_data["KeyGroup"] = key_group_name + + # Get the fieldmaps out and add their types + if 'FieldmapKey' in relational_params: + fieldmap_types = sorted([_file_to_key_group(fmap.path) for + fmap in fieldmap_lookup[path]]) + + # check if config says columns or bool + if relational_params['FieldmapKey']['display_mode'] == \ + 'bool': + if len(fieldmap_types) > 0: + example_data['HasFieldmap'] = True + else: + example_data['HasFieldmap'] = False else: - example_data['HasFieldmap'] = False - else: - for fmap_num, fmap_type in enumerate(fieldmap_types): - example_data['FieldmapKey%02d' % fmap_num] = fmap_type + for fmap_num, fmap_type in enumerate(fieldmap_types): + example_data['FieldmapKey%02d' % fmap_num] = fmap_type - # Add the number of slice times specified - if "NSliceTimes" in derived_params: - example_data["NSliceTimes"] = len(slice_times) + # Add the number of slice times specified + if "NSliceTimes" in derived_params: + example_data["NSliceTimes"] = len(slice_times) - example_data["FilePath"] = path + example_data["FilePath"] = path - # If it's a fieldmap, see what key group it's intended to correct - if "IntendedForKey" in relational_params: - intended_key_groups = sorted([_file_to_key_group(intention) for - intention in intentions]) + # If it's a fieldmap, see what key group it's intended to correct + if "IntendedForKey" in relational_params: + intended_key_groups = sorted([_file_to_key_group(intention) for + intention in intentions]) - # check if config says columns or bool - if relational_params['IntendedForKey']['display_mode'] == \ - 'bool': - if len(intended_key_groups) > 0: - example_data["UsedAsFieldmap"] = True + # check if config says columns or bool + if relational_params['IntendedForKey']['display_mode'] == \ + 'bool': + if len(intended_key_groups) > 0: + example_data["UsedAsFieldmap"] = True + else: + example_data["UsedAsFieldmap"] = False else: - example_data["UsedAsFieldmap"] = False - else: - for intention_num, intention_key_group in \ - enumerate(intended_key_groups): - example_data[ - "IntendedForKey%02d" % intention_num] = \ - intention_key_group + for intention_num, intention_key_group in \ + enumerate(intended_key_groups): + example_data[ + "IntendedForKey%02d" % intention_num] = \ + intention_key_group - dfs.append(example_data) + dfs.append(example_data) # Assign each file to a ParamGroup @@ -1406,10 +1425,13 @@ def _get_param_groups(files, layout, fieldmap_lookup, key_group_name, if "Cluster_" + col not in list(df.columns) and col != 'FilePath': check_cols.append(col) - # Find the unique ParamGroups and assign ID numbers in "ParamGroup" - deduped = df.drop('FilePath', axis=1).drop_duplicates(subset=check_cols, - ignore_index=True) + # Find the unique ParamGroups and assign ID numbers in "ParamGroup"\ + try: + deduped = df.drop('FilePath', axis=1) + except Exception: + return "erroneous sidecar found" + deduped = deduped.drop_duplicates(subset=check_cols, ignore_index=True) deduped["ParamGroup"] = np.arange(deduped.shape[0]) + 1 # add the modality as a column @@ -1477,10 +1499,10 @@ def get_sidecar_metadata(json_file): try: with open(json_file) as json_file: data = json.load(json_file) + return data except Exception: - print("Error loading sidecar: ", json_file) - return None - return data + # print("Error loading sidecar: ", json_filename) + return "Erroneous sidecar" def format_params(param_group_df, config, modality): From 87975b5a5e636096521d7b1716bc59eec3e01815 Mon Sep 17 00:00:00 2001 From: Sydney Covitz Date: Thu, 4 May 2023 17:28:13 -0400 Subject: [PATCH 7/8] fixed group print statements --- cubids/cubids.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cubids/cubids.py b/cubids/cubids.py index e6dec26d..82389019 100644 --- a/cubids/cubids.py +++ b/cubids/cubids.py @@ -56,9 +56,9 @@ def __init__(self, data_root, use_datalad=False, acq_group_level='subject', @property def layout(self): if self._layout is None: - print("SETTING LAYOUT OBJECT") + # print("SETTING LAYOUT OBJECT") self.reset_bids_layout() - print("LAYOUT OBJECT SET") + # print("LAYOUT OBJECT SET") return self._layout def reset_bids_layout(self, validate=False): @@ -1163,7 +1163,7 @@ def get_TSVs(self, path_prefix): group_by_acquisition_sets(path_prefix + "_files.tsv", path_prefix, self.acq_group_level) - print("Detected " + str(len(summary)) + " Parameter Groups.") + print("CuBIDS Detected " + str(len(summary)) + " Parameter Groups.") def get_key_groups(self): '''Identifies the key groups for the bids dataset''' From 8efa298a2ecc3a2e76b13e739f0bfdf21864c990 Mon Sep 17 00:00:00 2001 From: Sydney Covitz Date: Thu, 4 May 2023 17:28:36 -0400 Subject: [PATCH 8/8] fixed typo --- cubids/cubids.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cubids/cubids.py b/cubids/cubids.py index 82389019..bae9236b 100644 --- a/cubids/cubids.py +++ b/cubids/cubids.py @@ -1163,7 +1163,7 @@ def get_TSVs(self, path_prefix): group_by_acquisition_sets(path_prefix + "_files.tsv", path_prefix, self.acq_group_level) - print("CuBIDS Detected " + str(len(summary)) + " Parameter Groups.") + print("CuBIDS detected " + str(len(summary)) + " Parameter Groups.") def get_key_groups(self): '''Identifies the key groups for the bids dataset'''