From cf2a8fb83c69798225bf5e29fdca075812fb9f9a Mon Sep 17 00:00:00 2001 From: abisi Date: Fri, 21 Jun 2024 17:10:04 +0200 Subject: [PATCH 01/22] script to generate allen atlas with barrel annotations + pyproject.toml change --- .../allen_mouse_bluebrain_barrels.py | 224 ++++++++++++++++++ pyproject.toml | 4 + 2 files changed, 228 insertions(+) create mode 100644 brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py new file mode 100644 index 00000000..87b1248c --- /dev/null +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -0,0 +1,224 @@ +__version__ = "2" + +import sys +import json +import nrrd +from pathlib import Path + +from allensdk.api.queries.ontologies_api import OntologiesApi +from allensdk.api.queries.reference_space_api import ReferenceSpaceApi +from allensdk.core.reference_space_cache import ReferenceSpaceCache +from requests import exceptions +from tqdm import tqdm + + + +from brainglobe_atlasapi import descriptors +from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data + +def create_atlas(working_dir, resolution): + # Specify information about the atlas: + ATLAS_NAME = "allen_mouse_bluebrain_barrels" + SPECIES = "Mus musculus" + ATLAS_LINK = "http://www.brain-map.org" + CITATION = "Bolaños-Puchet S., Teska A., et al. (2024). https://doi.org/10.1101/2023.08.24.554204" + ATLAS_PACKAGER = "Axel Bisi" + ORIENTATION = "asr" + + # Temporary folder for nrrd files download: + download_dir_path = working_dir / "downloading_path" + download_dir_path.mkdir(exist_ok=True) + + # Download template volume: + ######################################### + spacecache = ReferenceSpaceCache( + manifest=download_dir_path / "manifest.json", + # downloaded files are stored relative to here + resolution=resolution, + reference_space_key="annotation/ccf_2017", + # use the latest version of the CCF + ) + + # Download + template_volume, _ = spacecache.get_template_volume() + print("Download completed...") + + ## TODO: import file + #sys.run("python transplant_barrels_nrrd.py --annotation_barrels.nrrd --annotation_10.nrrd --hierarchy.json") + #if resolution == 10: + # annotation_file = 'annotation_barrels_10.nrrd' + #elif resolution == 25: + # annotation_file = 'annotation_barrels_25.nrrd' + #else: + # raise ValueError("Resolution not supported.") + + # Load annotated volume: + annotation_dir_path = Path(r'C:\Users\bisi\Github\atlas-enhancement\barrel-annotations\data\atlas') + if resolution == 10: + annotation_dir_path = annotation_dir_path / 'atlas_10um' + annotation_file = 'annotation_barrels_10.nrrd' + elif resolution == 25: + annotation_dir_path = annotation_dir_path / 'atlas_25um' + annotation_file = 'annotation_barrels_25.nrrd' + annotated_volume = nrrd.read(annotation_dir_path / annotation_file)[0] + print("Annotation volume loaded...") + + # Download structures tree and meshes: + ###################################### + oapi = OntologiesApi() # ontologies + struct_tree = spacecache.get_structure_tree() # structures tree + + # Find id of set of regions with mesh: + select_set = ( + "Structures whose surfaces are represented by a precomputed mesh" + ) + + mesh_set_ids = [ + s["id"] + for s in oapi.get_structure_sets() + if s["description"] == select_set + ] + + # Get structures with mesh for both versions + structs_with_mesh = struct_tree.get_structures_by_set_id(mesh_set_ids) + structs_with_barrels = json.load(open(annotation_dir_path / 'hierarchy.json')) + + # Add barrels structures to Allen structures + def find_dicts_with_key_containing_substring(d, key, substring): + """ + Recursively find all dictionaries within a nested dictionary that contain a specific substring + in the value associated with a given key. + + Args: + d (dict): The input dictionary. + key (str): The key to search for. + substring (str): The substring to search for in the value associated with the key. + + Returns: + list: A list of dictionaries that contain the key with a value containing the substring. + """ + if not isinstance(d, dict): + raise ValueError("Input should be a dictionary") + + matching_dicts = [] + + def recurse(sub_d): + contains_substring = False + + for k, v in sub_d.items(): + if isinstance(v, dict): + recurse(v) + elif isinstance(v, list): + for item in v: + if isinstance(item, dict): + recurse(item) + if k == key and substring in str(v): + contains_substring = True + + if contains_substring: + matching_dicts.append(sub_d) + + recurse(d) + + return matching_dicts + + matching_dicts = find_dicts_with_key_containing_substring(structs_with_barrels, key="name", substring="barrel") + matching_dicts = [d for d in matching_dicts if d["graph_order"] in [52,53,54,55,56,57]] + main_barrel_parent_struct = [s for s in structs_with_mesh if s['acronym'] == 'SSp-bfd'][0] + structures_present = ['SSp-bfd1', 'SSp-bfd2/3', 'SSp-bfd4', 'SSp-bfd5', 'SSp-bfd6a', 'SSp-bfd6b'] # keep laminar structures + keys_to_keep = ['acronym', 'graph_id','graph_order', 'id', 'name', 'rgb_triplet', 'structure_set_ids', 'structure_id_path'] + dict_to_add = [] + for d in matching_dicts: + # Ignore parent-level SSp-bfd layers + if d['acronym'] in structures_present: + print('Skipping', d, 'already present.') + continue + # Ignore sub-structures layer 2 and 3 to keep layer 2/3 structure + if d['graph_order']==53 and d['acronym'] in ['SSp-bfd2', 'SSp-bfd3']: + print('Excluding', d, 'to keep layer 2/3 structure only.') + continue + elif d['graph_order']==54 and ('layer 2' in d['name'] or 'layer 3' in d['name']): + print('Excluding', d, 'to keep layer 2/3 structure only.') + continue + else: + current_id = d['id'] + # Find corresponding parent structure + if d['graph_order'] == 52: # barrel-level -> find SSp-bfd + # Create new structure_id_path for barrel structure + d['structure_id_path'] = main_barrel_parent_struct['structure_id_path'] + [current_id] + elif d['graph_order'] == 53: # barrel layer-level -> find SSp-bfd-barrel also + parent_struct_id = d['parent_structure_id'] + parent_struct = [s for s in matching_dicts if s['id'] == parent_struct_id][0] + parent_struct['structure_id_path'] = main_barrel_parent_struct['structure_id_path'] + [parent_struct_id] + # Create new structure_id_path for barrel-layer structure + d['structure_id_path'] = main_barrel_parent_struct['structure_id_path'] + [d['parent_structure_id']] + [current_id] + + # Complete with other keys + d['rgb_triplet'] = main_barrel_parent_struct['rgb_triplet'] + d['graph_id'] = 1 + d['structure_set_ids'] = None + dict_to_add.append({k: d[k] for k in keys_to_keep}) + + # Add list of dicts to structs_with_mesh + structs_with_mesh = structs_with_mesh + dict_to_add + print('Added the following structures to the atlas:') + for d in dict_to_add: + print(d['name']) + + # Directory for mesh saving: + meshes_dir = working_dir / descriptors.MESHES_DIRNAME + + space = ReferenceSpaceApi() + meshes_dict = dict() + for s in tqdm(structs_with_mesh): + name = s["id"] + filename = meshes_dir / f"{name}.obj" + try: + space.download_structure_mesh( + structure_id=s["id"], + ccf_version="annotation/ccf_2017", + file_name=filename, + ) + meshes_dict[name] = filename + except (exceptions.HTTPError, ConnectionError): + print(s) + + # Loop over structures, remove entries not used: + for struct in structs_with_mesh: + [ + struct.pop(k) + for k in ["graph_id", "structure_set_ids", "graph_order"] + ] + + # Wrap up, compress, and remove file:0 + print("Finalising atlas") + output_filename = wrapup_atlas_from_data( + atlas_name=ATLAS_NAME, + atlas_minor_version=__version__, + citation=CITATION, + atlas_link=ATLAS_LINK, + species=SPECIES, + resolution=(resolution,) * 3, + orientation=ORIENTATION, + root_id=997, + reference_stack=template_volume, + annotation_stack=annotated_volume, + structures_list=structs_with_mesh, + meshes_dict=meshes_dict, + working_dir=working_dir, + hemispheres_stack=None, + cleanup_files=False, + compress=True, + additional_metadata={"atlas_packager": ATLAS_PACKAGER}, + ) + + return output_filename + + +if __name__ == "__main__": + RES_UM = 25 + # Generated atlas path: + bg_root_dir = Path.home() / "Desktop" / "brainglobe_workingdir" / "allen_mouse_bluebrain_barrels" + bg_root_dir.mkdir(exist_ok=True) + + create_atlas(bg_root_dir, RES_UM) diff --git a/pyproject.toml b/pyproject.toml index 4bb956ed..ea763752 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,10 @@ dev = [ "tox", ] allenmouse = ["allensdk"] +allenmouse_barrels = [ + "allensdk", + "atlas-enhancement"] + atlasgen = [ "loguru", "PyMCubes", From e0cc4a01c37e893fa9a1f51c3a9dadaefc3833b9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:26:56 +0000 Subject: [PATCH 02/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../allen_mouse_bluebrain_barrels.py | 123 ++++++++++++------ 1 file changed, 84 insertions(+), 39 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 87b1248c..d069ea54 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -1,21 +1,19 @@ __version__ = "2" -import sys import json -import nrrd from pathlib import Path +import nrrd from allensdk.api.queries.ontologies_api import OntologiesApi from allensdk.api.queries.reference_space_api import ReferenceSpaceApi from allensdk.core.reference_space_cache import ReferenceSpaceCache from requests import exceptions from tqdm import tqdm - - from brainglobe_atlasapi import descriptors from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data + def create_atlas(working_dir, resolution): # Specify information about the atlas: ATLAS_NAME = "allen_mouse_bluebrain_barrels" @@ -44,22 +42,24 @@ def create_atlas(working_dir, resolution): print("Download completed...") ## TODO: import file - #sys.run("python transplant_barrels_nrrd.py --annotation_barrels.nrrd --annotation_10.nrrd --hierarchy.json") - #if resolution == 10: + # sys.run("python transplant_barrels_nrrd.py --annotation_barrels.nrrd --annotation_10.nrrd --hierarchy.json") + # if resolution == 10: # annotation_file = 'annotation_barrels_10.nrrd' - #elif resolution == 25: + # elif resolution == 25: # annotation_file = 'annotation_barrels_25.nrrd' - #else: + # else: # raise ValueError("Resolution not supported.") # Load annotated volume: - annotation_dir_path = Path(r'C:\Users\bisi\Github\atlas-enhancement\barrel-annotations\data\atlas') + annotation_dir_path = Path( + r"C:\Users\bisi\Github\atlas-enhancement\barrel-annotations\data\atlas" + ) if resolution == 10: - annotation_dir_path = annotation_dir_path / 'atlas_10um' - annotation_file = 'annotation_barrels_10.nrrd' + annotation_dir_path = annotation_dir_path / "atlas_10um" + annotation_file = "annotation_barrels_10.nrrd" elif resolution == 25: - annotation_dir_path = annotation_dir_path / 'atlas_25um' - annotation_file = 'annotation_barrels_25.nrrd' + annotation_dir_path = annotation_dir_path / "atlas_25um" + annotation_file = "annotation_barrels_25.nrrd" annotated_volume = nrrd.read(annotation_dir_path / annotation_file)[0] print("Annotation volume loaded...") @@ -81,7 +81,9 @@ def create_atlas(working_dir, resolution): # Get structures with mesh for both versions structs_with_mesh = struct_tree.get_structures_by_set_id(mesh_set_ids) - structs_with_barrels = json.load(open(annotation_dir_path / 'hierarchy.json')) + structs_with_barrels = json.load( + open(annotation_dir_path / "hierarchy.json") + ) # Add barrels structures to Allen structures def find_dicts_with_key_containing_substring(d, key, substring): @@ -122,48 +124,86 @@ def recurse(sub_d): return matching_dicts - matching_dicts = find_dicts_with_key_containing_substring(structs_with_barrels, key="name", substring="barrel") - matching_dicts = [d for d in matching_dicts if d["graph_order"] in [52,53,54,55,56,57]] - main_barrel_parent_struct = [s for s in structs_with_mesh if s['acronym'] == 'SSp-bfd'][0] - structures_present = ['SSp-bfd1', 'SSp-bfd2/3', 'SSp-bfd4', 'SSp-bfd5', 'SSp-bfd6a', 'SSp-bfd6b'] # keep laminar structures - keys_to_keep = ['acronym', 'graph_id','graph_order', 'id', 'name', 'rgb_triplet', 'structure_set_ids', 'structure_id_path'] + matching_dicts = find_dicts_with_key_containing_substring( + structs_with_barrels, key="name", substring="barrel" + ) + matching_dicts = [ + d + for d in matching_dicts + if d["graph_order"] in [52, 53, 54, 55, 56, 57] + ] + main_barrel_parent_struct = [ + s for s in structs_with_mesh if s["acronym"] == "SSp-bfd" + ][0] + structures_present = [ + "SSp-bfd1", + "SSp-bfd2/3", + "SSp-bfd4", + "SSp-bfd5", + "SSp-bfd6a", + "SSp-bfd6b", + ] # keep laminar structures + keys_to_keep = [ + "acronym", + "graph_id", + "graph_order", + "id", + "name", + "rgb_triplet", + "structure_set_ids", + "structure_id_path", + ] dict_to_add = [] for d in matching_dicts: # Ignore parent-level SSp-bfd layers - if d['acronym'] in structures_present: - print('Skipping', d, 'already present.') + if d["acronym"] in structures_present: + print("Skipping", d, "already present.") continue # Ignore sub-structures layer 2 and 3 to keep layer 2/3 structure - if d['graph_order']==53 and d['acronym'] in ['SSp-bfd2', 'SSp-bfd3']: - print('Excluding', d, 'to keep layer 2/3 structure only.') + if d["graph_order"] == 53 and d["acronym"] in ["SSp-bfd2", "SSp-bfd3"]: + print("Excluding", d, "to keep layer 2/3 structure only.") continue - elif d['graph_order']==54 and ('layer 2' in d['name'] or 'layer 3' in d['name']): - print('Excluding', d, 'to keep layer 2/3 structure only.') + elif d["graph_order"] == 54 and ( + "layer 2" in d["name"] or "layer 3" in d["name"] + ): + print("Excluding", d, "to keep layer 2/3 structure only.") continue else: - current_id = d['id'] + current_id = d["id"] # Find corresponding parent structure - if d['graph_order'] == 52: # barrel-level -> find SSp-bfd + if d["graph_order"] == 52: # barrel-level -> find SSp-bfd # Create new structure_id_path for barrel structure - d['structure_id_path'] = main_barrel_parent_struct['structure_id_path'] + [current_id] - elif d['graph_order'] == 53: # barrel layer-level -> find SSp-bfd-barrel also - parent_struct_id = d['parent_structure_id'] - parent_struct = [s for s in matching_dicts if s['id'] == parent_struct_id][0] - parent_struct['structure_id_path'] = main_barrel_parent_struct['structure_id_path'] + [parent_struct_id] + d["structure_id_path"] = main_barrel_parent_struct[ + "structure_id_path" + ] + [current_id] + elif ( + d["graph_order"] == 53 + ): # barrel layer-level -> find SSp-bfd-barrel also + parent_struct_id = d["parent_structure_id"] + parent_struct = [ + s for s in matching_dicts if s["id"] == parent_struct_id + ][0] + parent_struct["structure_id_path"] = main_barrel_parent_struct[ + "structure_id_path" + ] + [parent_struct_id] # Create new structure_id_path for barrel-layer structure - d['structure_id_path'] = main_barrel_parent_struct['structure_id_path'] + [d['parent_structure_id']] + [current_id] + d["structure_id_path"] = ( + main_barrel_parent_struct["structure_id_path"] + + [d["parent_structure_id"]] + + [current_id] + ) # Complete with other keys - d['rgb_triplet'] = main_barrel_parent_struct['rgb_triplet'] - d['graph_id'] = 1 - d['structure_set_ids'] = None + d["rgb_triplet"] = main_barrel_parent_struct["rgb_triplet"] + d["graph_id"] = 1 + d["structure_set_ids"] = None dict_to_add.append({k: d[k] for k in keys_to_keep}) # Add list of dicts to structs_with_mesh structs_with_mesh = structs_with_mesh + dict_to_add - print('Added the following structures to the atlas:') + print("Added the following structures to the atlas:") for d in dict_to_add: - print(d['name']) + print(d["name"]) # Directory for mesh saving: meshes_dir = working_dir / descriptors.MESHES_DIRNAME @@ -218,7 +258,12 @@ def recurse(sub_d): if __name__ == "__main__": RES_UM = 25 # Generated atlas path: - bg_root_dir = Path.home() / "Desktop" / "brainglobe_workingdir" / "allen_mouse_bluebrain_barrels" + bg_root_dir = ( + Path.home() + / "Desktop" + / "brainglobe_workingdir" + / "allen_mouse_bluebrain_barrels" + ) bg_root_dir.mkdir(exist_ok=True) create_atlas(bg_root_dir, RES_UM) From b617d3b71adedb6033c3012a17d67d2e5f1c13c6 Mon Sep 17 00:00:00 2001 From: abisi Date: Fri, 5 Jul 2024 11:09:16 +0200 Subject: [PATCH 03/22] voxcell library + update atlas script with 10um res only, meshes, creation from bluebrain code --- .../allen_mouse_bluebrain_barrels.py | 88 +++++++++++++------ pyproject.toml | 2 +- 2 files changed, 64 insertions(+), 26 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 87b1248c..1d76eee7 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -1,28 +1,34 @@ __version__ = "2" import sys +import os +import numpy as np import json import nrrd from pathlib import Path +import subprocess from allensdk.api.queries.ontologies_api import OntologiesApi from allensdk.api.queries.reference_space_api import ReferenceSpaceApi from allensdk.core.reference_space_cache import ReferenceSpaceCache from requests import exceptions from tqdm import tqdm - - +from brainglobe_atlasapi.atlas_generation.mesh_utils import ( + Region, + create_region_mesh, +) from brainglobe_atlasapi import descriptors from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data +from brainglobe_atlasapi.structure_tree_util import get_structures_tree def create_atlas(working_dir, resolution): # Specify information about the atlas: ATLAS_NAME = "allen_mouse_bluebrain_barrels" SPECIES = "Mus musculus" ATLAS_LINK = "http://www.brain-map.org" - CITATION = "Bolaños-Puchet S., Teska A., et al. (2024). https://doi.org/10.1101/2023.08.24.554204" - ATLAS_PACKAGER = "Axel Bisi" + CITATION = "Bolaños-Puchet S., Teska A., et al. (2024). https://doi.org/10.1162/imag_a_00209" + ATLAS_PACKAGER = "Axel Bisi, axel.bisi@gmail.com" ORIENTATION = "asr" # Temporary folder for nrrd files download: @@ -39,28 +45,25 @@ def create_atlas(working_dir, resolution): # use the latest version of the CCF ) - # Download + # Download original Allen atlas files template_volume, _ = spacecache.get_template_volume() - print("Download completed...") + annotated_volume_allen , _ = spacecache.get_annotation_volume() + + # Paths for atlas enhancement + sys.path.append(working_dir / "atlas-enhancement/barrel-annotations") + data_path = working_dir / "atlas-enhancement/barrel-annotations/data" + annotation_path = working_dir / "downloading_path/annotation/ccf_2017/" + os.chdir(working_dir/ "atlas-enhancement/barrel-annotations") - ## TODO: import file - #sys.run("python transplant_barrels_nrrd.py --annotation_barrels.nrrd --annotation_10.nrrd --hierarchy.json") - #if resolution == 10: - # annotation_file = 'annotation_barrels_10.nrrd' - #elif resolution == 25: - # annotation_file = 'annotation_barrels_25.nrrd' - #else: - # raise ValueError("Resolution not supported.") + # Transplant barrels into Allen annotation + subprocess.call(["python", "transplant_barrels_nrrd_script.py", data_path, annotation_path, str(resolution)]) # Load annotated volume: - annotation_dir_path = Path(r'C:\Users\bisi\Github\atlas-enhancement\barrel-annotations\data\atlas') - if resolution == 10: - annotation_dir_path = annotation_dir_path / 'atlas_10um' - annotation_file = 'annotation_barrels_10.nrrd' - elif resolution == 25: - annotation_dir_path = annotation_dir_path / 'atlas_25um' - annotation_file = 'annotation_barrels_25.nrrd' - annotated_volume = nrrd.read(annotation_dir_path / annotation_file)[0] + if resolution != 10: + annotation_file = "annotation_barrels_{}.nrrd".format(resolution) + else: + raise ValueError("Resolution not supported.") + annotated_volume = nrrd.read(data_path / annotation_file)[0] print("Annotation volume loaded...") # Download structures tree and meshes: @@ -81,7 +84,7 @@ def create_atlas(working_dir, resolution): # Get structures with mesh for both versions structs_with_mesh = struct_tree.get_structures_by_set_id(mesh_set_ids) - structs_with_barrels = json.load(open(annotation_dir_path / 'hierarchy.json')) + structs_with_barrels = json.load(open(data_path / 'hierarchy.json')) # Add barrels structures to Allen structures def find_dicts_with_key_containing_substring(d, key, substring): @@ -168,6 +171,22 @@ def recurse(sub_d): # Directory for mesh saving: meshes_dir = working_dir / descriptors.MESHES_DIRNAME + tree = get_structures_tree(structs_with_mesh) + print( + f"Number of brain regions: {tree.size()}, " + f"max tree depth: {tree.depth()}" + ) + + # generate binary mask for mesh creation + labels = np.unique(annotated_volume).astype(np.int_) + for key, node in tree.nodes.items(): + if key in labels: + is_label = True + else: + is_label = False + + node.data = Region(is_label) + space = ReferenceSpaceApi() meshes_dict = dict() for s in tqdm(structs_with_mesh): @@ -181,7 +200,26 @@ def recurse(sub_d): ) meshes_dict[name] = filename except (exceptions.HTTPError, ConnectionError): - print(s) + print(f"Failed to download mesh for {s['name']} ({s['id']})") + print('Creating mesh for', s['name']) + # Create mesh + root_id = 997 + closing_n_iters = 2 + decimate_fraction = 0.3 + smooth = True + create_region_mesh( + ( + meshes_dir, + node, + tree, + labels, + annotated_volume, + root_id, + closing_n_iters, + decimate_fraction, + smooth, + ) + ) # Loop over structures, remove entries not used: for struct in structs_with_mesh: @@ -216,7 +254,7 @@ def recurse(sub_d): if __name__ == "__main__": - RES_UM = 25 + RES_UM = 10 # Generated atlas path: bg_root_dir = Path.home() / "Desktop" / "brainglobe_workingdir" / "allen_mouse_bluebrain_barrels" bg_root_dir.mkdir(exist_ok=True) diff --git a/pyproject.toml b/pyproject.toml index ea763752..ec0e5083 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ dev = [ allenmouse = ["allensdk"] allenmouse_barrels = [ "allensdk", - "atlas-enhancement"] + "voxcell"] atlasgen = [ "loguru", From a20fb46d6858a81e45bd667ee2fe7ca266a7f503 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 5 Jul 2024 09:13:18 +0000 Subject: [PATCH 04/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../allen_mouse_bluebrain_barrels.py | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 5355db67..14f3609e 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -1,24 +1,24 @@ __version__ = "2" -import sys -import os -import numpy as np import json -from pathlib import Path +import os import subprocess +import sys +from pathlib import Path import nrrd +import numpy as np from allensdk.api.queries.ontologies_api import OntologiesApi from allensdk.api.queries.reference_space_api import ReferenceSpaceApi from allensdk.core.reference_space_cache import ReferenceSpaceCache from requests import exceptions from tqdm import tqdm + +from brainglobe_atlasapi import descriptors from brainglobe_atlasapi.atlas_generation.mesh_utils import ( Region, create_region_mesh, ) - -from brainglobe_atlasapi import descriptors from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data from brainglobe_atlasapi.structure_tree_util import get_structures_tree @@ -48,16 +48,24 @@ def create_atlas(working_dir, resolution): # Download original Allen atlas files template_volume, _ = spacecache.get_template_volume() - annotated_volume_allen , _ = spacecache.get_annotation_volume() + annotated_volume_allen, _ = spacecache.get_annotation_volume() # Paths for atlas enhancement sys.path.append(working_dir / "atlas-enhancement/barrel-annotations") data_path = working_dir / "atlas-enhancement/barrel-annotations/data" annotation_path = working_dir / "downloading_path/annotation/ccf_2017/" - os.chdir(working_dir/ "atlas-enhancement/barrel-annotations") + os.chdir(working_dir / "atlas-enhancement/barrel-annotations") # Transplant barrels into Allen annotation - subprocess.call(["python", "transplant_barrels_nrrd_script.py", data_path, annotation_path, str(resolution)]) + subprocess.call( + [ + "python", + "transplant_barrels_nrrd_script.py", + data_path, + annotation_path, + str(resolution), + ] + ) # Load annotated volume: if resolution != 10: @@ -85,8 +93,7 @@ def create_atlas(working_dir, resolution): # Get structures with mesh for both versions structs_with_mesh = struct_tree.get_structures_by_set_id(mesh_set_ids) - structs_with_barrels = json.load(open(data_path / 'hierarchy.json')) - + structs_with_barrels = json.load(open(data_path / "hierarchy.json")) # Add barrels structures to Allen structures def find_dicts_with_key_containing_substring(d, key, substring): @@ -241,7 +248,7 @@ def recurse(sub_d): meshes_dict[name] = filename except (exceptions.HTTPError, ConnectionError): print(f"Failed to download mesh for {s['name']} ({s['id']})") - print('Creating mesh for', s['name']) + print("Creating mesh for", s["name"]) # Create mesh root_id = 997 closing_n_iters = 2 @@ -297,9 +304,7 @@ def recurse(sub_d): RES_UM = 10 # Generated atlas path: bg_root_dir = ( - Path.home() - / "brainglobe_workingdir" - / "allen_mouse_bluebrain_barrels" + Path.home() / "brainglobe_workingdir" / "allen_mouse_bluebrain_barrels" ) bg_root_dir.mkdir(exist_ok=True) From 831e9e2594aa27ae5712bdd68404c6a0a2ec8438 Mon Sep 17 00:00:00 2001 From: abisi Date: Sun, 14 Jul 2024 13:25:13 +0200 Subject: [PATCH 05/22] using gin-hosted pregenerated files --- .../allen_mouse_bluebrain_barrels.py | 84 +++++++++++-------- 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 5355db67..213d0b3e 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -1,11 +1,9 @@ -__version__ = "2" +__version__ = "0" -import sys -import os import numpy as np import json from pathlib import Path -import subprocess +import pooch import nrrd from allensdk.api.queries.ontologies_api import OntologiesApi @@ -18,6 +16,7 @@ create_region_mesh, ) +from brainglobe_atlasapi import utils from brainglobe_atlasapi import descriptors from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data from brainglobe_atlasapi.structure_tree_util import get_structures_tree @@ -29,7 +28,7 @@ def create_atlas(working_dir, resolution): SPECIES = "Mus musculus" ATLAS_LINK = "http://www.brain-map.org" CITATION = "Bolaños-Puchet S., Teska A., et al. (2024). https://doi.org/10.1162/imag_a_00209" - ATLAS_PACKAGER = "Axel Bisi, axel.bisi@gmail.com" + ATLAS_PACKAGER = "Axel Bisi" ORIENTATION = "asr" # Temporary folder for nrrd files download: @@ -48,24 +47,30 @@ def create_atlas(working_dir, resolution): # Download original Allen atlas files template_volume, _ = spacecache.get_template_volume() - annotated_volume_allen , _ = spacecache.get_annotation_volume() - # Paths for atlas enhancement - sys.path.append(working_dir / "atlas-enhancement/barrel-annotations") - data_path = working_dir / "atlas-enhancement/barrel-annotations/data" - annotation_path = working_dir / "downloading_path/annotation/ccf_2017/" - os.chdir(working_dir/ "atlas-enhancement/barrel-annotations") - - # Transplant barrels into Allen annotation - subprocess.call(["python", "transplant_barrels_nrrd_script.py", data_path, annotation_path, str(resolution)]) + # Download enhanced barrel-containing Allen annotation files by BlueBrain, and hierarchy: + ######################################### + annotation_dir_path = working_dir / "downloading_path/annotation_enhanced" + annotation_dir_path.mkdir(exist_ok=True) - # Load annotated volume: - if resolution != 10: - annotation_file = "annotation_barrels_{}.nrrd".format(resolution) + if resolution == 10: + gin_url = "https://gin.g-node.org/BrainGlobe/bluebrain_barrel_materials/raw/master/annotation_barrels_10.nrrd" + elif resolution == 25: + gin_url = "https://gin.g-node.org/BrainGlobe/bluebrain_barrel_materials/raw/master/annotation_barrels_25.nrrd" else: - raise ValueError("Resolution not supported.") - annotated_volume = nrrd.read(data_path / annotation_file)[0] - print("Annotation volume loaded...") + raise ValueError("Resolution {}um not supported.".format(resolution)) + + utils.check_internet_connection() + annotation_file_path = pooch.retrieve( + gin_url, + known_hash=None, + path=annotation_dir_path, + progressbar=True, + ) + + # Load annotation volume: + annotated_volume = nrrd.read(annotation_file_path)[0] + # Download structures tree and meshes: ###################################### @@ -85,7 +90,18 @@ def create_atlas(working_dir, resolution): # Get structures with mesh for both versions structs_with_mesh = struct_tree.get_structures_by_set_id(mesh_set_ids) - structs_with_barrels = json.load(open(data_path / 'hierarchy.json')) + + # Download hierarchy: + #gin_url = "https://gin.g-node.org/BrainGlobe/bluebrain_barrel_materials/raw/master/hierarchy.json" + #hierarchy_path = pooch.retrieve( + # gin_url, + # known_hash=None, + # path=annotation_dir_path, + # fname="hierarchy.json", + # progressbar=True + #) + hierarchy_path = r'C:\Users\bisi\Desktop\atlas\atlas_10um' + "\hierarchy.json" + structs_with_barrels = json.load(open(hierarchy_path)) # Add barrels structures to Allen structures @@ -247,19 +263,19 @@ def recurse(sub_d): closing_n_iters = 2 decimate_fraction = 0.3 smooth = True - create_region_mesh( - ( - meshes_dir, - node, - tree, - labels, - annotated_volume, - root_id, - closing_n_iters, - decimate_fraction, - smooth, - ) - ) + #create_region_mesh( + # ( + # meshes_dir, + # node, + # tree, + # labels, + # annotated_volume, + # root_id, + # closing_n_iters, + # decimate_fraction, + # smooth, + # ) + #) # Loop over structures, remove entries not used: for struct in structs_with_mesh: From 83fac2088b649072351a799328103aa917de8591 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 11:36:59 +0000 Subject: [PATCH 06/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../atlas_scripts/allen_mouse_bluebrain_barrels.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 2e9ec0a2..2aa5ab15 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -1,26 +1,21 @@ __version__ = "0" -import numpy as np import json from pathlib import Path -import pooch - import nrrd import numpy as np +import pooch from allensdk.api.queries.ontologies_api import OntologiesApi from allensdk.api.queries.reference_space_api import ReferenceSpaceApi from allensdk.core.reference_space_cache import ReferenceSpaceCache from requests import exceptions from tqdm import tqdm -from brainglobe_atlasapi import descriptors +from brainglobe_atlasapi import descriptors, utils from brainglobe_atlasapi.atlas_generation.mesh_utils import ( Region, create_region_mesh, ) - -from brainglobe_atlasapi import utils -from brainglobe_atlasapi import descriptors from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data from brainglobe_atlasapi.structure_tree_util import get_structures_tree @@ -74,7 +69,6 @@ def create_atlas(working_dir, resolution): # Load annotation volume: annotated_volume = nrrd.read(annotation_file_path)[0] - # Download structures tree and meshes: ###################################### oapi = OntologiesApi() # ontologies @@ -101,7 +95,7 @@ def create_atlas(working_dir, resolution): known_hash=None, path=annotation_dir_path, fname="hierarchy.json", - progressbar=True + progressbar=True, ) structs_with_barrels = json.load(open(hierarchy_path)) From c8e1feb08fc5a9a639fcffdb33d09e9f59e688ff Mon Sep 17 00:00:00 2001 From: abisi Date: Mon, 15 Jul 2024 10:46:50 +0200 Subject: [PATCH 07/22] added comments --- .../atlas_scripts/allen_mouse_bluebrain_barrels.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 2e9ec0a2..e9337f13 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -1,5 +1,4 @@ __version__ = "0" -import numpy as np import json from pathlib import Path import pooch @@ -38,7 +37,7 @@ def create_atlas(working_dir, resolution): download_dir_path = working_dir / "downloading_path" download_dir_path.mkdir(exist_ok=True) - # Download template volume: + # Download original Allen template volume: ######################################### spacecache = ReferenceSpaceCache( manifest=download_dir_path / "manifest.json", @@ -48,7 +47,6 @@ def create_atlas(working_dir, resolution): # use the latest version of the CCF ) - # Download original Allen atlas files template_volume, _ = spacecache.get_template_volume() # Download enhanced barrel-containing Allen annotation files by BlueBrain, and hierarchy: @@ -71,10 +69,9 @@ def create_atlas(working_dir, resolution): progressbar=True, ) - # Load annotation volume: + # Load enhanced annotation volume: annotated_volume = nrrd.read(annotation_file_path)[0] - # Download structures tree and meshes: ###################################### oapi = OntologiesApi() # ontologies @@ -175,6 +172,7 @@ def recurse(sub_d): ] dict_to_add = [] for d in matching_dicts: + # Ignore parent-level SSp-bfd layers if d["acronym"] in structures_present: print("Skipping", d, "already present.") @@ -188,6 +186,8 @@ def recurse(sub_d): ): print("Excluding", d, "to keep layer 2/3 structure only.") continue + + # Add desired barrel-related structures, with corresponding fields else: current_id = d["id"] # Find corresponding parent structure From 1cbee61356c9b28d2d94ebf232b96b01afa942b7 Mon Sep 17 00:00:00 2001 From: abisi Date: Mon, 15 Jul 2024 15:35:38 +0200 Subject: [PATCH 08/22] mesh creation update and cleanup --- .../allen_mouse_bluebrain_barrels.py | 93 ++++++++++++------- 1 file changed, 60 insertions(+), 33 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index e9337f13..3db6b803 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -6,11 +6,13 @@ import nrrd import numpy as np +import time from allensdk.api.queries.ontologies_api import OntologiesApi from allensdk.api.queries.reference_space_api import ReferenceSpaceApi from allensdk.core.reference_space_cache import ReferenceSpaceCache from requests import exceptions from tqdm import tqdm +from rich.progress import track from brainglobe_atlasapi import descriptors from brainglobe_atlasapi.atlas_generation.mesh_utils import ( @@ -219,6 +221,7 @@ def recurse(sub_d): d["structure_set_ids"] = None dict_to_add.append({k: d[k] for k in keys_to_keep}) + # Add list of dicts to structs_with_mesh structs_with_mesh = structs_with_mesh + dict_to_add print("Added the following structures to the atlas:") @@ -228,6 +231,23 @@ def recurse(sub_d): # Directory for mesh saving: meshes_dir = working_dir / descriptors.MESHES_DIRNAME + # Download existing Allen meshes: + space = ReferenceSpaceApi() + meshes_dict = dict() + for s in tqdm(structs_with_mesh): + name = s["id"] + filename = meshes_dir / f"{name}.obj" + try: + #space.download_structure_mesh( + # structure_id=s["id"], + # ccf_version="annotation/ccf_2017", + # file_name=filename, + #) + meshes_dict[name] = filename + except (exceptions.HTTPError, ConnectionError): + print(f"Failed to download mesh for {s['name']} ({s['id']})") + + # Create missing meshes tree = get_structures_tree(structs_with_mesh) print( f"Number of brain regions: {tree.size()}, " @@ -243,39 +263,46 @@ def recurse(sub_d): is_label = False node.data = Region(is_label) - space = ReferenceSpaceApi() - meshes_dict = dict() - for s in tqdm(structs_with_mesh): - name = s["id"] - filename = meshes_dir / f"{name}.obj" - try: - space.download_structure_mesh( - structure_id=s["id"], - ccf_version="annotation/ccf_2017", - file_name=filename, - ) - meshes_dict[name] = filename - except (exceptions.HTTPError, ConnectionError): - print(f"Failed to download mesh for {s['name']} ({s['id']})") - print("Creating mesh for", s["name"]) - # Create mesh - root_id = 997 - closing_n_iters = 2 - decimate_fraction = 0.3 - smooth = True - create_region_mesh( - ( - meshes_dir, - node, - tree, - labels, - annotated_volume, - root_id, - closing_n_iters, - decimate_fraction, - smooth, - ) - ) + + start = time.time() + + root_id = 997 + closing_n_iters = 2 + decimate_fraction = 0.3 + smooth = True + + for node in track( + tree.nodes.values(), + total=tree.size(), + description="Creating meshes", + ): + # Check if mesh already exists + filename = meshes_dir / f"{node.identifier}.obj" + if filename.exists(): + continue + + #create_region_mesh( + # ( + # meshes_dir, + # node, + # tree, + # labels, + # annotated_volume, + # root_id, + # closing_n_iters, + # decimate_fraction, + # smooth, + # ) + #) + + meshes_dict[node.identifier] = filename + + print( + "Finished mesh extraction in : ", + round((time.time() - start) / 60, 2), + " minutes", + ) + # Loop over structures, remove entries not used: for struct in structs_with_mesh: From 8afd634f509d35762bc7132d2c22ab9e9ebc0478 Mon Sep 17 00:00:00 2001 From: abisi Date: Mon, 15 Jul 2024 15:36:37 +0200 Subject: [PATCH 09/22] removed comments --- .../allen_mouse_bluebrain_barrels.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 3db6b803..0f770d80 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -238,11 +238,11 @@ def recurse(sub_d): name = s["id"] filename = meshes_dir / f"{name}.obj" try: - #space.download_structure_mesh( - # structure_id=s["id"], - # ccf_version="annotation/ccf_2017", - # file_name=filename, - #) + space.download_structure_mesh( + structure_id=s["id"], + ccf_version="annotation/ccf_2017", + file_name=filename, + ) meshes_dict[name] = filename except (exceptions.HTTPError, ConnectionError): print(f"Failed to download mesh for {s['name']} ({s['id']})") @@ -281,19 +281,19 @@ def recurse(sub_d): if filename.exists(): continue - #create_region_mesh( - # ( - # meshes_dir, - # node, - # tree, - # labels, - # annotated_volume, - # root_id, - # closing_n_iters, - # decimate_fraction, - # smooth, - # ) - #) + create_region_mesh( + ( + meshes_dir, + node, + tree, + labels, + annotated_volume, + root_id, + closing_n_iters, + decimate_fraction, + smooth, + ) + ) meshes_dict[node.identifier] = filename From 308a975612f8f769f5319ded080bdcfad84275b8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 13:37:47 +0000 Subject: [PATCH 10/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../atlas_scripts/allen_mouse_bluebrain_barrels.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index c126ae58..3da9408b 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -1,17 +1,17 @@ __version__ = "0" import json +import time from pathlib import Path import nrrd import numpy as np -import time import pooch from allensdk.api.queries.ontologies_api import OntologiesApi from allensdk.api.queries.reference_space_api import ReferenceSpaceApi from allensdk.core.reference_space_cache import ReferenceSpaceCache from requests import exceptions -from tqdm import tqdm from rich.progress import track +from tqdm import tqdm from brainglobe_atlasapi import descriptors, utils from brainglobe_atlasapi.atlas_generation.mesh_utils import ( @@ -217,7 +217,6 @@ def recurse(sub_d): d["structure_set_ids"] = None dict_to_add.append({k: d[k] for k in keys_to_keep}) - # Add list of dicts to structs_with_mesh structs_with_mesh = structs_with_mesh + dict_to_add print("Added the following structures to the atlas:") @@ -268,9 +267,9 @@ def recurse(sub_d): smooth = True for node in track( - tree.nodes.values(), - total=tree.size(), - description="Creating meshes", + tree.nodes.values(), + total=tree.size(), + description="Creating meshes", ): # Check if mesh already exists filename = meshes_dir / f"{node.identifier}.obj" @@ -299,7 +298,6 @@ def recurse(sub_d): " minutes", ) - # Loop over structures, remove entries not used: for struct in structs_with_mesh: [ From 44aecac2348c393ea5ec89d29aa2eca044afb4d1 Mon Sep 17 00:00:00 2001 From: Adam Tyson Date: Tue, 16 Jul 2024 09:27:28 +0100 Subject: [PATCH 11/22] Make pre-commit happy --- .../atlas_scripts/allen_mouse_bluebrain_barrels.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 3da9408b..c0007885 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -47,7 +47,8 @@ def create_atlas(working_dir, resolution): template_volume, _ = spacecache.get_template_volume() - # Download enhanced barrel-containing Allen annotation files by BlueBrain, and hierarchy: + # Download enhanced barrel-containing Allen annotation files by BlueBrain, + # and hierarchy: ######################################### annotation_dir_path = working_dir / "downloading_path/annotation_enhanced" annotation_dir_path.mkdir(exist_ok=True) @@ -103,16 +104,18 @@ def create_atlas(working_dir, resolution): # Add barrels structures to Allen structures def find_dicts_with_key_containing_substring(d, key, substring): """ - Recursively find all dictionaries within a nested dictionary that contain a specific substring - in the value associated with a given key. + Recursively find all dictionaries within a nested dictionary that + contain a specific substring in the value associated with a given key. Args: d (dict): The input dictionary. key (str): The key to search for. - substring (str): The substring to search for in the value associated with the key. + substring (str): The substring to search for in the value associated + with the key. Returns: - list: A list of dictionaries that contain the key with a value containing the substring. + list: A list of dictionaries that contain the key with a value + containing the substring. """ if not isinstance(d, dict): raise ValueError("Input should be a dictionary") From 100f05ab0302f9102a35879464a129dd7c17588a Mon Sep 17 00:00:00 2001 From: Axel Date: Tue, 16 Jul 2024 13:36:34 +0200 Subject: [PATCH 12/22] Update brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py Co-authored-by: Adam Tyson --- .../atlas_scripts/allen_mouse_bluebrain_barrels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index c0007885..f1b64092 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -32,7 +32,7 @@ def create_atlas(working_dir, resolution): ORIENTATION = "asr" # Temporary folder for nrrd files download: - download_dir_path = working_dir / "downloading_path" + download_dir_path = working_dir / "downloads" download_dir_path.mkdir(exist_ok=True) # Download original Allen template volume: From bc3e472318f17d86b6abb012df1665e8497e9fec Mon Sep 17 00:00:00 2001 From: abisi Date: Tue, 16 Jul 2024 13:38:30 +0200 Subject: [PATCH 13/22] mesh params at file top --- .../allen_mouse_bluebrain_barrels.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index f1b64092..4a19e75f 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -31,6 +31,12 @@ def create_atlas(working_dir, resolution): ATLAS_PACKAGER = "Axel Bisi" ORIENTATION = "asr" + # Parameters for mesh creation: + ROOT_ID = 997 + CLOSING_N_ITERS = 2 + DECIMATE_FRACTION = 0.3 + SMOOTH = True + # Temporary folder for nrrd files download: download_dir_path = working_dir / "downloads" download_dir_path.mkdir(exist_ok=True) @@ -264,11 +270,6 @@ def recurse(sub_d): start = time.time() - root_id = 997 - closing_n_iters = 2 - decimate_fraction = 0.3 - smooth = True - for node in track( tree.nodes.values(), total=tree.size(), @@ -286,10 +287,10 @@ def recurse(sub_d): tree, labels, annotated_volume, - root_id, - closing_n_iters, - decimate_fraction, - smooth, + ROOT_ID, + CLOSING_N_ITERS, + DECIMATE_FRACTION, + SMOOTH, ) ) From 533fd8f29a3d93881c945b40b1ac5f60c35c3cb5 Mon Sep 17 00:00:00 2001 From: abisi Date: Thu, 18 Jul 2024 16:49:00 +0200 Subject: [PATCH 14/22] post-creation rescaling of new meshes --- .../allen_mouse_bluebrain_barrels.py | 63 +++++++++++++------ 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 4a19e75f..cb7806e3 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -1,4 +1,4 @@ -__version__ = "0" +__version__ = "0.1" import json import time from pathlib import Path @@ -6,6 +6,8 @@ import nrrd import numpy as np import pooch +import meshio as mio + from allensdk.api.queries.ontologies_api import OntologiesApi from allensdk.api.queries.reference_space_api import ReferenceSpaceApi from allensdk.core.reference_space_cache import ReferenceSpaceCache @@ -56,7 +58,7 @@ def create_atlas(working_dir, resolution): # Download enhanced barrel-containing Allen annotation files by BlueBrain, # and hierarchy: ######################################### - annotation_dir_path = working_dir / "downloading_path/annotation_enhanced" + annotation_dir_path = working_dir / "downloads/annotation_enhanced" annotation_dir_path.mkdir(exist_ok=True) if resolution == 10: @@ -233,7 +235,10 @@ def recurse(sub_d): print(d["name"]) # Directory for mesh saving: - meshes_dir = working_dir / descriptors.MESHES_DIRNAME + meshes_dir = working_dir / descriptors.MESHES_DIRNAME / 'meshes_{}'.format(resolution) + # If directory exists, then skip + if not meshes_dir.exists(): + meshes_dir.mkdir(exist_ok=False) # Download existing Allen meshes: space = ReferenceSpaceApi() @@ -241,11 +246,16 @@ def recurse(sub_d): for s in tqdm(structs_with_mesh): name = s["id"] filename = meshes_dir / f"{name}.obj" + + if filename.exists(): + meshes_dict[name] = filename + continue + try: space.download_structure_mesh( structure_id=s["id"], ccf_version="annotation/ccf_2017", - file_name=filename, + file_name=filename, ) meshes_dict[name] = filename except (exceptions.HTTPError, ConnectionError): @@ -275,26 +285,26 @@ def recurse(sub_d): total=tree.size(), description="Creating meshes", ): + # Check if mesh already exists - filename = meshes_dir / f"{node.identifier}.obj" - if filename.exists(): + file_name = meshes_dir / f"{node.identifier}.obj" + if file_name.exists(): continue - create_region_mesh( - ( - meshes_dir, - node, - tree, - labels, - annotated_volume, - ROOT_ID, - CLOSING_N_ITERS, - DECIMATE_FRACTION, - SMOOTH, + else: + create_region_mesh( + ( + meshes_dir, + node, + tree, + labels, + annotated_volume, + ROOT_ID, + CLOSING_N_ITERS, + DECIMATE_FRACTION, + SMOOTH, + ) ) - ) - - meshes_dict[node.identifier] = filename print( "Finished mesh extraction in : ", @@ -302,6 +312,19 @@ def recurse(sub_d): " minutes", ) + # Once mesh creation is over, rescale + for mesh_id, meshfile in meshes_dict.items(): + # Check if mesh is barrel-related + if mesh_id in [s["id"] for s in dict_to_add]: + + try: + mesh = mio.read(meshfile) + mesh.points *= resolution + mio.write(meshfile, mesh) + except: + print(f"Mesh file {meshfile} not found.") + + # Loop over structures, remove entries not used: for struct in structs_with_mesh: [ From d98b34a829bd0856cd52651398a90d8c74ccbd27 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:49:15 +0000 Subject: [PATCH 15/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../atlas_scripts/allen_mouse_bluebrain_barrels.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index cb7806e3..bc46f810 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -3,11 +3,10 @@ import time from pathlib import Path +import meshio as mio import nrrd import numpy as np import pooch -import meshio as mio - from allensdk.api.queries.ontologies_api import OntologiesApi from allensdk.api.queries.reference_space_api import ReferenceSpaceApi from allensdk.core.reference_space_cache import ReferenceSpaceCache @@ -235,7 +234,11 @@ def recurse(sub_d): print(d["name"]) # Directory for mesh saving: - meshes_dir = working_dir / descriptors.MESHES_DIRNAME / 'meshes_{}'.format(resolution) + meshes_dir = ( + working_dir + / descriptors.MESHES_DIRNAME + / "meshes_{}".format(resolution) + ) # If directory exists, then skip if not meshes_dir.exists(): meshes_dir.mkdir(exist_ok=False) @@ -255,7 +258,7 @@ def recurse(sub_d): space.download_structure_mesh( structure_id=s["id"], ccf_version="annotation/ccf_2017", - file_name=filename, + file_name=filename, ) meshes_dict[name] = filename except (exceptions.HTTPError, ConnectionError): @@ -324,7 +327,6 @@ def recurse(sub_d): except: print(f"Mesh file {meshfile} not found.") - # Loop over structures, remove entries not used: for struct in structs_with_mesh: [ From 9e1b1f759d2e2ddd25090cd4708747e54f9fdb7b Mon Sep 17 00:00:00 2001 From: Adam Tyson Date: Sun, 21 Jul 2024 09:12:06 +0100 Subject: [PATCH 16/22] Ensure parent directory exists --- .../atlas_scripts/allen_mouse_bluebrain_barrels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index bc46f810..bcb62591 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -241,7 +241,7 @@ def recurse(sub_d): ) # If directory exists, then skip if not meshes_dir.exists(): - meshes_dir.mkdir(exist_ok=False) + meshes_dir.mkdir(exist_ok=False, parents=True) # Download existing Allen meshes: space = ReferenceSpaceApi() From 84ce3b857ee3875bc33c560aa265f55c405d2967 Mon Sep 17 00:00:00 2001 From: Adam Tyson Date: Sun, 21 Jul 2024 09:12:19 +0100 Subject: [PATCH 17/22] Update version --- .../atlas_scripts/allen_mouse_bluebrain_barrels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index bcb62591..5c93d059 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -1,4 +1,4 @@ -__version__ = "0.1" +__version__ = "1" import json import time from pathlib import Path From eb431eaf4d98e85fe5beaf1d737c162346f83c16 Mon Sep 17 00:00:00 2001 From: Adam Tyson Date: Sun, 21 Jul 2024 10:59:03 +0100 Subject: [PATCH 18/22] Improve exception handling --- .../atlas_scripts/allen_mouse_bluebrain_barrels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 5c93d059..2fb4b481 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -324,7 +324,7 @@ def recurse(sub_d): mesh = mio.read(meshfile) mesh.points *= resolution mio.write(meshfile, mesh) - except: + except mio._exceptions.ReadError: print(f"Mesh file {meshfile} not found.") # Loop over structures, remove entries not used: @@ -360,7 +360,7 @@ def recurse(sub_d): if __name__ == "__main__": - RES_UM = 10 + RES_UM = 25 # Generated atlas path: bg_root_dir = ( Path.home() / "brainglobe_workingdir" / "allen_mouse_bluebrain_barrels" From a2f692022bf4d47236346af9bcd65aef2b7ff2eb Mon Sep 17 00:00:00 2001 From: Adam Tyson Date: Thu, 25 Jul 2024 09:06:17 +0100 Subject: [PATCH 19/22] Add all meshes to meshes_dict --- .../atlas_scripts/allen_mouse_bluebrain_barrels.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 2fb4b481..82167034 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -292,6 +292,7 @@ def recurse(sub_d): # Check if mesh already exists file_name = meshes_dir / f"{node.identifier}.obj" if file_name.exists(): + meshes_dict[name] = filename continue else: @@ -308,6 +309,7 @@ def recurse(sub_d): SMOOTH, ) ) + meshes_dict[name] = filename print( "Finished mesh extraction in : ", From b951f20f1979174fdf405b48816d2d0734c168d3 Mon Sep 17 00:00:00 2001 From: abisi Date: Tue, 30 Jul 2024 21:40:17 +0200 Subject: [PATCH 20/22] test --- .../allen_mouse_bluebrain_barrels.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 2fb4b481..efd037b1 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -1,4 +1,4 @@ -__version__ = "1" +__version__ = "0" import json import time from pathlib import Path @@ -77,6 +77,8 @@ def create_atlas(working_dir, resolution): # Load enhanced annotation volume: annotated_volume = nrrd.read(annotation_file_path)[0] + annotation_file_path = r'C:\Users\bisi\Desktop\barrel_annotations\2017\atlas_barrels_25um\annotation_barrels_25.nrrd' + annotated_volume = nrrd.read(annotation_file_path)[0] # Download structures tree and meshes: ###################################### @@ -107,6 +109,8 @@ def create_atlas(working_dir, resolution): progressbar=True, ) structs_with_barrels = json.load(open(hierarchy_path)) + hierarchy_path = r'C:\Users\bisi\Desktop\barrel_annotations\2017\atlas_barrels_25um\hierarchy.json' + structs_with_barrels = json.load(open(hierarchy_path)) # Add barrels structures to Allen structures def find_dicts_with_key_containing_substring(d, key, substring): @@ -183,7 +187,7 @@ def recurse(sub_d): # Ignore parent-level SSp-bfd layers if d["acronym"] in structures_present: - print("Skipping", d, "already present.") + print("Skipping because already present:", d) continue # Ignore sub-structures layer 2 and 3 to keep layer 2/3 structure if d["graph_order"] == 53 and d["acronym"] in ["SSp-bfd2", "SSp-bfd3"]: @@ -229,9 +233,6 @@ def recurse(sub_d): # Add list of dicts to structs_with_mesh structs_with_mesh = structs_with_mesh + dict_to_add - print("Added the following structures to the atlas:") - for d in dict_to_add: - print(d["name"]) # Directory for mesh saving: meshes_dir = ( @@ -255,11 +256,11 @@ def recurse(sub_d): continue try: - space.download_structure_mesh( - structure_id=s["id"], - ccf_version="annotation/ccf_2017", - file_name=filename, - ) + #space.download_structure_mesh( + # structure_id=s["id"], + # ccf_version="annotation/ccf_2017", + # file_name=filename, + #) meshes_dict[name] = filename except (exceptions.HTTPError, ConnectionError): print(f"Failed to download mesh for {s['name']} ({s['id']})") @@ -273,6 +274,10 @@ def recurse(sub_d): # generate binary mask for mesh creation labels = np.unique(annotated_volume).astype(np.int_) + + # debug: print unique labels + print("Unique labels in annotation volume: ", labels, labels.shape) + for key, node in tree.nodes.items(): if key in labels: is_label = True @@ -360,7 +365,7 @@ def recurse(sub_d): if __name__ == "__main__": - RES_UM = 25 + RES_UM = 10 # Generated atlas path: bg_root_dir = ( Path.home() / "brainglobe_workingdir" / "allen_mouse_bluebrain_barrels" From be74f37205240e01b7d5285d9463f622063e59d6 Mon Sep 17 00:00:00 2001 From: abisi Date: Tue, 30 Jul 2024 23:09:26 +0200 Subject: [PATCH 21/22] test with new file + handling strct 545 --- .../allen_mouse_bluebrain_barrels.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index cd83c1a0..620cf7e4 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -77,8 +77,6 @@ def create_atlas(working_dir, resolution): # Load enhanced annotation volume: annotated_volume = nrrd.read(annotation_file_path)[0] - annotation_file_path = r'C:\Users\bisi\Desktop\barrel_annotations\2017\atlas_barrels_25um\annotation_barrels_25.nrrd' - annotated_volume = nrrd.read(annotation_file_path)[0] # Download structures tree and meshes: ###################################### @@ -109,8 +107,6 @@ def create_atlas(working_dir, resolution): progressbar=True, ) structs_with_barrels = json.load(open(hierarchy_path)) - hierarchy_path = r'C:\Users\bisi\Desktop\barrel_annotations\2017\atlas_barrels_25um\hierarchy.json' - structs_with_barrels = json.load(open(hierarchy_path)) # Add barrels structures to Allen structures def find_dicts_with_key_containing_substring(d, key, substring): @@ -256,11 +252,11 @@ def recurse(sub_d): continue try: - #space.download_structure_mesh( - # structure_id=s["id"], - # ccf_version="annotation/ccf_2017", - # file_name=filename, - #) + space.download_structure_mesh( + structure_id=s["id"], + ccf_version="annotation/ccf_2017", + file_name=filename, + ) meshes_dict[name] = filename except (exceptions.HTTPError, ConnectionError): print(f"Failed to download mesh for {s['name']} ({s['id']})") @@ -275,9 +271,6 @@ def recurse(sub_d): # generate binary mask for mesh creation labels = np.unique(annotated_volume).astype(np.int_) - # debug: print unique labels - print("Unique labels in annotation volume: ", labels, labels.shape) - for key, node in tree.nodes.items(): if key in labels: is_label = True @@ -341,8 +334,13 @@ def recurse(sub_d): for k in ["graph_id", "structure_set_ids", "graph_order"] ] + # Remove problematic 545 structure + if 545 in meshes_dict.keys(): + meshes_dict.pop(545) + # Wrap up, compress, and remove file:0 print("Finalising atlas") + output_filename = wrapup_atlas_from_data( atlas_name=ATLAS_NAME, atlas_minor_version=__version__, @@ -367,7 +365,7 @@ def recurse(sub_d): if __name__ == "__main__": - RES_UM = 10 + RES_UM = 25 # Generated atlas path: bg_root_dir = ( Path.home() / "brainglobe_workingdir" / "allen_mouse_bluebrain_barrels" From 1d6f6846dcc718298e39dfc8ee8b1debadf4fdb5 Mon Sep 17 00:00:00 2001 From: Adam Tyson Date: Fri, 2 Aug 2024 09:36:34 +0100 Subject: [PATCH 22/22] Update filenames in meshes_dict --- .../atlas_scripts/allen_mouse_bluebrain_barrels.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py index 620cf7e4..b6801773 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/allen_mouse_bluebrain_barrels.py @@ -290,7 +290,7 @@ def recurse(sub_d): # Check if mesh already exists file_name = meshes_dir / f"{node.identifier}.obj" if file_name.exists(): - meshes_dict[name] = filename + meshes_dict[node.identifier] = file_name continue else: @@ -307,7 +307,7 @@ def recurse(sub_d): SMOOTH, ) ) - meshes_dict[name] = filename + meshes_dict[node.identifier] = file_name print( "Finished mesh extraction in : ",