From f0128e7302e06d739dc8a32c918af2278a73ab01 Mon Sep 17 00:00:00 2001 From: Viktor Plattner Date: Mon, 20 Jan 2025 09:01:01 +0000 Subject: [PATCH 01/14] first commit, refactoring wrapup and validation --- .../atlas_generation/validate_atlases.py | 34 ++++++++++--- .../atlas_generation/wrapup.py | 49 +++++++++++++++++++ 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/validate_atlases.py b/brainglobe_atlasapi/atlas_generation/validate_atlases.py index cf4183e2..d76129d1 100644 --- a/brainglobe_atlasapi/atlas_generation/validate_atlases.py +++ b/brainglobe_atlasapi/atlas_generation/validate_atlases.py @@ -15,14 +15,22 @@ ) from brainglobe_atlasapi.update_atlases import update_atlas +from brainglobe_atlasapi.config import DEFAULT_WORKDIR def validate_atlas_files(atlas: BrainGlobeAtlas): """Checks if basic files exist in the atlas folder""" + # atlas_path = ( + # Path(get_brainglobe_dir()) + # / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + # ) + atlas_path = ( - Path(get_brainglobe_dir()) - / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + Path(DEFAULT_WORKDIR + / f"{atlas.metadata['name']}" + / f"{atlas.atlas_name}_v{atlas.metadata['version']}") ) + assert atlas_path.is_dir(), f"Atlas path {atlas_path} not found" expected_files = [ "annotation.tiff", @@ -150,10 +158,17 @@ def catch_missing_mesh_files(atlas: BrainGlobeAtlas): ids_from_bg_atlas_api = list(atlas.structures.keys()) + # atlas_path = ( + # Path(get_brainglobe_dir()) + # / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + # ) + atlas_path = ( - Path(get_brainglobe_dir()) - / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + Path(DEFAULT_WORKDIR + / f"{atlas.metadata['name']}" + / f"{atlas.atlas_name}_v{atlas.metadata['version']}") ) + obj_path = Path(atlas_path / "meshes") ids_from_mesh_files = [ @@ -182,10 +197,17 @@ def catch_missing_structures(atlas: BrainGlobeAtlas): ids_from_bg_atlas_api = list(atlas.structures.keys()) + # atlas_path = ( + # Path(get_brainglobe_dir()) + # / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + # ) + atlas_path = ( - Path(get_brainglobe_dir()) - / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + Path(DEFAULT_WORKDIR + / f"{atlas.metadata['name']}" + / f"{atlas.atlas_name}_v{atlas.metadata['version']}") ) + obj_path = Path(atlas_path / "meshes") ids_from_mesh_files = [ diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index fc86fe1d..62a41065 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -24,6 +24,19 @@ ) from brainglobe_atlasapi.utils import atlas_name_from_repr +from brainglobe_atlasapi import BrainGlobeAtlas +from brainglobe_atlasapi.atlas_generation.validate_atlases import ( + validate_atlas_files, + validate_mesh_matches_image_extents, + open_for_visual_check, + validate_checksum, + validate_image_dimensions, + validate_additional_references, + catch_missing_structures, + catch_missing_mesh_files +) + + # This should be changed every time we make changes in the atlas # structure: ATLAS_VERSION = brainglobe_atlasapi.atlas_generation.__version__ @@ -129,6 +142,11 @@ def wrapup_atlas_from_data( atlas_dir_name = atlas_name_from_repr( atlas_name, resolution[0], ATLAS_VERSION, atlas_minor_version ) + + atlas_name_for_validation = atlas_name_from_repr( + atlas_name, resolution[0] + ) + dest_dir = Path(working_dir) / atlas_dir_name # exist_ok would be more permissive but error-prone here as there might @@ -231,6 +249,36 @@ def wrapup_atlas_from_data( additional_metadata=additional_metadata, ) + atlas_to_validate = BrainGlobeAtlas( + atlas_name=atlas_name_for_validation, + brainglobe_dir=working_dir, + check_latest=False + ) + + # Run validation functions + print(f"Running atlas validation on {atlas_dir_name}") + + validation_results = {} + + for func in [ + validate_atlas_files, + validate_mesh_matches_image_extents, + open_for_visual_check, + validate_checksum, + validate_image_dimensions, + validate_additional_references, + catch_missing_structures, + catch_missing_mesh_files + ]: + try: + func(atlas_to_validate) + validation_results[func.__name__] = "Pass" + except AssertionError as e: + print(f"Validation failed: {e}") + validation_results[func.__name__] = f"Fail: {str(e)}" + + print("Validation complete") + # Compress if required: if compress: output_filename = dest_dir.parent / f"{dest_dir.name}.tar.gz" @@ -240,6 +288,7 @@ def wrapup_atlas_from_data( # Cleanup if required: if cleanup_files: + print(f"Cleaning up atlas data at: {dest_dir}") # Clean temporary directory and remove it: shutil.rmtree(dest_dir) From 85b9e8b236c84d174e47a31e97de1eae5a65e6c4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 14:43:05 +0000 Subject: [PATCH 02/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../atlas_generation/validate_atlases.py | 28 +++++++++---------- .../atlas_generation/wrapup.py | 25 +++++++---------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/validate_atlases.py b/brainglobe_atlasapi/atlas_generation/validate_atlases.py index d76129d1..5e1b83f9 100644 --- a/brainglobe_atlasapi/atlas_generation/validate_atlases.py +++ b/brainglobe_atlasapi/atlas_generation/validate_atlases.py @@ -7,15 +7,13 @@ import numpy as np from brainglobe_atlasapi import BrainGlobeAtlas -from brainglobe_atlasapi.config import get_brainglobe_dir +from brainglobe_atlasapi.config import DEFAULT_WORKDIR, get_brainglobe_dir from brainglobe_atlasapi.list_atlases import ( get_all_atlases_lastversions, get_atlases_lastversions, - get_local_atlas_version, ) from brainglobe_atlasapi.update_atlases import update_atlas -from brainglobe_atlasapi.config import DEFAULT_WORKDIR def validate_atlas_files(atlas: BrainGlobeAtlas): """Checks if basic files exist in the atlas folder""" @@ -25,10 +23,10 @@ def validate_atlas_files(atlas: BrainGlobeAtlas): # / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" # ) - atlas_path = ( - Path(DEFAULT_WORKDIR - / f"{atlas.metadata['name']}" - / f"{atlas.atlas_name}_v{atlas.metadata['version']}") + atlas_path = Path( + DEFAULT_WORKDIR + / f"{atlas.metadata['name']}" + / f"{atlas.atlas_name}_v{atlas.metadata['version']}" ) assert atlas_path.is_dir(), f"Atlas path {atlas_path} not found" @@ -163,10 +161,10 @@ def catch_missing_mesh_files(atlas: BrainGlobeAtlas): # / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" # ) - atlas_path = ( - Path(DEFAULT_WORKDIR - / f"{atlas.metadata['name']}" - / f"{atlas.atlas_name}_v{atlas.metadata['version']}") + atlas_path = Path( + DEFAULT_WORKDIR + / f"{atlas.metadata['name']}" + / f"{atlas.atlas_name}_v{atlas.metadata['version']}" ) obj_path = Path(atlas_path / "meshes") @@ -202,10 +200,10 @@ def catch_missing_structures(atlas: BrainGlobeAtlas): # / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" # ) - atlas_path = ( - Path(DEFAULT_WORKDIR - / f"{atlas.metadata['name']}" - / f"{atlas.atlas_name}_v{atlas.metadata['version']}") + atlas_path = Path( + DEFAULT_WORKDIR + / f"{atlas.metadata['name']}" + / f"{atlas.atlas_name}_v{atlas.metadata['version']}" ) obj_path = Path(atlas_path / "meshes") diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index 62a41065..d0d1f166 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -8,7 +8,7 @@ import tifffile import brainglobe_atlasapi.atlas_generation -from brainglobe_atlasapi import descriptors +from brainglobe_atlasapi import BrainGlobeAtlas, descriptors from brainglobe_atlasapi.atlas_generation.metadata_utils import ( create_metadata_files, generate_metadata_dict, @@ -22,20 +22,17 @@ from brainglobe_atlasapi.atlas_generation.structures import ( check_struct_consistency, ) -from brainglobe_atlasapi.utils import atlas_name_from_repr - -from brainglobe_atlasapi import BrainGlobeAtlas from brainglobe_atlasapi.atlas_generation.validate_atlases import ( - validate_atlas_files, - validate_mesh_matches_image_extents, + catch_missing_mesh_files, + catch_missing_structures, open_for_visual_check, + validate_additional_references, + validate_atlas_files, validate_checksum, validate_image_dimensions, - validate_additional_references, - catch_missing_structures, - catch_missing_mesh_files + validate_mesh_matches_image_extents, ) - +from brainglobe_atlasapi.utils import atlas_name_from_repr # This should be changed every time we make changes in the atlas # structure: @@ -143,9 +140,7 @@ def wrapup_atlas_from_data( atlas_name, resolution[0], ATLAS_VERSION, atlas_minor_version ) - atlas_name_for_validation = atlas_name_from_repr( - atlas_name, resolution[0] - ) + atlas_name_for_validation = atlas_name_from_repr(atlas_name, resolution[0]) dest_dir = Path(working_dir) / atlas_dir_name @@ -252,7 +247,7 @@ def wrapup_atlas_from_data( atlas_to_validate = BrainGlobeAtlas( atlas_name=atlas_name_for_validation, brainglobe_dir=working_dir, - check_latest=False + check_latest=False, ) # Run validation functions @@ -268,7 +263,7 @@ def wrapup_atlas_from_data( validate_image_dimensions, validate_additional_references, catch_missing_structures, - catch_missing_mesh_files + catch_missing_mesh_files, ]: try: func(atlas_to_validate) From 31b518bf5cf9ade91f98788076f72e71c85d0acb Mon Sep 17 00:00:00 2001 From: Viktor Plattner Date: Mon, 20 Jan 2025 16:25:58 +0000 Subject: [PATCH 03/14] adding custom_atlas_path argument to validation functions --- .../atlas_generation/validate_atlases.py | 66 +++++++++---------- .../atlas_generation/wrapup.py | 36 +++++----- 2 files changed, 52 insertions(+), 50 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/validate_atlases.py b/brainglobe_atlasapi/atlas_generation/validate_atlases.py index d76129d1..be108edb 100644 --- a/brainglobe_atlasapi/atlas_generation/validate_atlases.py +++ b/brainglobe_atlasapi/atlas_generation/validate_atlases.py @@ -17,19 +17,19 @@ from brainglobe_atlasapi.config import DEFAULT_WORKDIR -def validate_atlas_files(atlas: BrainGlobeAtlas): +def validate_atlas_files(atlas: BrainGlobeAtlas, custom_atlas_path: Path = None): """Checks if basic files exist in the atlas folder""" - # atlas_path = ( - # Path(get_brainglobe_dir()) - # / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" - # ) - - atlas_path = ( - Path(DEFAULT_WORKDIR - / f"{atlas.metadata['name']}" - / f"{atlas.atlas_name}_v{atlas.metadata['version']}") - ) + if custom_atlas_path is None: + atlas_path = ( + Path(get_brainglobe_dir()) + / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + ) + else: + atlas_path = ( + Path(custom_atlas_path + / f"{atlas.atlas_name}_v{atlas.metadata['version']}") + ) assert atlas_path.is_dir(), f"Atlas path {atlas_path} not found" expected_files = [ @@ -151,23 +151,23 @@ def validate_additional_references(atlas: BrainGlobeAtlas): return True -def catch_missing_mesh_files(atlas: BrainGlobeAtlas): +def catch_missing_mesh_files(atlas: BrainGlobeAtlas, custom_atlas_path: Path = None): """ Checks if all the structures in the atlas have a corresponding mesh file """ ids_from_bg_atlas_api = list(atlas.structures.keys()) - # atlas_path = ( - # Path(get_brainglobe_dir()) - # / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" - # ) - - atlas_path = ( - Path(DEFAULT_WORKDIR - / f"{atlas.metadata['name']}" - / f"{atlas.atlas_name}_v{atlas.metadata['version']}") - ) + if custom_atlas_path is None: + atlas_path = ( + Path(get_brainglobe_dir()) + / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + ) + else: + atlas_path = ( + Path(custom_atlas_path + / f"{atlas.atlas_name}_v{atlas.metadata['version']}") + ) obj_path = Path(atlas_path / "meshes") @@ -189,7 +189,7 @@ def catch_missing_mesh_files(atlas: BrainGlobeAtlas): ) -def catch_missing_structures(atlas: BrainGlobeAtlas): +def catch_missing_structures(atlas: BrainGlobeAtlas, custom_atlas_path: Path = None): """ Checks if all the mesh files in the atlas folder are listed as a structure in the atlas. @@ -197,16 +197,16 @@ def catch_missing_structures(atlas: BrainGlobeAtlas): ids_from_bg_atlas_api = list(atlas.structures.keys()) - # atlas_path = ( - # Path(get_brainglobe_dir()) - # / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" - # ) - - atlas_path = ( - Path(DEFAULT_WORKDIR - / f"{atlas.metadata['name']}" - / f"{atlas.atlas_name}_v{atlas.metadata['version']}") - ) + if custom_atlas_path is None: + atlas_path = ( + Path(get_brainglobe_dir()) + / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + ) + else: + atlas_path = ( + Path(custom_atlas_path + / f"{atlas.atlas_name}_v{atlas.metadata['version']}") + ) obj_path = Path(atlas_path / "meshes") diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index 62a41065..417af263 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -143,10 +143,6 @@ def wrapup_atlas_from_data( atlas_name, resolution[0], ATLAS_VERSION, atlas_minor_version ) - atlas_name_for_validation = atlas_name_from_repr( - atlas_name, resolution[0] - ) - dest_dir = Path(working_dir) / atlas_dir_name # exist_ok would be more permissive but error-prone here as there might @@ -249,6 +245,11 @@ def wrapup_atlas_from_data( additional_metadata=additional_metadata, ) + atlas_name_for_validation = atlas_name_from_repr( + atlas_name, resolution[0] + ) + + # creating BrainGlobe object from local folder (working_dir) atlas_to_validate = BrainGlobeAtlas( atlas_name=atlas_name_for_validation, brainglobe_dir=working_dir, @@ -260,25 +261,26 @@ def wrapup_atlas_from_data( validation_results = {} - for func in [ - validate_atlas_files, - validate_mesh_matches_image_extents, - open_for_visual_check, - validate_checksum, - validate_image_dimensions, - validate_additional_references, - catch_missing_structures, - catch_missing_mesh_files - ]: + functions = [ + (validate_atlas_files, atlas_to_validate, working_dir), + (catch_missing_structures, atlas_to_validate, working_dir), + (catch_missing_mesh_files, atlas_to_validate, working_dir), + (validate_mesh_matches_image_extents, atlas_to_validate), + (open_for_visual_check, atlas_to_validate), + (validate_checksum, atlas_to_validate), + (validate_image_dimensions, atlas_to_validate), + (validate_additional_references, atlas_to_validate) + ] + + for func, *args in functions: try: - func(atlas_to_validate) + func(*args) validation_results[func.__name__] = "Pass" + print(f"{atlas_dir_name} passed {func.__name__}") except AssertionError as e: print(f"Validation failed: {e}") validation_results[func.__name__] = f"Fail: {str(e)}" - print("Validation complete") - # Compress if required: if compress: output_filename = dest_dir.parent / f"{dest_dir.name}.tar.gz" From ba6a9aa51c3c73c12688f41c3807ba3240dc027c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 16:35:43 +0000 Subject: [PATCH 04/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../atlas_generation/validate_atlases.py | 43 +++++++++++-------- .../atlas_generation/wrapup.py | 25 +++++------ 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/validate_atlases.py b/brainglobe_atlasapi/atlas_generation/validate_atlases.py index be108edb..a3de8500 100644 --- a/brainglobe_atlasapi/atlas_generation/validate_atlases.py +++ b/brainglobe_atlasapi/atlas_generation/validate_atlases.py @@ -15,20 +15,21 @@ ) from brainglobe_atlasapi.update_atlases import update_atlas -from brainglobe_atlasapi.config import DEFAULT_WORKDIR -def validate_atlas_files(atlas: BrainGlobeAtlas, custom_atlas_path: Path = None): +def validate_atlas_files( + atlas: BrainGlobeAtlas, custom_atlas_path: Path = None +): """Checks if basic files exist in the atlas folder""" if custom_atlas_path is None: atlas_path = ( - Path(get_brainglobe_dir()) - / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + Path(get_brainglobe_dir()) + / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" ) else: - atlas_path = ( - Path(custom_atlas_path - / f"{atlas.atlas_name}_v{atlas.metadata['version']}") + atlas_path = Path( + custom_atlas_path + / f"{atlas.atlas_name}_v{atlas.metadata['version']}" ) assert atlas_path.is_dir(), f"Atlas path {atlas_path} not found" @@ -151,7 +152,9 @@ def validate_additional_references(atlas: BrainGlobeAtlas): return True -def catch_missing_mesh_files(atlas: BrainGlobeAtlas, custom_atlas_path: Path = None): +def catch_missing_mesh_files( + atlas: BrainGlobeAtlas, custom_atlas_path: Path = None +): """ Checks if all the structures in the atlas have a corresponding mesh file """ @@ -160,13 +163,13 @@ def catch_missing_mesh_files(atlas: BrainGlobeAtlas, custom_atlas_path: Path = N if custom_atlas_path is None: atlas_path = ( - Path(get_brainglobe_dir()) - / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + Path(get_brainglobe_dir()) + / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" ) else: - atlas_path = ( - Path(custom_atlas_path - / f"{atlas.atlas_name}_v{atlas.metadata['version']}") + atlas_path = Path( + custom_atlas_path + / f"{atlas.atlas_name}_v{atlas.metadata['version']}" ) obj_path = Path(atlas_path / "meshes") @@ -189,7 +192,9 @@ def catch_missing_mesh_files(atlas: BrainGlobeAtlas, custom_atlas_path: Path = N ) -def catch_missing_structures(atlas: BrainGlobeAtlas, custom_atlas_path: Path = None): +def catch_missing_structures( + atlas: BrainGlobeAtlas, custom_atlas_path: Path = None +): """ Checks if all the mesh files in the atlas folder are listed as a structure in the atlas. @@ -199,13 +204,13 @@ def catch_missing_structures(atlas: BrainGlobeAtlas, custom_atlas_path: Path = N if custom_atlas_path is None: atlas_path = ( - Path(get_brainglobe_dir()) - / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + Path(get_brainglobe_dir()) + / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" ) else: - atlas_path = ( - Path(custom_atlas_path - / f"{atlas.atlas_name}_v{atlas.metadata['version']}") + atlas_path = Path( + custom_atlas_path + / f"{atlas.atlas_name}_v{atlas.metadata['version']}" ) obj_path = Path(atlas_path / "meshes") diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index 417af263..5b1a8a82 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -8,7 +8,7 @@ import tifffile import brainglobe_atlasapi.atlas_generation -from brainglobe_atlasapi import descriptors +from brainglobe_atlasapi import BrainGlobeAtlas, descriptors from brainglobe_atlasapi.atlas_generation.metadata_utils import ( create_metadata_files, generate_metadata_dict, @@ -22,20 +22,17 @@ from brainglobe_atlasapi.atlas_generation.structures import ( check_struct_consistency, ) -from brainglobe_atlasapi.utils import atlas_name_from_repr - -from brainglobe_atlasapi import BrainGlobeAtlas from brainglobe_atlasapi.atlas_generation.validate_atlases import ( - validate_atlas_files, - validate_mesh_matches_image_extents, + catch_missing_mesh_files, + catch_missing_structures, open_for_visual_check, + validate_additional_references, + validate_atlas_files, validate_checksum, validate_image_dimensions, - validate_additional_references, - catch_missing_structures, - catch_missing_mesh_files + validate_mesh_matches_image_extents, ) - +from brainglobe_atlasapi.utils import atlas_name_from_repr # This should be changed every time we make changes in the atlas # structure: @@ -245,15 +242,13 @@ def wrapup_atlas_from_data( additional_metadata=additional_metadata, ) - atlas_name_for_validation = atlas_name_from_repr( - atlas_name, resolution[0] - ) + atlas_name_for_validation = atlas_name_from_repr(atlas_name, resolution[0]) # creating BrainGlobe object from local folder (working_dir) atlas_to_validate = BrainGlobeAtlas( atlas_name=atlas_name_for_validation, brainglobe_dir=working_dir, - check_latest=False + check_latest=False, ) # Run validation functions @@ -269,7 +264,7 @@ def wrapup_atlas_from_data( (open_for_visual_check, atlas_to_validate), (validate_checksum, atlas_to_validate), (validate_image_dimensions, atlas_to_validate), - (validate_additional_references, atlas_to_validate) + (validate_additional_references, atlas_to_validate), ] for func, *args in functions: From 41b9d283afb6d9de874b2d491e33c0e013a1c643 Mon Sep 17 00:00:00 2001 From: Viktor Plattner Date: Mon, 20 Jan 2025 17:22:08 +0000 Subject: [PATCH 05/14] updating docstrings --- .../atlas_generation/validate_atlases.py | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/validate_atlases.py b/brainglobe_atlasapi/atlas_generation/validate_atlases.py index a3de8500..59f73294 100644 --- a/brainglobe_atlasapi/atlas_generation/validate_atlases.py +++ b/brainglobe_atlasapi/atlas_generation/validate_atlases.py @@ -19,12 +19,18 @@ def validate_atlas_files( atlas: BrainGlobeAtlas, custom_atlas_path: Path = None ): - """Checks if basic files exist in the atlas folder""" + """ + Checks if basic files exist in the atlas folder + + custom_atlas_path is used when the function is called as part of + the wrapup function in the atlas packaging script. The expected + input is working_dir + """ if custom_atlas_path is None: atlas_path = ( - Path(get_brainglobe_dir()) - / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + Path(get_brainglobe_dir()) / f"{atlas.atlas_name}_v" + f"{get_local_atlas_version(atlas.atlas_name)}" ) else: atlas_path = Path( @@ -157,14 +163,18 @@ def catch_missing_mesh_files( ): """ Checks if all the structures in the atlas have a corresponding mesh file + + custom_atlas_path is used when the function is called as part of + the wrapup function in the atlas packaging script. The expected + input is working_dir """ ids_from_bg_atlas_api = list(atlas.structures.keys()) if custom_atlas_path is None: atlas_path = ( - Path(get_brainglobe_dir()) - / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + Path(get_brainglobe_dir()) / f"{atlas.atlas_name}_v" + f"{get_local_atlas_version(atlas.atlas_name)}" ) else: atlas_path = Path( @@ -198,14 +208,18 @@ def catch_missing_structures( """ Checks if all the mesh files in the atlas folder are listed as a structure in the atlas. + + custom_atlas_path is used when the function is called as part of + the wrapup function in the atlas packaging script. The expected + input is working_dir """ ids_from_bg_atlas_api = list(atlas.structures.keys()) if custom_atlas_path is None: atlas_path = ( - Path(get_brainglobe_dir()) - / f"{atlas.atlas_name}_v{get_local_atlas_version(atlas.atlas_name)}" + Path(get_brainglobe_dir()) / f"{atlas.atlas_name}_v" + f"{get_local_atlas_version(atlas.atlas_name)}" ) else: atlas_path = Path( From c6ea6b302884728e5d80cc2eeb07551aa36edf19 Mon Sep 17 00:00:00 2001 From: Viktor Plattner Date: Mon, 20 Jan 2025 17:23:28 +0000 Subject: [PATCH 06/14] updating validation messages --- brainglobe_atlasapi/atlas_generation/wrapup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index 5b1a8a82..92cdc20f 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -271,9 +271,9 @@ def wrapup_atlas_from_data( try: func(*args) validation_results[func.__name__] = "Pass" - print(f"{atlas_dir_name} passed {func.__name__}") + print(f"passed {func.__name__}") except AssertionError as e: - print(f"Validation failed: {e}") + print(f"failed {func.__name__}: {e}") validation_results[func.__name__] = f"Fail: {str(e)}" # Compress if required: From 76873203034c72022657e0ad0eff8988c90b620b Mon Sep 17 00:00:00 2001 From: Viktor Plattner Date: Thu, 23 Jan 2025 18:48:59 +0000 Subject: [PATCH 07/14] simplifying atlas path --- .../atlas_generation/validate_atlases.py | 15 +++------------ brainglobe_atlasapi/atlas_generation/wrapup.py | 14 +++++++++++--- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/validate_atlases.py b/brainglobe_atlasapi/atlas_generation/validate_atlases.py index 59f73294..7638018a 100644 --- a/brainglobe_atlasapi/atlas_generation/validate_atlases.py +++ b/brainglobe_atlasapi/atlas_generation/validate_atlases.py @@ -33,10 +33,7 @@ def validate_atlas_files( f"{get_local_atlas_version(atlas.atlas_name)}" ) else: - atlas_path = Path( - custom_atlas_path - / f"{atlas.atlas_name}_v{atlas.metadata['version']}" - ) + atlas_path = custom_atlas_path assert atlas_path.is_dir(), f"Atlas path {atlas_path} not found" expected_files = [ @@ -177,10 +174,7 @@ def catch_missing_mesh_files( f"{get_local_atlas_version(atlas.atlas_name)}" ) else: - atlas_path = Path( - custom_atlas_path - / f"{atlas.atlas_name}_v{atlas.metadata['version']}" - ) + atlas_path = custom_atlas_path obj_path = Path(atlas_path / "meshes") @@ -222,10 +216,7 @@ def catch_missing_structures( f"{get_local_atlas_version(atlas.atlas_name)}" ) else: - atlas_path = Path( - custom_atlas_path - / f"{atlas.atlas_name}_v{atlas.metadata['version']}" - ) + atlas_path = custom_atlas_path obj_path = Path(atlas_path / "meshes") diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index 92cdc20f..5aa38d5c 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -257,9 +257,17 @@ def wrapup_atlas_from_data( validation_results = {} functions = [ - (validate_atlas_files, atlas_to_validate, working_dir), - (catch_missing_structures, atlas_to_validate, working_dir), - (catch_missing_mesh_files, atlas_to_validate, working_dir), + (validate_atlas_files, atlas_to_validate, atlas_to_validate.root_dir), + ( + catch_missing_structures, + atlas_to_validate, + atlas_to_validate.root_dir, + ), + ( + catch_missing_mesh_files, + atlas_to_validate, + atlas_to_validate.root_dir, + ), (validate_mesh_matches_image_extents, atlas_to_validate), (open_for_visual_check, atlas_to_validate), (validate_checksum, atlas_to_validate), From 5676c35189f46916becb67111c9111ef2077bc45 Mon Sep 17 00:00:00 2001 From: Viktor Plattner Date: Thu, 23 Jan 2025 19:16:14 +0000 Subject: [PATCH 08/14] simplifying validation print output, adding helper function to check if all validations passed --- .../atlas_generation/wrapup.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index 5aa38d5c..d0364944 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -279,11 +279,35 @@ def wrapup_atlas_from_data( try: func(*args) validation_results[func.__name__] = "Pass" - print(f"passed {func.__name__}") except AssertionError as e: - print(f"failed {func.__name__}: {e}") validation_results[func.__name__] = f"Fail: {str(e)}" + def _check_validations(validation_results): + # Helper function to check if all validations passed + all_passed = all( + result == "Pass" for result in validation_results.values() + ) + + if all_passed: + print("This atlas is valid") + else: + failed_functions = [ + func + for func, result in validation_results.items() + if result != "Pass" + ] + error_messages = [ + result.split(": ")[1] + for result in validation_results.values() + if result != "Pass" + ] + + print("These validation functions have failed:") + for func, error in zip(failed_functions, error_messages): + print(f"- {func}: {error}") + + _check_validations(validation_results) + # Compress if required: if compress: output_filename = dest_dir.parent / f"{dest_dir.name}.tar.gz" From 1fa55e8b81268abbbd6c0de60211be56ae5df2a7 Mon Sep 17 00:00:00 2001 From: Viktor Plattner Date: Fri, 7 Feb 2025 12:58:18 +0000 Subject: [PATCH 09/14] replacing custom_atlas_path by atlas.root_dir --- .../atlas_generation/validate_atlases.py | 37 +++---------------- .../atlas_generation/wrapup.py | 14 ++----- 2 files changed, 9 insertions(+), 42 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/validate_atlases.py b/brainglobe_atlasapi/atlas_generation/validate_atlases.py index 7638018a..17d1adf1 100644 --- a/brainglobe_atlasapi/atlas_generation/validate_atlases.py +++ b/brainglobe_atlasapi/atlas_generation/validate_atlases.py @@ -11,14 +11,11 @@ from brainglobe_atlasapi.list_atlases import ( get_all_atlases_lastversions, get_atlases_lastversions, - get_local_atlas_version, ) from brainglobe_atlasapi.update_atlases import update_atlas -def validate_atlas_files( - atlas: BrainGlobeAtlas, custom_atlas_path: Path = None -): +def validate_atlas_files(atlas: BrainGlobeAtlas): """ Checks if basic files exist in the atlas folder @@ -27,13 +24,7 @@ def validate_atlas_files( input is working_dir """ - if custom_atlas_path is None: - atlas_path = ( - Path(get_brainglobe_dir()) / f"{atlas.atlas_name}_v" - f"{get_local_atlas_version(atlas.atlas_name)}" - ) - else: - atlas_path = custom_atlas_path + atlas_path = atlas.root_dir assert atlas_path.is_dir(), f"Atlas path {atlas_path} not found" expected_files = [ @@ -155,9 +146,7 @@ def validate_additional_references(atlas: BrainGlobeAtlas): return True -def catch_missing_mesh_files( - atlas: BrainGlobeAtlas, custom_atlas_path: Path = None -): +def catch_missing_mesh_files(atlas: BrainGlobeAtlas): """ Checks if all the structures in the atlas have a corresponding mesh file @@ -168,13 +157,7 @@ def catch_missing_mesh_files( ids_from_bg_atlas_api = list(atlas.structures.keys()) - if custom_atlas_path is None: - atlas_path = ( - Path(get_brainglobe_dir()) / f"{atlas.atlas_name}_v" - f"{get_local_atlas_version(atlas.atlas_name)}" - ) - else: - atlas_path = custom_atlas_path + atlas_path = atlas.root_dir obj_path = Path(atlas_path / "meshes") @@ -196,9 +179,7 @@ def catch_missing_mesh_files( ) -def catch_missing_structures( - atlas: BrainGlobeAtlas, custom_atlas_path: Path = None -): +def catch_missing_structures(atlas: BrainGlobeAtlas): """ Checks if all the mesh files in the atlas folder are listed as a structure in the atlas. @@ -210,13 +191,7 @@ def catch_missing_structures( ids_from_bg_atlas_api = list(atlas.structures.keys()) - if custom_atlas_path is None: - atlas_path = ( - Path(get_brainglobe_dir()) / f"{atlas.atlas_name}_v" - f"{get_local_atlas_version(atlas.atlas_name)}" - ) - else: - atlas_path = custom_atlas_path + atlas_path = atlas.root_dir obj_path = Path(atlas_path / "meshes") diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index d0364944..06002024 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -257,17 +257,9 @@ def wrapup_atlas_from_data( validation_results = {} functions = [ - (validate_atlas_files, atlas_to_validate, atlas_to_validate.root_dir), - ( - catch_missing_structures, - atlas_to_validate, - atlas_to_validate.root_dir, - ), - ( - catch_missing_mesh_files, - atlas_to_validate, - atlas_to_validate.root_dir, - ), + (validate_atlas_files, atlas_to_validate), + (catch_missing_structures, atlas_to_validate), + (catch_missing_mesh_files, atlas_to_validate), (validate_mesh_matches_image_extents, atlas_to_validate), (open_for_visual_check, atlas_to_validate), (validate_checksum, atlas_to_validate), From ed011a24f0a43e75191f1f5852f42760954467c3 Mon Sep 17 00:00:00 2001 From: Viktor Plattner Date: Fri, 7 Feb 2025 13:35:47 +0000 Subject: [PATCH 10/14] adding new validation functions from main --- brainglobe_atlasapi/atlas_generation/validate_atlases.py | 5 +++++ brainglobe_atlasapi/atlas_generation/wrapup.py | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/brainglobe_atlasapi/atlas_generation/validate_atlases.py b/brainglobe_atlasapi/atlas_generation/validate_atlases.py index 04b25138..38f438e2 100644 --- a/brainglobe_atlasapi/atlas_generation/validate_atlases.py +++ b/brainglobe_atlasapi/atlas_generation/validate_atlases.py @@ -259,6 +259,8 @@ def get_all_validation_functions(): return [ validate_atlas_files, validate_mesh_matches_image_extents, + open_for_visual_check, + validate_checksum, validate_image_dimensions, validate_additional_references, catch_missing_mesh_files, @@ -305,6 +307,9 @@ def validate_atlas(atlas_name, version, validation_functions): validate_additional_references, catch_missing_mesh_files, catch_missing_structures, + validate_reference_image_pixels, + validate_annotation_symmetry, + validate_atlas_name, ] valid_atlases = [] diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index 06002024..1b04dab0 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -27,10 +27,13 @@ catch_missing_structures, open_for_visual_check, validate_additional_references, + validate_annotation_symmetry, validate_atlas_files, + validate_atlas_name, validate_checksum, validate_image_dimensions, validate_mesh_matches_image_extents, + validate_reference_image_pixels, ) from brainglobe_atlasapi.utils import atlas_name_from_repr @@ -265,6 +268,9 @@ def wrapup_atlas_from_data( (validate_checksum, atlas_to_validate), (validate_image_dimensions, atlas_to_validate), (validate_additional_references, atlas_to_validate), + (validate_reference_image_pixels, atlas_to_validate), + (validate_annotation_symmetry, atlas_to_validate), + (validate_atlas_name, atlas_to_validate), ] for func, *args in functions: From e13b34e11818d23e15e1064fe08c32a670d2bb0a Mon Sep 17 00:00:00 2001 From: viktorpm <50667179+viktorpm@users.noreply.github.com> Date: Tue, 11 Feb 2025 13:20:22 +0000 Subject: [PATCH 11/14] Update brainglobe_atlasapi/atlas_generation/validate_atlases.py Co-authored-by: Alessandro Felder --- brainglobe_atlasapi/atlas_generation/validate_atlases.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/validate_atlases.py b/brainglobe_atlasapi/atlas_generation/validate_atlases.py index 38f438e2..3b5d9553 100644 --- a/brainglobe_atlasapi/atlas_generation/validate_atlases.py +++ b/brainglobe_atlasapi/atlas_generation/validate_atlases.py @@ -185,10 +185,6 @@ def catch_missing_structures(atlas: BrainGlobeAtlas): """ Checks if all the mesh files in the atlas folder are listed as a structure in the atlas. - - custom_atlas_path is used when the function is called as part of - the wrapup function in the atlas packaging script. The expected - input is working_dir """ ids_from_bg_atlas_api = list(atlas.structures.keys()) From 607816b371357fb5ef08bf27567d6f1411ba6846 Mon Sep 17 00:00:00 2001 From: viktorpm <50667179+viktorpm@users.noreply.github.com> Date: Tue, 11 Feb 2025 13:22:02 +0000 Subject: [PATCH 12/14] Update brainglobe_atlasapi/atlas_generation/wrapup.py Co-authored-by: Alessandro Felder --- brainglobe_atlasapi/atlas_generation/wrapup.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index 1b04dab0..76c0b3c5 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -22,19 +22,7 @@ from brainglobe_atlasapi.atlas_generation.structures import ( check_struct_consistency, ) -from brainglobe_atlasapi.atlas_generation.validate_atlases import ( - catch_missing_mesh_files, - catch_missing_structures, - open_for_visual_check, - validate_additional_references, - validate_annotation_symmetry, - validate_atlas_files, - validate_atlas_name, - validate_checksum, - validate_image_dimensions, - validate_mesh_matches_image_extents, - validate_reference_image_pixels, -) +from brainglobe_atlasapi.atlas_generation.validate_atlases import get_all_validation_functions from brainglobe_atlasapi.utils import atlas_name_from_repr # This should be changed every time we make changes in the atlas From 057bd97efa41da1f8e3cc2cb0778f115e2f674f4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Feb 2025 13:22:10 +0000 Subject: [PATCH 13/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- brainglobe_atlasapi/atlas_generation/wrapup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index 76c0b3c5..7d2598c5 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -22,7 +22,6 @@ from brainglobe_atlasapi.atlas_generation.structures import ( check_struct_consistency, ) -from brainglobe_atlasapi.atlas_generation.validate_atlases import get_all_validation_functions from brainglobe_atlasapi.utils import atlas_name_from_repr # This should be changed every time we make changes in the atlas From 932f6c1f9d49bad3dcde218f5036d8757828ff3b Mon Sep 17 00:00:00 2001 From: Viktor Plattner Date: Tue, 11 Feb 2025 13:47:11 +0000 Subject: [PATCH 14/14] Refactor atlas validation to use `get_all_validation_functions()` - Replaced hardcoded list of validation functions in `wrapup.py` with `get_all_validation_functions()` for better maintainability. - Modified `validate_atlases.py` to return `True` instead of `pass` for unimplemented validation functions. --- .../atlas_generation/validate_atlases.py | 4 ++-- .../atlas_generation/wrapup.py | 21 +++++-------------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/validate_atlases.py b/brainglobe_atlasapi/atlas_generation/validate_atlases.py index 3b5d9553..908fd789 100644 --- a/brainglobe_atlasapi/atlas_generation/validate_atlases.py +++ b/brainglobe_atlasapi/atlas_generation/validate_atlases.py @@ -107,12 +107,12 @@ def validate_mesh_matches_image_extents(atlas: BrainGlobeAtlas): def open_for_visual_check(atlas: BrainGlobeAtlas): # implement visual checks later - pass + return True def validate_checksum(atlas: BrainGlobeAtlas): # implement later - pass + return True def validate_image_dimensions(atlas: BrainGlobeAtlas): diff --git a/brainglobe_atlasapi/atlas_generation/wrapup.py b/brainglobe_atlasapi/atlas_generation/wrapup.py index 7d2598c5..ab6e096b 100644 --- a/brainglobe_atlasapi/atlas_generation/wrapup.py +++ b/brainglobe_atlasapi/atlas_generation/wrapup.py @@ -22,6 +22,9 @@ from brainglobe_atlasapi.atlas_generation.structures import ( check_struct_consistency, ) +from brainglobe_atlasapi.atlas_generation.validate_atlases import ( + get_all_validation_functions, +) from brainglobe_atlasapi.utils import atlas_name_from_repr # This should be changed every time we make changes in the atlas @@ -246,23 +249,9 @@ def wrapup_atlas_from_data( validation_results = {} - functions = [ - (validate_atlas_files, atlas_to_validate), - (catch_missing_structures, atlas_to_validate), - (catch_missing_mesh_files, atlas_to_validate), - (validate_mesh_matches_image_extents, atlas_to_validate), - (open_for_visual_check, atlas_to_validate), - (validate_checksum, atlas_to_validate), - (validate_image_dimensions, atlas_to_validate), - (validate_additional_references, atlas_to_validate), - (validate_reference_image_pixels, atlas_to_validate), - (validate_annotation_symmetry, atlas_to_validate), - (validate_atlas_name, atlas_to_validate), - ] - - for func, *args in functions: + for func in get_all_validation_functions(): try: - func(*args) + func(atlas_to_validate) validation_results[func.__name__] = "Pass" except AssertionError as e: validation_results[func.__name__] = f"Fail: {str(e)}"