Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/add data checks #179

Open
wants to merge 33 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
d55523e
fix: add check for missing/invalid args in project
AngRodrigues Dec 17, 2024
596ddb3
chore: add mention to issue
AngRodrigues Dec 17, 2024
3345bbe
chore: clear warnings
AngRodrigues Jan 7, 2025
b1a94bd
chore: make warnings clear
AngRodrigues Jan 7, 2025
1bb6e47
chore: make warnings clear
AngRodrigues Jan 7, 2025
27ea51d
tests: bypass the necessary dataset requirement
AngRodrigues Jan 7, 2025
c9da10d
fix: skip required file checks if using loop server data
AngRodrigues Jan 7, 2025
713abff
fix: make the config check through project
AngRodrigues Jan 7, 2025
b52d540
tests: add tests for config checks
AngRodrigues Jan 7, 2025
6322872
style: style fixes by ruff and autoformatting by black
AngRodrigues Jan 7, 2025
554ed6a
fiz: actually use libmamba on build
AngRodrigues Jan 7, 2025
f9752b2
revert previous commit
AngRodrigues Jan 7, 2025
9be3ac6
fix: add init commit
AngRodrigues Jan 8, 2025
29466fe
style: style fixes by ruff and autoformatting by black
AngRodrigues Jan 8, 2025
848f31c
tests: add tests for data_checks for each datatype
AngRodrigues Jan 8, 2025
0257a55
Merge branch 'fix/add_data_checks' of https://github.com/Loop3D/map2l…
AngRodrigues Jan 8, 2025
c68ce9a
fix: actually abort the process if validation fails
AngRodrigues Jan 8, 2025
b9663d6
Merge branch 'fix--162' of https://github.com/Loop3D/map2loop into fi…
AngRodrigues Jan 8, 2025
06ff551
fix: add config check step to project
AngRodrigues Jan 8, 2025
6529bf8
fix: add extra checks for config dictionary
AngRodrigues Jan 8, 2025
ce44489
tests: add config test and reorganise
AngRodrigues Jan 8, 2025
0a85eac
fix: add data checks for fold data and update tests accordingly
AngRodrigues Jan 8, 2025
9eb8fed
Merge branch 'master' of https://github.com/Loop3D/map2loop into fix/…
AngRodrigues Jan 13, 2025
273b82d
typos from merging
AngRodrigues Jan 13, 2025
04b43f1
chore: another typo
AngRodrigues Jan 13, 2025
fc2e393
fix: remove kwargs from project
AngRodrigues Jan 13, 2025
e919f94
chore: refactor geometry checks
AngRodrigues Jan 13, 2025
0002262
chore: update tests for geometry refactor
AngRodrigues Jan 13, 2025
b066a22
chore: refactor id checks
AngRodrigues Jan 14, 2025
141c39f
Merge branch 'master' of https://github.com/Loop3D/map2loop into fix/…
AngRodrigues Jan 14, 2025
9001a08
chore: refactor mandatory fields for str and geo
AngRodrigues Jan 14, 2025
652281c
finalise refactor
AngRodrigues Jan 14, 2025
d1f67a9
chore: finalise details
AngRodrigues Jan 14, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 1 addition & 61 deletions map2loop/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,6 @@ def update_from_dictionary(self, dictionary: dict, lower: bool = True):
Args:
dictionary (dict): The dictionary to update from
"""
# make sure dictionary doesn't contain legacy keys
self.check_for_legacy_keys(dictionary)

# make sure it has the minimum requirements
self.validate_config_dictionary(dictionary)

if "structure" in dictionary:
self.structure_config.update(dictionary["structure"])
Expand Down Expand Up @@ -214,59 +209,4 @@ def update_from_file(
else:
err_string += "Please check the file exists and is accessible then\n"
err_string += "Check the contents for mismatched quotes or brackets!"
raise Exception(err_string)

@beartype.beartype
def validate_config_dictionary(self, config_dict: dict) -> None:
"""
Validate the structure and keys of the configuration dictionary.

Args:
config_dict (dict): The config dictionary to validate.

Raises:
ValueError: If the dictionary does not meet the minimum requirements for ma2p2loop.
"""
required_keys = {
"structure": {"dipdir_column", "dip_column"},
"geology": {"unitname_column", "alt_unitname_column"},
}

for section, keys in required_keys.items():
if section not in config_dict:
logger.error(f"Missing required section '{section}' in config dictionary.")
raise ValueError(f"Missing required section '{section}' in config dictionary.")

for key in keys:
if key not in config_dict[section]:
logger.error(
f"Missing required key '{key}' for '{section}' section of the config dictionary."
)
raise ValueError(
f"Missing required key '{key}' for '{section}' section of the config dictionary."
)

@beartype.beartype
def check_for_legacy_keys(self, config_dict: dict) -> None:

legacy_keys = {
"otype", "dd", "d", "sf", "bedding", "bo", "btype", "gi", "c", "u",
"g", "g2", "ds", "min", "max", "r1", "r2", "sill", "intrusive", "volcanic",
"f", "fdipnull", "fdipdip_flag", "fdipdir", "fdip", "fdipest",
"fdipest_vals", "n", "ff", "t", "syn"
}

# Recursively search for keys in the dictionary
def check_keys(d: dict, parent_key=""):
for key, value in d.items():
if key in legacy_keys:
logger.error(
f"Legacy key found in config - '{key}' at '{parent_key + key}'. Please use the new config format. Use map2loop.utils.update_from_legacy_file to convert between the formats if needed"
)
raise ValueError(
f"Legacy key found in config - '{key}' at '{parent_key + key}'. Please use the new config format. Use map2loop.utils.update_from_legacy_file to convert between the formats if needed"
)
if isinstance(value, dict):
check_keys(value, parent_key=f"{parent_key}{key}.")

check_keys(config_dict)
raise Exception(err_string)
Loading