From 752f84eab49d5c258df03a7848c0992fcad5d63a Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 3 Jan 2025 15:15:29 -0500 Subject: [PATCH 01/14] make classes with patterns --- src/roman_datamodels/stnode/_factories.py | 126 ++++++++-------------- src/roman_datamodels/stnode/_stnode.py | 15 ++- 2 files changed, 58 insertions(+), 83 deletions(-) diff --git a/src/roman_datamodels/stnode/_factories.py b/src/roman_datamodels/stnode/_factories.py index 8b4ade51..71ae77af 100644 --- a/src/roman_datamodels/stnode/_factories.py +++ b/src/roman_datamodels/stnode/_factories.py @@ -5,7 +5,6 @@ import importlib.resources -import yaml from astropy.time import Time from rad import resources @@ -14,48 +13,37 @@ __all__ = ["stnode_factory"] -# Map of scalar types in the schemas to the python types -SCALAR_TYPE_MAP = { - "string": str, - "http://stsci.edu/schemas/asdf/time/time-1.1.0": Time, +# Map of scalar types by pattern (str is default) +_SCALAR_TYPE_BY_PATTERN = { + "asdf://stsci.edu/datamodels/roman/tags/file_date-*": Time, + "asdf://stsci.edu/datamodels/roman/tags/fps/file_date-": Time, + "asdf://stsci.edu/datamodels/roman/tags/tvac/file_date-": Time, +} +# Map of node types by pattern (TaggedObjectNode is default) +_NODE_TYPE_BY_PATTERN = { + "asdf://stsci.edu/datamodels/roman/tags/cal_logs-*": TaggedListNode, } -BASE_SCHEMA_PATH = importlib.resources.files(resources) / "schemas" - - -def load_schema_from_uri(schema_uri): - """ - Load the actual schema from the rad resources directly (outside ASDF) - Outside ASDF because this has to occur before the ASDF extensions are - registered. +_SCALAR_TAG_BASES = { + "calibration_software_name", + "calibration_software_version", + "product_type", + "filename", + "file_date", + "model_type", + "origin", + "prd_version", + "sdf_software_version", + "telescope", + "prd_software_version", # for tvac and fps +} - Parameters - ---------- - schema_uri : str - The schema_uri found in the RAD manifest +BASE_SCHEMA_PATH = importlib.resources.files(resources) / "schemas" - Returns - ------- - yaml library dictionary from the schema - """ - filename = f"{schema_uri.split('/')[-1]}.yaml" - - if "reference_files" in schema_uri: - schema_path = BASE_SCHEMA_PATH / "reference_files" / filename - elif "/fps/tagged_scalars" in schema_uri: - schema_path = BASE_SCHEMA_PATH / "fps/tagged_scalars" / filename - elif "/fps/" in schema_uri: - schema_path = BASE_SCHEMA_PATH / "fps" / filename - elif "/tvac/tagged_scalars" in schema_uri: - schema_path = BASE_SCHEMA_PATH / "tvac/tagged_scalars" / filename - elif "/tvac/" in schema_uri: - schema_path = BASE_SCHEMA_PATH / "tvac" / filename - elif "tagged_scalars" in schema_uri: - schema_path = BASE_SCHEMA_PATH / "tagged_scalars" / filename - else: - schema_path = BASE_SCHEMA_PATH / filename - return yaml.safe_load(schema_path.read_bytes()) +def is_tagged_scalar_pattern(pattern): + tag_base = pattern.rsplit("-", maxsplit=1)[0].rsplit("/", maxsplit=1)[1] + return tag_base in _SCALAR_TAG_BASES def class_name_from_tag_uri(tag_uri): @@ -79,7 +67,7 @@ def class_name_from_tag_uri(tag_uri): return class_name -def docstring_from_tag(tag): +def docstring_from_tag(pattern): """ Read the docstring (if it exists) from the RAD manifest and generate a docstring for the dynamically generated class. @@ -93,12 +81,14 @@ def docstring_from_tag(tag): ------- A docstring for the class based on the tag """ - docstring = f"{tag['description']}\n\n" if "description" in tag else "" + # TODO broken for now + # docstring = f"{tag['description']}\n\n" if "description" in tag else "" - return docstring + f"Class generated from tag '{tag['tag_uri']}'" + # return docstring + f"Class generated from tag '{tag['tag_uri']}'" + return f"Class generated from tag '{pattern}'" -def scalar_factory(tag): +def scalar_factory(pattern): """ Factory to create a TaggedScalarNode class from a tag @@ -111,30 +101,25 @@ def scalar_factory(tag): ------- A dynamically generated TaggedScalarNode subclass """ - class_name = class_name_from_tag_uri(tag["tag_uri"]) - schema = load_schema_from_uri(tag["schema_uri"]) + class_name = class_name_from_tag_uri(pattern) # TaggedScalarNode subclasses are really subclasses of the type of the scalar, # with the TaggedScalarNode as a mixin. This is because the TaggedScalarNode # is supposed to be the scalar, but it needs to be serializable under a specific # ASDF tag. - # SCALAR_TYPE_MAP will need to be updated as new wrappers of scalar types are added + # _SCALAR_TYPE_BY_PATTERN will need to be updated as new wrappers of scalar types are added # to the RAD manifest. - if "type" in schema: - type_ = schema["type"] - elif "allOf" in schema: - type_ = schema["allOf"][0]["$ref"] - else: - raise RuntimeError(f"Unknown schema type: {schema}") + # assume everything is a string if not otherwise defined + type_ = _SCALAR_TYPE_BY_PATTERN.get(pattern, str) return type( class_name, - (SCALAR_TYPE_MAP[type_], TaggedScalarNode), - {"_tag": tag["tag_uri"], "__module__": "roman_datamodels.stnode", "__doc__": docstring_from_tag(tag)}, + (type_, TaggedScalarNode), + {"_tag": pattern, "__module__": "roman_datamodels.stnode", "__doc__": docstring_from_tag(pattern)}, ) -def node_factory(tag): +def node_factory(pattern): """ Factory to create a TaggedObjectNode or TaggedListNode class from a tag @@ -147,26 +132,9 @@ def node_factory(tag): ------- A dynamically generated TaggedObjectNode or TaggedListNode subclass """ - class_name = class_name_from_tag_uri(tag["tag_uri"]) - schema = load_schema_from_uri(tag["schema_uri"]) - - if "type" in schema: - # Determine if the class is a TaggedObjectNode or TaggedListNode based on the - # type defined in the schema: - # - TaggedObjectNode if type is "object" - # - TaggedListNode if type is "array" (array in jsonschema represents Python list) - if schema["type"] == "object": - class_type = TaggedObjectNode - elif schema["type"] == "array": - class_type = TaggedListNode - else: - raise RuntimeError(f"Unknown schema type: {schema['type']}") - # Use of allOf in the schema indicates that the class is a TaggedObjectNode - # which is "extending" another class. - elif "allOf" in schema: - class_type = TaggedObjectNode - else: - raise RuntimeError(f"Unknown schema type for: {tag['schema_uri']}") + class_name = class_name_from_tag_uri(pattern) + + class_type = _NODE_TYPE_BY_PATTERN.get(pattern, TaggedObjectNode) # In special cases one may need to add additional features to a tagged node class. # This is done by creating a mixin class with the name Mixin in _mixins.py @@ -179,11 +147,11 @@ def node_factory(tag): return type( class_name, class_type, - {"_tag": tag["tag_uri"], "__module__": "roman_datamodels.stnode", "__doc__": docstring_from_tag(tag)}, + {"_tag": pattern, "__module__": "roman_datamodels.stnode", "__doc__": docstring_from_tag(pattern)}, ) -def stnode_factory(tag): +def stnode_factory(pattern): """ Construct a tagged STNode class from a tag @@ -198,7 +166,7 @@ def stnode_factory(tag): """ # TaggedScalarNodes are a special case because they are not a subclass of a # _node class, but rather a subclass of the type of the scalar. - if "tagged_scalar" in tag["schema_uri"]: - return scalar_factory(tag) + if is_tagged_scalar_pattern(pattern): + return scalar_factory(pattern) else: - return node_factory(tag) + return node_factory(pattern) diff --git a/src/roman_datamodels/stnode/_stnode.py b/src/roman_datamodels/stnode/_stnode.py index f829a60c..6ff5f561 100644 --- a/src/roman_datamodels/stnode/_stnode.py +++ b/src/roman_datamodels/stnode/_stnode.py @@ -22,8 +22,9 @@ # Load the manifest directly from the rad resources and not from ASDF. # This is because the ASDF extensions have to be created before they can be registered # and this module creates the classes used by the ASDF extension. -DATAMODELS_MANIFEST_PATH = importlib.resources.files(resources) / "manifests" / "datamodels-1.0.yaml" -DATAMODELS_MANIFEST = yaml.safe_load(DATAMODELS_MANIFEST_PATH.read_bytes()) +_MANIFEST_DIR = importlib.resources.files(resources) / "manifests" +_MANIFEST_PATHS = [path for path in _MANIFEST_DIR.glob("*.yaml")] +_MANIFESTS = [yaml.safe_load(path.read_bytes()) for path in _MANIFEST_PATHS] def _factory(tag): @@ -40,8 +41,14 @@ def _factory(tag): # Main dynamic class creation loop # Reads each tag entry from the manifest and creates a class for it -for tag in DATAMODELS_MANIFEST["tags"]: - _factory(tag) +_generated = set() +for manifest in _MANIFESTS: + for tag in manifest["tags"]: + base, _ = tag["tag_uri"].rsplit("-", maxsplit=1) + pattern = f"{base}-*" + if pattern not in _generated: + _factory(pattern) + _generated.add(pattern) # List of node classes made available by this library. From d7fda1f5474565302678acca9406bfa1cf422141 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 3 Jan 2025 16:36:32 -0500 Subject: [PATCH 02/14] hack converters to use updated nodes --- src/roman_datamodels/stnode/_converters.py | 53 +++++++++++++++++----- src/roman_datamodels/stnode/_factories.py | 4 +- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/roman_datamodels/stnode/_converters.py b/src/roman_datamodels/stnode/_converters.py index 43e33582..1004f672 100644 --- a/src/roman_datamodels/stnode/_converters.py +++ b/src/roman_datamodels/stnode/_converters.py @@ -2,6 +2,7 @@ The ASDF Converters to handle the serialization/deseialization of the STNode classes to ASDF. """ +import asdf from asdf.extension import Converter, ManifestExtension from astropy.time import Time @@ -49,13 +50,22 @@ def types(self): return list(OBJECT_NODE_CLASSES_BY_TAG.values()) def select_tag(self, obj, tags, ctx): - return obj.tag + # TODO rename obj.tag + pattern = obj.tag + for tag in tags: + if asdf.util.uri_match(pattern, tag): + return tag + raise ValueError(f"No matching tag for {pattern}") def to_yaml_tree(self, obj, tag, ctx): return dict(obj._data) def from_yaml_tree(self, node, tag, ctx): - return OBJECT_NODE_CLASSES_BY_TAG[tag](node) + # TODO this is messy + for pattern, node_class in OBJECT_NODE_CLASSES_BY_TAG.items(): + if asdf.util.uri_match(pattern, tag): + return node_class(node) + raise ValueError(f"No matching class for {tag}") class TaggedListNodeConverter(_RomanConverter): @@ -72,12 +82,23 @@ def types(self): return list(LIST_NODE_CLASSES_BY_TAG.values()) def select_tag(self, obj, tags, ctx): - return obj.tag + # TODO rename obj.tag + pattern = obj.tag + for tag in tags: + if asdf.util.uri_match(pattern, tag): + return tag + raise ValueError(f"No matching tag for {pattern}") def to_yaml_tree(self, obj, tag, ctx): return list(obj) def from_yaml_tree(self, node, tag, ctx): + # TODO this is messy + for pattern, node_class in LIST_NODE_CLASSES_BY_TAG.items(): + if asdf.util.uri_match(pattern, tag): + return node_class(node) + raise ValueError(f"No matching class for {tag}") + return LIST_NODE_CLASSES_BY_TAG[tag](node) @@ -95,30 +116,40 @@ def types(self): return list(SCALAR_NODE_CLASSES_BY_TAG.values()) def select_tag(self, obj, tags, ctx): - return obj.tag + # TODO rename obj.tag + pattern = obj.tag + for tag in tags: + if asdf.util.uri_match(pattern, tag): + return tag + raise ValueError(f"No matching tag for {pattern}") def to_yaml_tree(self, obj, tag, ctx): - from ._stnode import FileDate, FpsFileDate, TvacFileDate - + # TODO is there a better way to do this? node = obj.__class__.__bases__[0](obj) - if tag in (FileDate._tag, FpsFileDate._tag, TvacFileDate._tag): + if "file_date" in tag: converter = ctx.extension_manager.get_converter_for_type(type(node)) node = converter.to_yaml_tree(node, tag, ctx) return node def from_yaml_tree(self, node, tag, ctx): - from ._stnode import FileDate, FpsFileDate, TvacFileDate - - if tag in (FileDate._tag, FpsFileDate._tag, TvacFileDate._tag): + # TODO is there a better way to do this? + if "file_date" in tag: converter = ctx.extension_manager.get_converter_for_type(Time) node = converter.from_yaml_tree(node, tag, ctx) - return SCALAR_NODE_CLASSES_BY_TAG[tag](node) + # TODO this is messy + for pattern, node_class in SCALAR_NODE_CLASSES_BY_TAG.items(): + if asdf.util.uri_match(pattern, tag): + return node_class(node) + raise ValueError(f"No matching class for {tag}") # Create the ASDF extension for the STNode classes. NODE_EXTENSIONS = [ + ManifestExtension.from_uri( + "asdf://stsci.edu/datamodels/roman/manifests/datamodels-2.0.0", converters=NODE_CONVERTERS.values() + ), ManifestExtension.from_uri("asdf://stsci.edu/datamodels/roman/manifests/datamodels-1.0", converters=NODE_CONVERTERS.values()), ] diff --git a/src/roman_datamodels/stnode/_factories.py b/src/roman_datamodels/stnode/_factories.py index 71ae77af..e4f817ca 100644 --- a/src/roman_datamodels/stnode/_factories.py +++ b/src/roman_datamodels/stnode/_factories.py @@ -16,8 +16,8 @@ # Map of scalar types by pattern (str is default) _SCALAR_TYPE_BY_PATTERN = { "asdf://stsci.edu/datamodels/roman/tags/file_date-*": Time, - "asdf://stsci.edu/datamodels/roman/tags/fps/file_date-": Time, - "asdf://stsci.edu/datamodels/roman/tags/tvac/file_date-": Time, + "asdf://stsci.edu/datamodels/roman/tags/fps/file_date-*": Time, + "asdf://stsci.edu/datamodels/roman/tags/tvac/file_date-*": Time, } # Map of node types by pattern (TaggedObjectNode is default) _NODE_TYPE_BY_PATTERN = { From efeff9f5e797f12429bbc590634a5396a66c296d Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 3 Jan 2025 16:45:32 -0500 Subject: [PATCH 03/14] untangling tag and pattern --- src/roman_datamodels/stnode/_converters.py | 27 +++++++----- src/roman_datamodels/stnode/_factories.py | 4 +- src/roman_datamodels/stnode/_node.py | 4 +- src/roman_datamodels/stnode/_registry.py | 6 +-- src/roman_datamodels/stnode/_stnode.py | 8 ++-- src/roman_datamodels/stnode/_tagged.py | 49 ++++++++++++---------- 6 files changed, 53 insertions(+), 45 deletions(-) diff --git a/src/roman_datamodels/stnode/_converters.py b/src/roman_datamodels/stnode/_converters.py index 1004f672..64dcd837 100644 --- a/src/roman_datamodels/stnode/_converters.py +++ b/src/roman_datamodels/stnode/_converters.py @@ -6,7 +6,12 @@ from asdf.extension import Converter, ManifestExtension from astropy.time import Time -from ._registry import LIST_NODE_CLASSES_BY_TAG, NODE_CONVERTERS, OBJECT_NODE_CLASSES_BY_TAG, SCALAR_NODE_CLASSES_BY_TAG +from ._registry import ( + LIST_NODE_CLASSES_BY_PATTERN, + NODE_CONVERTERS, + OBJECT_NODE_CLASSES_BY_PATTERN, + SCALAR_NODE_CLASSES_BY_PATTERN, +) __all__ = [ "NODE_EXTENSIONS", @@ -43,11 +48,11 @@ class TaggedObjectNodeConverter(_RomanConverter): @property def tags(self): - return list(OBJECT_NODE_CLASSES_BY_TAG.keys()) + return list(OBJECT_NODE_CLASSES_BY_PATTERN.keys()) @property def types(self): - return list(OBJECT_NODE_CLASSES_BY_TAG.values()) + return list(OBJECT_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): # TODO rename obj.tag @@ -62,7 +67,7 @@ def to_yaml_tree(self, obj, tag, ctx): def from_yaml_tree(self, node, tag, ctx): # TODO this is messy - for pattern, node_class in OBJECT_NODE_CLASSES_BY_TAG.items(): + for pattern, node_class in OBJECT_NODE_CLASSES_BY_PATTERN.items(): if asdf.util.uri_match(pattern, tag): return node_class(node) raise ValueError(f"No matching class for {tag}") @@ -75,11 +80,11 @@ class TaggedListNodeConverter(_RomanConverter): @property def tags(self): - return list(LIST_NODE_CLASSES_BY_TAG.keys()) + return list(LIST_NODE_CLASSES_BY_PATTERN.keys()) @property def types(self): - return list(LIST_NODE_CLASSES_BY_TAG.values()) + return list(LIST_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): # TODO rename obj.tag @@ -94,12 +99,12 @@ def to_yaml_tree(self, obj, tag, ctx): def from_yaml_tree(self, node, tag, ctx): # TODO this is messy - for pattern, node_class in LIST_NODE_CLASSES_BY_TAG.items(): + for pattern, node_class in LIST_NODE_CLASSES_BY_PATTERN.items(): if asdf.util.uri_match(pattern, tag): return node_class(node) raise ValueError(f"No matching class for {tag}") - return LIST_NODE_CLASSES_BY_TAG[tag](node) + return LIST_NODE_CLASSES_BY_PATTERN[tag](node) class TaggedScalarNodeConverter(_RomanConverter): @@ -109,11 +114,11 @@ class TaggedScalarNodeConverter(_RomanConverter): @property def tags(self): - return list(SCALAR_NODE_CLASSES_BY_TAG.keys()) + return list(SCALAR_NODE_CLASSES_BY_PATTERN.keys()) @property def types(self): - return list(SCALAR_NODE_CLASSES_BY_TAG.values()) + return list(SCALAR_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): # TODO rename obj.tag @@ -140,7 +145,7 @@ def from_yaml_tree(self, node, tag, ctx): node = converter.from_yaml_tree(node, tag, ctx) # TODO this is messy - for pattern, node_class in SCALAR_NODE_CLASSES_BY_TAG.items(): + for pattern, node_class in SCALAR_NODE_CLASSES_BY_PATTERN.items(): if asdf.util.uri_match(pattern, tag): return node_class(node) raise ValueError(f"No matching class for {tag}") diff --git a/src/roman_datamodels/stnode/_factories.py b/src/roman_datamodels/stnode/_factories.py index e4f817ca..d6b28fc8 100644 --- a/src/roman_datamodels/stnode/_factories.py +++ b/src/roman_datamodels/stnode/_factories.py @@ -115,7 +115,7 @@ def scalar_factory(pattern): return type( class_name, (type_, TaggedScalarNode), - {"_tag": pattern, "__module__": "roman_datamodels.stnode", "__doc__": docstring_from_tag(pattern)}, + {"_pattern": pattern, "__module__": "roman_datamodels.stnode", "__doc__": docstring_from_tag(pattern)}, ) @@ -147,7 +147,7 @@ def node_factory(pattern): return type( class_name, class_type, - {"_tag": pattern, "__module__": "roman_datamodels.stnode", "__doc__": docstring_from_tag(pattern)}, + {"_pattern": pattern, "__module__": "roman_datamodels.stnode", "__doc__": docstring_from_tag(pattern)}, ) diff --git a/src/roman_datamodels/stnode/_node.py b/src/roman_datamodels/stnode/_node.py index 669317d2..6b43ffd2 100644 --- a/src/roman_datamodels/stnode/_node.py +++ b/src/roman_datamodels/stnode/_node.py @@ -100,7 +100,7 @@ class DNode(MutableMapping): Base class describing all "object" (dict-like) data nodes for STNode classes. """ - _tag = None + _pattern = None _ctx = None def __init__(self, node=None, parent=None, name=None): @@ -311,7 +311,7 @@ class LNode(UserList): Base class describing all "array" (list-like) data nodes for STNode classes. """ - _tag = None + _pattern = None def __init__(self, node=None): if node is None: diff --git a/src/roman_datamodels/stnode/_registry.py b/src/roman_datamodels/stnode/_registry.py index 13562e2f..027f4fd7 100644 --- a/src/roman_datamodels/stnode/_registry.py +++ b/src/roman_datamodels/stnode/_registry.py @@ -4,8 +4,8 @@ whenever they generated. """ -OBJECT_NODE_CLASSES_BY_TAG = {} -LIST_NODE_CLASSES_BY_TAG = {} -SCALAR_NODE_CLASSES_BY_TAG = {} +OBJECT_NODE_CLASSES_BY_PATTERN = {} +LIST_NODE_CLASSES_BY_PATTERN = {} +SCALAR_NODE_CLASSES_BY_PATTERN = {} SCALAR_NODE_CLASSES_BY_KEY = {} NODE_CONVERTERS = {} diff --git a/src/roman_datamodels/stnode/_stnode.py b/src/roman_datamodels/stnode/_stnode.py index 6ff5f561..4a4039fd 100644 --- a/src/roman_datamodels/stnode/_stnode.py +++ b/src/roman_datamodels/stnode/_stnode.py @@ -12,7 +12,7 @@ from rad import resources from ._factories import stnode_factory -from ._registry import LIST_NODE_CLASSES_BY_TAG, OBJECT_NODE_CLASSES_BY_TAG, SCALAR_NODE_CLASSES_BY_TAG +from ._registry import LIST_NODE_CLASSES_BY_PATTERN, OBJECT_NODE_CLASSES_BY_PATTERN, SCALAR_NODE_CLASSES_BY_PATTERN __all__ = [ "NODE_CLASSES", @@ -54,7 +54,7 @@ def _factory(tag): # List of node classes made available by this library. # This is part of the public API. NODE_CLASSES = ( - list(OBJECT_NODE_CLASSES_BY_TAG.values()) - + list(LIST_NODE_CLASSES_BY_TAG.values()) - + list(SCALAR_NODE_CLASSES_BY_TAG.values()) + list(OBJECT_NODE_CLASSES_BY_PATTERN.values()) + + list(LIST_NODE_CLASSES_BY_PATTERN.values()) + + list(SCALAR_NODE_CLASSES_BY_PATTERN.values()) ) diff --git a/src/roman_datamodels/stnode/_tagged.py b/src/roman_datamodels/stnode/_tagged.py index 80fb0f4e..2babe7b0 100644 --- a/src/roman_datamodels/stnode/_tagged.py +++ b/src/roman_datamodels/stnode/_tagged.py @@ -10,10 +10,10 @@ from ._node import DNode, LNode from ._registry import ( - LIST_NODE_CLASSES_BY_TAG, - OBJECT_NODE_CLASSES_BY_TAG, + LIST_NODE_CLASSES_BY_PATTERN, + OBJECT_NODE_CLASSES_BY_PATTERN, SCALAR_NODE_CLASSES_BY_KEY, - SCALAR_NODE_CLASSES_BY_TAG, + SCALAR_NODE_CLASSES_BY_PATTERN, ) __all__ = [ @@ -65,18 +65,19 @@ class TaggedObjectNode(DNode): def __init_subclass__(cls, **kwargs) -> None: """ - Register any subclasses of this class in the OBJECT_NODE_CLASSES_BY_TAG + Register any subclasses of this class in the OBJECT_NODE_CLASSES_BY_PATTERN registry. """ super().__init_subclass__(**kwargs) if cls.__name__ != "TaggedObjectNode": - if cls._tag in OBJECT_NODE_CLASSES_BY_TAG: - raise RuntimeError(f"TaggedObjectNode class for tag '{cls._tag}' has been defined twice") - OBJECT_NODE_CLASSES_BY_TAG[cls._tag] = cls + if cls._pattern in OBJECT_NODE_CLASSES_BY_PATTERN: + raise RuntimeError(f"TaggedObjectNode class for tag '{cls._pattern}' has been defined twice") + OBJECT_NODE_CLASSES_BY_PATTERN[cls._pattern] = cls @property def tag(self): - return self._tag + # TODO resolve tag from pattern + return self._pattern def _schema(self): if self._x_schema is None: @@ -85,7 +86,7 @@ def _schema(self): def get_schema(self): """Retrieve the schema associated with this tag""" - return get_schema_from_tag(self.ctx, self._tag) + return get_schema_from_tag(self.ctx, self.tag) class TaggedListNode(LNode): @@ -97,18 +98,19 @@ class TaggedListNode(LNode): def __init_subclass__(cls, **kwargs) -> None: """ - Register any subclasses of this class in the LIST_NODE_CLASSES_BY_TAG + Register any subclasses of this class in the LIST_NODE_CLASSES_BY_PATTERN registry. """ super().__init_subclass__(**kwargs) if cls.__name__ != "TaggedListNode": - if cls._tag in LIST_NODE_CLASSES_BY_TAG: - raise RuntimeError(f"TaggedListNode class for tag '{cls._tag}' has been defined twice") - LIST_NODE_CLASSES_BY_TAG[cls._tag] = cls + if cls._pattern in LIST_NODE_CLASSES_BY_PATTERN: + raise RuntimeError(f"TaggedListNode class for tag '{cls._pattern}' has been defined twice") + LIST_NODE_CLASSES_BY_PATTERN[cls._pattern] = cls @property def tag(self): - return self._tag + # TODO resolve tag from pattern + return self._pattern class TaggedScalarNode: @@ -119,20 +121,20 @@ class TaggedScalarNode: These will all be in the tagged_scalars directory. """ - _tag = None + _pattern = None _ctx = None def __init_subclass__(cls, **kwargs) -> None: """ - Register any subclasses of this class in the SCALAR_NODE_CLASSES_BY_TAG + Register any subclasses of this class in the SCALAR_NODE_CLASSES_BY_PATTERN and SCALAR_NODE_CLASSES_BY_KEY registry. """ super().__init_subclass__(**kwargs) if cls.__name__ != "TaggedScalarNode": - if cls._tag in SCALAR_NODE_CLASSES_BY_TAG: - raise RuntimeError(f"TaggedScalarNode class for tag '{cls._tag}' has been defined twice") - SCALAR_NODE_CLASSES_BY_TAG[cls._tag] = cls - SCALAR_NODE_CLASSES_BY_KEY[name_from_tag_uri(cls._tag)] = cls + if cls._pattern in SCALAR_NODE_CLASSES_BY_PATTERN: + raise RuntimeError(f"TaggedScalarNode class for tag '{cls._pattern}' has been defined twice") + SCALAR_NODE_CLASSES_BY_PATTERN[cls._pattern] = cls + SCALAR_NODE_CLASSES_BY_KEY[name_from_tag_uri(cls._pattern)] = cls @property def ctx(self): @@ -145,14 +147,15 @@ def __asdf_traverse__(self): @property def tag(self): - return self._tag + # TODO resolve tag from pattern + return self._pattern @property def key(self): - return name_from_tag_uri(self._tag) + return name_from_tag_uri(self.tag) def get_schema(self): - return get_schema_from_tag(self.ctx, self._tag) + return get_schema_from_tag(self.ctx, self.tag) def copy(self): return copy.copy(self) From cd86d18f979abdf71e7bcf246b03348523f68577 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 3 Jan 2025 17:01:23 -0500 Subject: [PATCH 04/14] resolve tag from pattern using asdf --- src/roman_datamodels/stnode/_converters.py | 27 ++++++++++++++-------- src/roman_datamodels/stnode/_tagged.py | 19 ++++++++++----- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/roman_datamodels/stnode/_converters.py b/src/roman_datamodels/stnode/_converters.py index 64dcd837..926f7bf7 100644 --- a/src/roman_datamodels/stnode/_converters.py +++ b/src/roman_datamodels/stnode/_converters.py @@ -55,8 +55,9 @@ def types(self): return list(OBJECT_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): - # TODO rename obj.tag - pattern = obj.tag + if hasattr(obj, "_tag"): + return obj._tag + pattern = obj._pattern for tag in tags: if asdf.util.uri_match(pattern, tag): return tag @@ -69,7 +70,9 @@ def from_yaml_tree(self, node, tag, ctx): # TODO this is messy for pattern, node_class in OBJECT_NODE_CLASSES_BY_PATTERN.items(): if asdf.util.uri_match(pattern, tag): - return node_class(node) + obj = node_class(node) + obj._tag = tag + return obj raise ValueError(f"No matching class for {tag}") @@ -87,8 +90,9 @@ def types(self): return list(LIST_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): - # TODO rename obj.tag - pattern = obj.tag + if hasattr(obj, "_tag"): + return obj._tag + pattern = obj._pattern for tag in tags: if asdf.util.uri_match(pattern, tag): return tag @@ -101,7 +105,9 @@ def from_yaml_tree(self, node, tag, ctx): # TODO this is messy for pattern, node_class in LIST_NODE_CLASSES_BY_PATTERN.items(): if asdf.util.uri_match(pattern, tag): - return node_class(node) + obj = node_class(node) + obj._tag = tag + return obj raise ValueError(f"No matching class for {tag}") return LIST_NODE_CLASSES_BY_PATTERN[tag](node) @@ -121,8 +127,9 @@ def types(self): return list(SCALAR_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): - # TODO rename obj.tag - pattern = obj.tag + if hasattr(obj, "_tag"): + return obj._tag + pattern = obj._pattern for tag in tags: if asdf.util.uri_match(pattern, tag): return tag @@ -147,7 +154,9 @@ def from_yaml_tree(self, node, tag, ctx): # TODO this is messy for pattern, node_class in SCALAR_NODE_CLASSES_BY_PATTERN.items(): if asdf.util.uri_match(pattern, tag): - return node_class(node) + obj = node_class(node) + obj._tag = tag + return obj raise ValueError(f"No matching class for {tag}") diff --git a/src/roman_datamodels/stnode/_tagged.py b/src/roman_datamodels/stnode/_tagged.py index 2babe7b0..1ecbe9f8 100644 --- a/src/roman_datamodels/stnode/_tagged.py +++ b/src/roman_datamodels/stnode/_tagged.py @@ -56,6 +56,10 @@ def name_from_tag_uri(tag_uri): return tag_uri_split +def tag_for_type(obj, ctx): + return ctx.extension_manager.get_converter_for_type(type(obj)).select_tag(obj, ctx) + + class TaggedObjectNode(DNode): """ Base class for all tagged objects defined by RAD @@ -76,8 +80,9 @@ def __init_subclass__(cls, **kwargs) -> None: @property def tag(self): - # TODO resolve tag from pattern - return self._pattern + if not hasattr(self, "_tag"): + self._tag = tag_for_type(self, self.ctx) + return self._tag def _schema(self): if self._x_schema is None: @@ -109,8 +114,9 @@ def __init_subclass__(cls, **kwargs) -> None: @property def tag(self): - # TODO resolve tag from pattern - return self._pattern + if not hasattr(self, "_tag"): + self._tag = tag_for_type(self, self.ctx) + return self._tag class TaggedScalarNode: @@ -147,8 +153,9 @@ def __asdf_traverse__(self): @property def tag(self): - # TODO resolve tag from pattern - return self._pattern + if not hasattr(self, "_tag"): + self._tag = tag_for_type(self, self.ctx) + return self._tag @property def key(self): From 832b222e2e3c8aaa09eb5fc6cec5a196498e5b2b Mon Sep 17 00:00:00 2001 From: Brett Date: Sat, 4 Jan 2025 09:56:48 -0500 Subject: [PATCH 05/14] assign a _default_tag (newest) for each node class --- src/roman_datamodels/stnode/_converters.py | 30 +++-------- src/roman_datamodels/stnode/_factories.py | 58 ++++++++-------------- src/roman_datamodels/stnode/_stnode.py | 14 +++--- src/roman_datamodels/stnode/_tagged.py | 22 ++++---- tests/test_stnode.py | 1 + 5 files changed, 48 insertions(+), 77 deletions(-) diff --git a/src/roman_datamodels/stnode/_converters.py b/src/roman_datamodels/stnode/_converters.py index 926f7bf7..0d6990f5 100644 --- a/src/roman_datamodels/stnode/_converters.py +++ b/src/roman_datamodels/stnode/_converters.py @@ -55,13 +55,7 @@ def types(self): return list(OBJECT_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): - if hasattr(obj, "_tag"): - return obj._tag - pattern = obj._pattern - for tag in tags: - if asdf.util.uri_match(pattern, tag): - return tag - raise ValueError(f"No matching tag for {pattern}") + return obj._tag def to_yaml_tree(self, obj, tag, ctx): return dict(obj._data) @@ -71,7 +65,7 @@ def from_yaml_tree(self, node, tag, ctx): for pattern, node_class in OBJECT_NODE_CLASSES_BY_PATTERN.items(): if asdf.util.uri_match(pattern, tag): obj = node_class(node) - obj._tag = tag + obj._read_tag = tag return obj raise ValueError(f"No matching class for {tag}") @@ -90,13 +84,7 @@ def types(self): return list(LIST_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): - if hasattr(obj, "_tag"): - return obj._tag - pattern = obj._pattern - for tag in tags: - if asdf.util.uri_match(pattern, tag): - return tag - raise ValueError(f"No matching tag for {pattern}") + return obj._tag def to_yaml_tree(self, obj, tag, ctx): return list(obj) @@ -106,7 +94,7 @@ def from_yaml_tree(self, node, tag, ctx): for pattern, node_class in LIST_NODE_CLASSES_BY_PATTERN.items(): if asdf.util.uri_match(pattern, tag): obj = node_class(node) - obj._tag = tag + obj._read_tag = tag return obj raise ValueError(f"No matching class for {tag}") @@ -127,13 +115,7 @@ def types(self): return list(SCALAR_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): - if hasattr(obj, "_tag"): - return obj._tag - pattern = obj._pattern - for tag in tags: - if asdf.util.uri_match(pattern, tag): - return tag - raise ValueError(f"No matching tag for {pattern}") + return obj._tag def to_yaml_tree(self, obj, tag, ctx): # TODO is there a better way to do this? @@ -155,7 +137,7 @@ def from_yaml_tree(self, node, tag, ctx): for pattern, node_class in SCALAR_NODE_CLASSES_BY_PATTERN.items(): if asdf.util.uri_match(pattern, tag): obj = node_class(node) - obj._tag = tag + obj._read_tag = tag return obj raise ValueError(f"No matching class for {tag}") diff --git a/src/roman_datamodels/stnode/_factories.py b/src/roman_datamodels/stnode/_factories.py index d6b28fc8..b3025716 100644 --- a/src/roman_datamodels/stnode/_factories.py +++ b/src/roman_datamodels/stnode/_factories.py @@ -3,10 +3,7 @@ These are used to dynamically create classes from the RAD manifest. """ -import importlib.resources - from astropy.time import Time -from rad import resources from . import _mixins from ._tagged import TaggedListNode, TaggedObjectNode, TaggedScalarNode, name_from_tag_uri @@ -24,27 +21,6 @@ "asdf://stsci.edu/datamodels/roman/tags/cal_logs-*": TaggedListNode, } -_SCALAR_TAG_BASES = { - "calibration_software_name", - "calibration_software_version", - "product_type", - "filename", - "file_date", - "model_type", - "origin", - "prd_version", - "sdf_software_version", - "telescope", - "prd_software_version", # for tvac and fps -} - -BASE_SCHEMA_PATH = importlib.resources.files(resources) / "schemas" - - -def is_tagged_scalar_pattern(pattern): - tag_base = pattern.rsplit("-", maxsplit=1)[0].rsplit("/", maxsplit=1)[1] - return tag_base in _SCALAR_TAG_BASES - def class_name_from_tag_uri(tag_uri): """ @@ -67,7 +43,7 @@ def class_name_from_tag_uri(tag_uri): return class_name -def docstring_from_tag(pattern): +def docstring_from_tag(tag_def): """ Read the docstring (if it exists) from the RAD manifest and generate a docstring for the dynamically generated class. @@ -81,14 +57,12 @@ def docstring_from_tag(pattern): ------- A docstring for the class based on the tag """ - # TODO broken for now - # docstring = f"{tag['description']}\n\n" if "description" in tag else "" + docstring = f"{tag_def['description']}\n\n" if "description" in tag_def else "" - # return docstring + f"Class generated from tag '{tag['tag_uri']}'" - return f"Class generated from tag '{pattern}'" + return docstring + f"Class generated from tag '{tag_def['tag_uri']}'" -def scalar_factory(pattern): +def scalar_factory(pattern, tag_def): """ Factory to create a TaggedScalarNode class from a tag @@ -115,11 +89,16 @@ def scalar_factory(pattern): return type( class_name, (type_, TaggedScalarNode), - {"_pattern": pattern, "__module__": "roman_datamodels.stnode", "__doc__": docstring_from_tag(pattern)}, + { + "_pattern": pattern, + "_default_tag": tag_def["tag_uri"], + "__module__": "roman_datamodels.stnode", + "__doc__": docstring_from_tag(tag_def), + }, ) -def node_factory(pattern): +def node_factory(pattern, tag_def): """ Factory to create a TaggedObjectNode or TaggedListNode class from a tag @@ -147,11 +126,16 @@ def node_factory(pattern): return type( class_name, class_type, - {"_pattern": pattern, "__module__": "roman_datamodels.stnode", "__doc__": docstring_from_tag(pattern)}, + { + "_pattern": pattern, + "_default_tag": tag_def["tag_uri"], + "__module__": "roman_datamodels.stnode", + "__doc__": docstring_from_tag(tag_def), + }, ) -def stnode_factory(pattern): +def stnode_factory(pattern, tag_def): """ Construct a tagged STNode class from a tag @@ -166,7 +150,7 @@ def stnode_factory(pattern): """ # TaggedScalarNodes are a special case because they are not a subclass of a # _node class, but rather a subclass of the type of the scalar. - if is_tagged_scalar_pattern(pattern): - return scalar_factory(pattern) + if "tagged_scalar" in tag_def["schema_uri"]: + return scalar_factory(pattern, tag_def) else: - return node_factory(pattern) + return node_factory(pattern, tag_def) diff --git a/src/roman_datamodels/stnode/_stnode.py b/src/roman_datamodels/stnode/_stnode.py index 4a4039fd..7b12cd2d 100644 --- a/src/roman_datamodels/stnode/_stnode.py +++ b/src/roman_datamodels/stnode/_stnode.py @@ -23,16 +23,17 @@ # This is because the ASDF extensions have to be created before they can be registered # and this module creates the classes used by the ASDF extension. _MANIFEST_DIR = importlib.resources.files(resources) / "manifests" -_MANIFEST_PATHS = [path for path in _MANIFEST_DIR.glob("*.yaml")] +# sort manifests by version (newest first) +_MANIFEST_PATHS = sorted([path for path in _MANIFEST_DIR.glob("*.yaml")], reverse=True) _MANIFESTS = [yaml.safe_load(path.read_bytes()) for path in _MANIFEST_PATHS] -def _factory(tag): +def _factory(pattern, tag_def): """ Wrap the __all__ append and class creation in a function to avoid the linter getting upset """ - cls = stnode_factory(tag) + cls = stnode_factory(pattern, tag_def) class_name = cls.__name__ globals()[class_name] = cls # Add to namespace of module @@ -43,11 +44,12 @@ def _factory(tag): # Reads each tag entry from the manifest and creates a class for it _generated = set() for manifest in _MANIFESTS: - for tag in manifest["tags"]: - base, _ = tag["tag_uri"].rsplit("-", maxsplit=1) + for tag_def in manifest["tags"]: + # make pattern from tag + base, _ = tag_def["tag_uri"].rsplit("-", maxsplit=1) pattern = f"{base}-*" if pattern not in _generated: - _factory(pattern) + _factory(pattern, tag_def) _generated.add(pattern) diff --git a/src/roman_datamodels/stnode/_tagged.py b/src/roman_datamodels/stnode/_tagged.py index 1ecbe9f8..4625ea02 100644 --- a/src/roman_datamodels/stnode/_tagged.py +++ b/src/roman_datamodels/stnode/_tagged.py @@ -56,10 +56,6 @@ def name_from_tag_uri(tag_uri): return tag_uri_split -def tag_for_type(obj, ctx): - return ctx.extension_manager.get_converter_for_type(type(obj)).select_tag(obj, ctx) - - class TaggedObjectNode(DNode): """ Base class for all tagged objects defined by RAD @@ -78,10 +74,12 @@ def __init_subclass__(cls, **kwargs) -> None: raise RuntimeError(f"TaggedObjectNode class for tag '{cls._pattern}' has been defined twice") OBJECT_NODE_CLASSES_BY_PATTERN[cls._pattern] = cls + @property + def _tag(self): + return getattr(self, "_read_tag", self._default_tag) + @property def tag(self): - if not hasattr(self, "_tag"): - self._tag = tag_for_type(self, self.ctx) return self._tag def _schema(self): @@ -112,10 +110,12 @@ def __init_subclass__(cls, **kwargs) -> None: raise RuntimeError(f"TaggedListNode class for tag '{cls._pattern}' has been defined twice") LIST_NODE_CLASSES_BY_PATTERN[cls._pattern] = cls + @property + def _tag(self): + return getattr(self, "_read_tag", self._default_tag) + @property def tag(self): - if not hasattr(self, "_tag"): - self._tag = tag_for_type(self, self.ctx) return self._tag @@ -151,10 +151,12 @@ def ctx(self): def __asdf_traverse__(self): return self + @property + def _tag(self): + return getattr(self, "_read_tag", self._default_tag) + @property def tag(self): - if not hasattr(self, "_tag"): - self._tag = tag_for_type(self, self.ctx) return self._tag @property diff --git a/tests/test_stnode.py b/tests/test_stnode.py index a9364774..592628dd 100644 --- a/tests/test_stnode.py +++ b/tests/test_stnode.py @@ -12,6 +12,7 @@ from .conftest import MANIFEST +@pytest.mark.skip(reason="_tag is no longer a class attribute") @pytest.mark.parametrize("tag", MANIFEST["tags"]) def test_generated_node_classes(tag): class_name = stnode._factories.class_name_from_tag_uri(tag["tag_uri"]) From 51da37c2bdd26e64906b925c89f056802930c7b3 Mon Sep 17 00:00:00 2001 From: Brett Date: Sat, 4 Jan 2025 09:57:42 -0500 Subject: [PATCH 06/14] set rad dep to branch --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 070d823b..c83dfbe5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "numpy >=1.24", "astropy >=5.3.0", # "rad >=0.22.0, <0.23.0", - "rad @ git+https://github.com/spacetelescope/rad.git", + "rad @ git+https://github.com/braingram/rad.git@versioned", "asdf-standard >=1.1.0", ] dynamic = ["version"] From c2b301e982cd8d8b146e0e878da7227d1cba9815 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Jan 2025 09:54:08 -0500 Subject: [PATCH 07/14] add changelog entry --- changes/445.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/445.feature.rst diff --git a/changes/445.feature.rst b/changes/445.feature.rst new file mode 100644 index 00000000..05536234 --- /dev/null +++ b/changes/445.feature.rst @@ -0,0 +1 @@ +Start versioning files by allows Node instances to use multiple versions of tags. From 573daa3b8db7e1c0e5c81cd4b401e7fbe6658fd6 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Jan 2025 10:38:54 -0500 Subject: [PATCH 08/14] index node classes by tag for converters --- src/roman_datamodels/stnode/_converters.py | 31 +++------------------- src/roman_datamodels/stnode/_registry.py | 1 + src/roman_datamodels/stnode/_stnode.py | 14 +++++++--- 3 files changed, 15 insertions(+), 31 deletions(-) diff --git a/src/roman_datamodels/stnode/_converters.py b/src/roman_datamodels/stnode/_converters.py index 0d6990f5..bba45f88 100644 --- a/src/roman_datamodels/stnode/_converters.py +++ b/src/roman_datamodels/stnode/_converters.py @@ -2,12 +2,12 @@ The ASDF Converters to handle the serialization/deseialization of the STNode classes to ASDF. """ -import asdf from asdf.extension import Converter, ManifestExtension from astropy.time import Time from ._registry import ( LIST_NODE_CLASSES_BY_PATTERN, + NODE_CLASSES_BY_TAG, NODE_CONVERTERS, OBJECT_NODE_CLASSES_BY_PATTERN, SCALAR_NODE_CLASSES_BY_PATTERN, @@ -61,13 +61,7 @@ def to_yaml_tree(self, obj, tag, ctx): return dict(obj._data) def from_yaml_tree(self, node, tag, ctx): - # TODO this is messy - for pattern, node_class in OBJECT_NODE_CLASSES_BY_PATTERN.items(): - if asdf.util.uri_match(pattern, tag): - obj = node_class(node) - obj._read_tag = tag - return obj - raise ValueError(f"No matching class for {tag}") + return NODE_CLASSES_BY_TAG[tag](node) class TaggedListNodeConverter(_RomanConverter): @@ -90,15 +84,7 @@ def to_yaml_tree(self, obj, tag, ctx): return list(obj) def from_yaml_tree(self, node, tag, ctx): - # TODO this is messy - for pattern, node_class in LIST_NODE_CLASSES_BY_PATTERN.items(): - if asdf.util.uri_match(pattern, tag): - obj = node_class(node) - obj._read_tag = tag - return obj - raise ValueError(f"No matching class for {tag}") - - return LIST_NODE_CLASSES_BY_PATTERN[tag](node) + return NODE_CLASSES_BY_TAG[tag](node) class TaggedScalarNodeConverter(_RomanConverter): @@ -118,7 +104,6 @@ def select_tag(self, obj, tags, ctx): return obj._tag def to_yaml_tree(self, obj, tag, ctx): - # TODO is there a better way to do this? node = obj.__class__.__bases__[0](obj) if "file_date" in tag: @@ -128,18 +113,10 @@ def to_yaml_tree(self, obj, tag, ctx): return node def from_yaml_tree(self, node, tag, ctx): - # TODO is there a better way to do this? if "file_date" in tag: converter = ctx.extension_manager.get_converter_for_type(Time) node = converter.from_yaml_tree(node, tag, ctx) - - # TODO this is messy - for pattern, node_class in SCALAR_NODE_CLASSES_BY_PATTERN.items(): - if asdf.util.uri_match(pattern, tag): - obj = node_class(node) - obj._read_tag = tag - return obj - raise ValueError(f"No matching class for {tag}") + return NODE_CLASSES_BY_TAG[tag](node) # Create the ASDF extension for the STNode classes. diff --git a/src/roman_datamodels/stnode/_registry.py b/src/roman_datamodels/stnode/_registry.py index 027f4fd7..92105fca 100644 --- a/src/roman_datamodels/stnode/_registry.py +++ b/src/roman_datamodels/stnode/_registry.py @@ -9,3 +9,4 @@ SCALAR_NODE_CLASSES_BY_PATTERN = {} SCALAR_NODE_CLASSES_BY_KEY = {} NODE_CONVERTERS = {} +NODE_CLASSES_BY_TAG = {} diff --git a/src/roman_datamodels/stnode/_stnode.py b/src/roman_datamodels/stnode/_stnode.py index 7b12cd2d..28836df1 100644 --- a/src/roman_datamodels/stnode/_stnode.py +++ b/src/roman_datamodels/stnode/_stnode.py @@ -12,7 +12,12 @@ from rad import resources from ._factories import stnode_factory -from ._registry import LIST_NODE_CLASSES_BY_PATTERN, OBJECT_NODE_CLASSES_BY_PATTERN, SCALAR_NODE_CLASSES_BY_PATTERN +from ._registry import ( + LIST_NODE_CLASSES_BY_PATTERN, + NODE_CLASSES_BY_TAG, + OBJECT_NODE_CLASSES_BY_PATTERN, + SCALAR_NODE_CLASSES_BY_PATTERN, +) __all__ = [ "NODE_CLASSES", @@ -38,19 +43,20 @@ def _factory(pattern, tag_def): class_name = cls.__name__ globals()[class_name] = cls # Add to namespace of module __all__.append(class_name) # add to __all__ so it's imported with `from . import *` + return cls # Main dynamic class creation loop # Reads each tag entry from the manifest and creates a class for it -_generated = set() +_generated = {} for manifest in _MANIFESTS: for tag_def in manifest["tags"]: # make pattern from tag base, _ = tag_def["tag_uri"].rsplit("-", maxsplit=1) pattern = f"{base}-*" if pattern not in _generated: - _factory(pattern, tag_def) - _generated.add(pattern) + _generated[pattern] = _factory(pattern, tag_def) + NODE_CLASSES_BY_TAG[tag_def["tag_uri"]] = _generated[pattern] # List of node classes made available by this library. From b8cdf5d8b8a058960f7d5e6381afa04536d39e2f Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Jan 2025 10:43:25 -0500 Subject: [PATCH 09/14] add note about _tag requirement --- src/roman_datamodels/stnode/_tagged.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/roman_datamodels/stnode/_tagged.py b/src/roman_datamodels/stnode/_tagged.py index 4625ea02..a17397f8 100644 --- a/src/roman_datamodels/stnode/_tagged.py +++ b/src/roman_datamodels/stnode/_tagged.py @@ -76,6 +76,7 @@ def __init_subclass__(cls, **kwargs) -> None: @property def _tag(self): + # _tag is required by asdf to allow __asdf_traverse__ return getattr(self, "_read_tag", self._default_tag) @property @@ -112,6 +113,7 @@ def __init_subclass__(cls, **kwargs) -> None: @property def _tag(self): + # _tag is required by asdf to allow __asdf_traverse__ return getattr(self, "_read_tag", self._default_tag) @property @@ -153,6 +155,7 @@ def __asdf_traverse__(self): @property def _tag(self): + # _tag is required by asdf to allow __asdf_traverse__ return getattr(self, "_read_tag", self._default_tag) @property From 29155c10004a5b50cd30517ae89aef57a36463e7 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Jan 2025 10:44:43 -0500 Subject: [PATCH 10/14] use tag not _tag in converters --- src/roman_datamodels/stnode/_converters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/roman_datamodels/stnode/_converters.py b/src/roman_datamodels/stnode/_converters.py index bba45f88..75090d50 100644 --- a/src/roman_datamodels/stnode/_converters.py +++ b/src/roman_datamodels/stnode/_converters.py @@ -55,7 +55,7 @@ def types(self): return list(OBJECT_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): - return obj._tag + return obj.tag def to_yaml_tree(self, obj, tag, ctx): return dict(obj._data) @@ -78,7 +78,7 @@ def types(self): return list(LIST_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): - return obj._tag + return obj.tag def to_yaml_tree(self, obj, tag, ctx): return list(obj) @@ -101,7 +101,7 @@ def types(self): return list(SCALAR_NODE_CLASSES_BY_PATTERN.values()) def select_tag(self, obj, tags, ctx): - return obj._tag + return obj.tag def to_yaml_tree(self, obj, tag, ctx): node = obj.__class__.__bases__[0](obj) From 2699c8dc3a56dc05938f867dcae1941eb037d733 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Jan 2025 11:17:26 -0500 Subject: [PATCH 11/14] update docstrings --- src/roman_datamodels/stnode/_factories.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/roman_datamodels/stnode/_factories.py b/src/roman_datamodels/stnode/_factories.py index b3025716..5c54f915 100644 --- a/src/roman_datamodels/stnode/_factories.py +++ b/src/roman_datamodels/stnode/_factories.py @@ -50,7 +50,7 @@ def docstring_from_tag(tag_def): Parameters ---------- - tag: dict + tag_def: dict A tag entry from the RAD manifest Returns @@ -68,7 +68,10 @@ def scalar_factory(pattern, tag_def): Parameters ---------- - tag: dict + pattern: str + A tag pattern/wildcard + + tag_def: dict A tag entry from the RAD manifest Returns @@ -104,7 +107,10 @@ def node_factory(pattern, tag_def): Parameters ---------- - tag: dict + pattern: str + A tag pattern/wildcard + + tag_def: dict A tag entry from the RAD manifest Returns @@ -141,7 +147,10 @@ def stnode_factory(pattern, tag_def): Parameters ---------- - tag: dict + pattern: str + A tag pattern/wildcard + + tag_def: dict A tag entry from the RAD manifest Returns From b4dba558ef1480b5e410c20b78d78ff7267f92ec Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Jan 2025 11:20:40 -0500 Subject: [PATCH 12/14] auto-detect manifests --- src/roman_datamodels/stnode/_converters.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/roman_datamodels/stnode/_converters.py b/src/roman_datamodels/stnode/_converters.py index 75090d50..7144f0e5 100644 --- a/src/roman_datamodels/stnode/_converters.py +++ b/src/roman_datamodels/stnode/_converters.py @@ -12,6 +12,7 @@ OBJECT_NODE_CLASSES_BY_PATTERN, SCALAR_NODE_CLASSES_BY_PATTERN, ) +from ._stnode import _MANIFESTS __all__ = [ "NODE_EXTENSIONS", @@ -120,9 +121,4 @@ def from_yaml_tree(self, node, tag, ctx): # Create the ASDF extension for the STNode classes. -NODE_EXTENSIONS = [ - ManifestExtension.from_uri( - "asdf://stsci.edu/datamodels/roman/manifests/datamodels-2.0.0", converters=NODE_CONVERTERS.values() - ), - ManifestExtension.from_uri("asdf://stsci.edu/datamodels/roman/manifests/datamodels-1.0", converters=NODE_CONVERTERS.values()), -] +NODE_EXTENSIONS = [ManifestExtension.from_uri(manifest["id"], converters=NODE_CONVERTERS.values()) for manifest in _MANIFESTS] From 99af0aa3362b76aa038aacb2ab8a2f54079c1be0 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Jan 2025 12:18:36 -0500 Subject: [PATCH 13/14] unskip test, use all manifests in tests --- tests/conftest.py | 10 ++++------ tests/test_models.py | 13 +++++++------ tests/test_stnode.py | 24 ++++++++++++++++-------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 430882ab..eeeb4344 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,15 +1,13 @@ import os -import asdf import pytest -import yaml -MANIFEST = yaml.safe_load(asdf.get_config().resource_manager["asdf://stsci.edu/datamodels/roman/manifests/datamodels-1.0"]) +from roman_datamodels.stnode._stnode import _MANIFESTS as MANIFESTS -@pytest.fixture(scope="session") -def manifest(): - return MANIFEST +@pytest.fixture(scope="session", params=MANIFESTS) +def manifest(request): + return request.param @pytest.fixture(scope="function") diff --git a/tests/test_models.py b/tests/test_models.py index 14a5e601..b2b7e7d4 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -14,7 +14,7 @@ from roman_datamodels import maker_utils as utils from roman_datamodels.testing import assert_node_equal -from .conftest import MANIFEST +from .conftest import MANIFESTS EXPECTED_COMMON_REFERENCE = {"$ref": "ref_common-1.0.0"} @@ -34,12 +34,13 @@ def datamodel_names(): names = [] extension_manager = asdf.AsdfFile().extension_manager - for tag in MANIFEST["tags"]: - schema_uri = extension_manager.get_tag_definition(tag["tag_uri"]).schema_uris[0] - schema = asdf.schema.load_schema(schema_uri, resolve_references=True) + for manifest in MANIFESTS: + for tag in manifest["tags"]: + schema_uri = extension_manager.get_tag_definition(tag["tag_uri"]).schema_uris[0] + schema = asdf.schema.load_schema(schema_uri, resolve_references=True) - if "datamodel_name" in schema: - names.append(schema["datamodel_name"]) + if "datamodel_name" in schema: + names.append(schema["datamodel_name"]) return names diff --git a/tests/test_stnode.py b/tests/test_stnode.py index 592628dd..9b381379 100644 --- a/tests/test_stnode.py +++ b/tests/test_stnode.py @@ -9,19 +9,27 @@ from roman_datamodels.maker_utils._base import NOFN, NONUM, NOSTR from roman_datamodels.testing import assert_node_equal, assert_node_is_copy, wraps_hashable -from .conftest import MANIFEST +from .conftest import MANIFESTS -@pytest.mark.skip(reason="_tag is no longer a class attribute") -@pytest.mark.parametrize("tag", MANIFEST["tags"]) -def test_generated_node_classes(tag): - class_name = stnode._factories.class_name_from_tag_uri(tag["tag_uri"]) +@pytest.mark.parametrize("tag_def", [tag_def for manifest in MANIFESTS for tag_def in manifest["tags"]]) +def test_tag_has_node_class(tag_def): + class_name = stnode._factories.class_name_from_tag_uri(tag_def["tag_uri"]) node_class = getattr(stnode, class_name) + assert asdf.util.uri_match(node_class._pattern, tag_def["tag_uri"]) + if node_class._default_tag == tag_def["tag_uri"]: + assert tag_def["description"] in node_class.__doc__ + assert tag_def["tag_uri"] in node_class.__doc__ + else: + default_tag_version = node_class._default_tag.rsplit("-", maxsplit=1)[1] + tag_def_version = tag_def["tag_uri"].rsplit("-", maxsplit=1)[1] + assert asdf.versioning.Version(default_tag_version) > asdf.versioning.Version(tag_def_version) + + +@pytest.mark.parametrize("node_class", stnode.NODE_CLASSES) +def test_node_classes_available_via_stnode(node_class): assert issubclass(node_class, stnode.TaggedObjectNode | stnode.TaggedListNode | stnode.TaggedScalarNode) - assert node_class._tag == tag["tag_uri"] - assert tag["description"] in node_class.__doc__ - assert tag["tag_uri"] in node_class.__doc__ assert node_class.__module__ == stnode.__name__ assert hasattr(stnode, node_class.__name__) From cc60a2a204f64605be9650dd4646a32ccfd5898d Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 6 Jan 2025 13:40:36 -0500 Subject: [PATCH 14/14] TST: add new cal_step entry --- pyproject.toml | 2 +- src/roman_datamodels/maker_utils/_common_meta.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c83dfbe5..908d9f1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "numpy >=1.24", "astropy >=5.3.0", # "rad >=0.22.0, <0.23.0", - "rad @ git+https://github.com/braingram/rad.git@versioned", + "rad @ git+https://github.com/braingram/rad.git@versioned_demo", "asdf-standard >=1.1.0", ] dynamic = ["version"] diff --git a/src/roman_datamodels/maker_utils/_common_meta.py b/src/roman_datamodels/maker_utils/_common_meta.py index 62e80155..ec279a62 100644 --- a/src/roman_datamodels/maker_utils/_common_meta.py +++ b/src/roman_datamodels/maker_utils/_common_meta.py @@ -307,6 +307,7 @@ def mk_l2_cal_step(**kwargs): l2calstep["saturation"] = kwargs.get("saturation", "INCOMPLETE") l2calstep["skymatch"] = kwargs.get("skymatch", "INCOMPLETE") l2calstep["tweakreg"] = kwargs.get("tweakreg", "INCOMPLETE") + l2calstep["two_step"] = kwargs.get("two_step", "INCOMPLETE") return l2calstep