From 00af57e3609908236fd62aff3340de15a4393cff Mon Sep 17 00:00:00 2001 From: Gunnar Andersson Date: Tue, 12 Nov 2024 21:34:37 +0100 Subject: [PATCH 1/5] create-new-release workflow: Minor cosmetic fix to release comments. Signed-off-by: Gunnar Andersson --- .github/workflows/create-new-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-new-release.yml b/.github/workflows/create-new-release.yml index 6f50c54..1f3f00b 100644 --- a/.github/workflows/create-new-release.yml +++ b/.github/workflows/create-new-release.yml @@ -63,7 +63,7 @@ jobs: tag_name: "${{ steps.vars.outputs.TAG }}" body: | The JSON Schema file for the IFEX Core IDL is a resulting artifact, generated from the internal model representation. - This one was generated from commit hash ${{ steps.vars.outputs.HASH }} and was assigned the release-name: ${{ steps.vars.outputs.TAG }}." + This one was generated from commit hash ${{ steps.vars.outputs.HASH }} and was assigned the release-name: ${{ steps.vars.outputs.TAG }}. env: GITHUB_TOKEN: "${{ github.token }}" From 924df208c5f2412fcd174d53a935cba46039ebd4 Mon Sep 17 00:00:00 2001 From: Gunnar Andersson Date: Tue, 3 Dec 2024 16:59:20 +0100 Subject: [PATCH 2/5] ifex_ast: Prepare support for variant type Signed-off-by: Gunnar Andersson --- ifex/model/ifex_ast.py | 6 +- ifex/model/ifex_ast_introspect.py | 114 +++++++++++++++++++++++++----- 2 files changed, 103 insertions(+), 17 deletions(-) diff --git a/ifex/model/ifex_ast.py b/ifex/model/ifex_ast.py index 4835caa..902278f 100644 --- a/ifex/model/ifex_ast.py +++ b/ifex/model/ifex_ast.py @@ -436,9 +436,12 @@ class Typedef: name: str """ Specifies the name of the typedef. """ - datatype: str + datatype: Optional[str] = str() """ Specifies datatype name of the typedef. """ + datatypes: Optional[List[str]] = field(default_factory=EmptyList) + """ If specified, then the type is a variant type. At least two datatypes should be listed. The single datatype: field must *not* be used at the same time. """ + description: Optional[str] = str() """ Specifies the description of the typedef. """ @@ -685,5 +688,6 @@ class FundamentalTypes: # name, description, min value, max value ["set", "A set (unique values), each of the same type. Format: set", "N/A", "N/A"], ["map", "A key-value mapping type. Format: map", "N/A", "N/A"], + ["variant", "A variant (union) type that can carry any of a predefined set of types, akin to Union in C. Format: variant", "N/A", "N/A"], ["opaque", "Indicates a complex type which is not explicitly defined in this context.", "N/A","N/A"] ] diff --git a/ifex/model/ifex_ast_introspect.py b/ifex/model/ifex_ast_introspect.py index 7f53c12..566a804 100644 --- a/ifex/model/ifex_ast_introspect.py +++ b/ifex/model/ifex_ast_introspect.py @@ -9,17 +9,18 @@ """ Provide helper functions to inspect the IFEX Core IDL language definition, -as it is defined by the class tree/hierarchy (not an inheritance hierarchy) -in the `ifex_ast` python file. These function can be used by any other code -that needs to process this underlying meta-model. It helps to ensure that the -fundamental language is defined in a single file. """ +as it is defined by the AST structure tree/hierarchy (not an inheritance hierarchy) +in the `ifex_ast` python file. These function can be used by implementations +that process the IFEX AST, or any other model designed in the same way. +""" -from ifex.model import ifex_ast +import re +import ifex_ast from dataclasses import is_dataclass, fields from typing import get_args, get_origin, List, Optional, Union, Any, ForwardRef import typing -# As we traverse the "tree" of dataclass definitions, it can be quite difficult +# As we traverse the tree of dataclass definitions, it can be quite difficult # to keep track of which type each variable has. Here is an explanation of how # we try to keep track: # @@ -27,22 +28,25 @@ # take an object, which is an instance of typing. which is # essentially an object that indicates the "type hint" using concepts from # the 'typing' python module. Examples are: Optional, List, Union, Any, etc. -# We here call variables that reference such a typing.Something object a -# type_indicator. It corresponds to the type hint information on the right -# side of the colon : in an expression like this: +# In these functions we call variables that reference such a typing.Something +# object, a type_indicator. The type_indicator corresponds to the type hint +# information on the right side of the colon : in an expression like this: # # namespaces: Optional[List[Namespace]] # # The type_indicator is the: `Optional[List[Namespace]]` # (or if fully qualified: `typing.Optional[typing.List[ifex_ast.Namespace]]`) -# Note that instead of being a dataclass like ifex_ast.Namespace, the inner -# type can of course be a built-in simple type like str. e.g. typing.List[str] +# The inner type can be an instance of an AST node, in other words an instance +# of a @dataclass like ifex_ast.Namespace, or it can be a built-in simple type +# like str. e.g. typing.List[str] # -# Next, in the 'dataclasses' python module we find the function fields(). -# It returns a list that represents the fields (members) of the dataclass. +# Next, in the 'dataclasses' python module we find the function: fields(). +# fields() returns a list of the fields (i.e member variables) of the dataclass. # Each field is represented by an object (an instance of the dataclasses.Field -# class). We name variables that refer to such Field() instances as `field`. -# A field thus represents a member variable in the python (data)class. +# class). In the following function we name variables that refer to such +# Field() instances as `field`. A variable named field thus represents a +# member variable in the python (data)class. +# # A field object contains several informations such as the name of the member # variable (field.name), and the `.type` member, which gives us the # type_indicator as described above. @@ -55,7 +59,7 @@ # NOTE: Here in the descriptions we might refer to an object's "type" when we # strictly mean its Type Indicator. Since typing in python is dynamic, # the actual type of an object could be different (and can be somewhat fungible -# too in theory, but generally not in this code). +# too in theory, but usually not in this code). def is_dataclass_type(cls): """Check if a class is a dataclass.""" @@ -142,6 +146,73 @@ def field_referenced_type(f): else: return field_actual_type(f) + +# --- End of generic functions -- + +# ------------------------------------------------------------------------------------------------ +# Above thes line we had generic functions that give information about a AST-model built +# from a number of @dataclasses and field typing information. +# +# We can notice that those only refers to *python* concepts such as @dataclasses and the typing module. +# This means, those functions could also be used for other similarly described AST models, beyond +# the IFEX Core IDL model. +# +# Below this line follows functions that are specific to IFEX. For example is_ifex_variant_typedef evaluates +# an actual IFEX-specific concern about the IFEX variant type. It is not a variant type of the python +# typing concept (which is called typing.Union anyhow) - these functions are about IFEX-specific concerns. + +# Check if string is "variant" where something can be empty +p_shortform_variant0 = r'variant<\s*([^,]*\s*(?:,\s*[^,]*)*)\s*>' + +# ...and this for variant +p_shortform_variant1 = r'variant<\s*([^,]+(?:\s*,\s*[^,]+)*)\s*>' + +# ...and this is the one we actually need. A valid variant has at least 2 +# types listed or it would not make sense. This last pattern guarantees this: +p_shortform_variant2 = r'variant<\s*[^,]+(?:\s*,\s*[^,]+)+\s*>' + +def is_ifex_variant_shortform(s): + """ Answer if a Typedef object has datatype defined to using the short form: variant """ + + # Convert "truthy" result object to actual bool for nicer debugging + return bool(s and s != '' and re.match(p_shortform_variant2, s)) + +def is_ifex_variant_typedef(f): + """ Answer if a Typedef object uses a variant type. + A variant type can be defined in either one of these two ways: + 1. The field "datatypes" has a value, in other words there is a *list* of datatypes, as opposed to only one + or: + 2. That the short form syntax is used in the datatype name: variant.""" + + # Convert "truthy" result object to actual bool for nicer debugging + return bool( isinstance(f, ifex_ast.Typedef) and (f.datatypes or is_ifex_variant_shortform(f.datatype)) ) + +def is_ifex_invalid_typedef(f): + """Check if both a single and multiple datatypes are defined. That is invalid.""" + return is_ifex_variant_typedef(f) and f.datatype and f.datatypes + +def get_variant_types(obj): + """Return a list of the types handled by the given variant type. The function accepts either a Typedef object or a string with the type name. The string must then be the variant fundamental type - it cannot be the name of a typedef.""" + if isinstance(obj, ifex_ast.Typedef): + if is_ifex_invalid_typedef(obj): + raise TypeException('Provided variant object is misconfigured') + # Process datatypes list + if obj.datatypes: + return obj.datatypes + # or process single datatype (string) + else: + return get_variant_types(obj.datatype) + elif type(obj) == str: + match = re.search(r'variant *<(.*?)>', obj) + if match: + types = match.group(1).split(',') + return [t.strip() for t in types] + + # (else) Any other cases = error + raise Exception('Provided object is not a variant type: {obj=}') + +# ------------------------------------------------------------------------------------------------ + VERBOSE = False # Tree processing function: @@ -210,3 +281,14 @@ def _simple_process(arg): if __name__ == "__main__": print("TEST: Note that already seen types are skipped, and this is a depth-first search => The structure of the tree is not easily seen from this output.") walk_type_tree(ifex_ast.Namespace, _simple_process) + + x = ifex_ast.Typedef("name", datatype="variant") + print(f"{is_ifex_variant_shortform(x.datatype)=}") + print(f"{is_ifex_variant_typedef(x)=}") + + y = ifex_ast.Typedef("name", datatypes=["foo", "bar", "baz"]) + print(f"{is_ifex_variant_shortform(y.datatype)=}") + print(f"{is_ifex_variant_typedef(y)=}") + print(f"The types of y are: {get_variant_types(y)}") + print(f"The types of x are: {get_variant_types(x)}") + print(f"The types of variant are: {get_variant_types("variant")}") From ad0ea0a2135df09942eb1cc0699f8716bdf5fcd8 Mon Sep 17 00:00:00 2001 From: Gunnar Andersson Date: Tue, 10 Dec 2024 17:30:35 +0100 Subject: [PATCH 3/5] tests: Add variant type tests --- ifex/model/ifex_ast_introspect.py | 2 +- tests/gen_test.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/ifex/model/ifex_ast_introspect.py b/ifex/model/ifex_ast_introspect.py index 566a804..29616b2 100644 --- a/ifex/model/ifex_ast_introspect.py +++ b/ifex/model/ifex_ast_introspect.py @@ -15,7 +15,7 @@ """ import re -import ifex_ast +import ifex.model.ifex_ast as ifex_ast from dataclasses import is_dataclass, fields from typing import get_args, get_origin, List, Optional, Union, Any, ForwardRef import typing diff --git a/tests/gen_test.py b/tests/gen_test.py index f7a83cd..fc53d95 100644 --- a/tests/gen_test.py +++ b/tests/gen_test.py @@ -11,6 +11,7 @@ import yaml from ifex.model import ifex_ast, ifex_parser, ifex_generator +import ifex.model.ifex_ast_introspect as introspect import dacite, pytest import os @@ -55,6 +56,28 @@ def test_ast_gen(): assert service.minor_version == 0 +# This does not assert anything -> but if printouts are captured they can be studied +def test_print(): + ast = ifex_parser.get_ast_from_yaml_file(os.path.join(TestPath, 'test.variant', 'input.yaml')) + print(yaml.dump(ast, sort_keys=False)) + print(introspect.get_variant_types(ast.namespaces[0].typedefs[0])) + +# Test expectations and helper-functions on the variant type +def test_variant(): + ast = ifex_parser.get_ast_from_yaml_file(os.path.join(TestPath, 'test.variant', 'input.yaml')) + # Method argument + v0type = ast.namespaces[0].methods[0].input[0].datatype + assert not introspect.is_ifex_variant_typedef(v0type) + assert introspect.is_ifex_variant_shortform(v0type) + + # Typdefs + v1 = ast.namespaces[0].typedefs[0] + v2 = ast.namespaces[0].typedefs[1] + assert introspect.is_ifex_variant_typedef(v1) + assert introspect.is_ifex_variant_typedef(v2) + assert introspect.is_ifex_variant_shortform(v2.datatype) + + def test_ast_manual(): service = ifex_ast.AST(name='test', description='test', major_version=1, minor_version=0) From 6b3dc8ce2e06c15aecee6786d7e17a36ac5df354 Mon Sep 17 00:00:00 2001 From: Gunnar Andersson Date: Tue, 3 Dec 2024 23:10:44 +0100 Subject: [PATCH 4/5] Docker: Update ubuntu base to 24.04 Ubuntu 23.04 is EOL now Signed-off-by: Gunnar Andersson --- docker/Dockerfile.ubuntu | 2 +- ifex/model/ifex_ast_introspect.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu index ef546fd..61a9046 100644 --- a/docker/Dockerfile.ubuntu +++ b/docker/Dockerfile.ubuntu @@ -4,7 +4,7 @@ # This file is part of IFEX project # --------------------------------------------------------------------------- -FROM ubuntu:23.04 +FROM ubuntu:24.04 # Install what's needed to compile python from source using pyenv RUN apt-get update && apt-get install -y bash build-essential curl git libbz2-dev libffi-dev liblzma-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev llvm lzma-dev make ncurses-dev python3-openssl python3 sudo tk-dev wget xz-utils zlib1g-dev diff --git a/ifex/model/ifex_ast_introspect.py b/ifex/model/ifex_ast_introspect.py index 29616b2..bb82779 100644 --- a/ifex/model/ifex_ast_introspect.py +++ b/ifex/model/ifex_ast_introspect.py @@ -292,3 +292,4 @@ def _simple_process(arg): print(f"The types of y are: {get_variant_types(y)}") print(f"The types of x are: {get_variant_types(x)}") print(f"The types of variant are: {get_variant_types("variant")}") + From 9fbcccfeaddf4b093c10e5a308e07d25a59d68ae Mon Sep 17 00:00:00 2001 From: Gunnar Andersson Date: Tue, 10 Dec 2024 17:41:25 +0100 Subject: [PATCH 5/5] test fix --- ifex/model/ifex_ast_introspect.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ifex/model/ifex_ast_introspect.py b/ifex/model/ifex_ast_introspect.py index bb82779..a8f029b 100644 --- a/ifex/model/ifex_ast_introspect.py +++ b/ifex/model/ifex_ast_introspect.py @@ -291,5 +291,6 @@ def _simple_process(arg): print(f"{is_ifex_variant_typedef(y)=}") print(f"The types of y are: {get_variant_types(y)}") print(f"The types of x are: {get_variant_types(x)}") - print(f"The types of variant are: {get_variant_types("variant")}") + s = "variant" + print(f"The types of {s} are: {get_variant_types(s)}")