Draft: Variant type support

.github/workflows/create-new-release.yml

@@ -63,7 +63,7 @@ jobs:
           tag_name: "${{ steps.vars.outputs.TAG }}"
           body: |
             The JSON Schema file for the IFEX Core IDL is a resulting artifact, generated from the internal model representation.
-            This one was generated from commit hash ${{ steps.vars.outputs.HASH }} and was assigned the release-name: ${{ steps.vars.outputs.TAG }}."
+            This one was generated from commit hash ${{ steps.vars.outputs.HASH }} and was assigned the release-name: ${{ steps.vars.outputs.TAG }}.
         env:
           GITHUB_TOKEN: "${{ github.token }}"
 

docker/Dockerfile.ubuntu

@@ -4,7 +4,7 @@
 # This file is part of IFEX project
 # ---------------------------------------------------------------------------
 
-FROM ubuntu:23.04
+FROM ubuntu:24.04
 
 # Install what's needed to compile python from source using pyenv
 RUN apt-get update && apt-get install -y bash build-essential curl git libbz2-dev libffi-dev liblzma-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev llvm lzma-dev make ncurses-dev python3-openssl python3 sudo tk-dev wget xz-utils zlib1g-dev

ifex/model/ifex_ast.py

@@ -436,9 +436,12 @@ class Typedef:
     name: str
     """ Specifies the name of the typedef. """
 
-    datatype: str
+    datatype: Optional[str] = str()
     """ Specifies datatype name of the typedef. """
 
+    datatypes: Optional[List[str]] = field(default_factory=EmptyList)
+    """ If specified, then the type is a variant type.  At least two datatypes should be listed.  The single datatype: field must *not* be used at the same time. """
+
     description: Optional[str] = str()
     """ Specifies the description of the typedef. """
 
@@ -685,5 +688,6 @@ class FundamentalTypes:
             # name, description, min value, max value
             ["set", "A set (unique values), each of the same type. Format: set<ItemType>", "N/A", "N/A"],
             ["map", "A key-value mapping type.  Format: map<keytype,valuetype>", "N/A", "N/A"],
+            ["variant", "A variant (union) type that can carry any of a predefined set of types, akin to Union in C.  Format: variant<type1,type2,type3...>", "N/A", "N/A"],
             ["opaque", "Indicates a complex type which is not explicitly defined in this context.", "N/A","N/A"]
             ]

ifex/model/ifex_ast_introspect.py

@@ -9,40 +9,44 @@
 
 """
 Provide helper functions to inspect the IFEX Core IDL language definition,
-as it is defined by the class tree/hierarchy (not an inheritance hierarchy)
-in the `ifex_ast` python file.  These function can be used by any other code
-that needs to process this underlying meta-model. It helps to ensure that the
-fundamental language is defined in a single file.  """
+as it is defined by the AST structure tree/hierarchy (not an inheritance hierarchy)
+in the `ifex_ast` python file.  These function can be used by implementations
+that process the IFEX AST, or any other model designed in the same way.
+"""
 
-from ifex.model import ifex_ast
+import re
+import ifex.model.ifex_ast as ifex_ast
 from dataclasses import is_dataclass, fields
 from typing import get_args, get_origin, List, Optional, Union, Any, ForwardRef
 import typing
 
-# As we traverse the "tree" of dataclass definitions, it can be quite difficult
+# As we traverse the tree of dataclass definitions, it can be quite difficult
 # to keep track of which type each variable has.  Here is an explanation of how
 # we try to keep track:
 #
 # The following functions come in two flavors each.  Functions like: is_xxx()
 # take an object, which is an instance of typing.<some class> which is
 # essentially an object that indicates the "type hint" using concepts from
 # the 'typing' python module. Examples are: Optional, List, Union, Any, etc.
-# We here call variables that reference such a typing.Something object a
-# type_indicator.  It corresponds to the type hint information on the right
-# side of the colon : in an expression like this:
+# In these functions we call variables that reference such a typing.Something
+# object, a type_indicator.  The type_indicator corresponds to the type hint
+# information on the right side of the colon : in an expression like this:
 #
 #    namespaces:  Optional[List[Namespace]]
 #
 # The type_indicator is the:  `Optional[List[Namespace]]`
 # (or if fully qualified: `typing.Optional[typing.List[ifex_ast.Namespace]]`)
-# Note that instead of being a dataclass like ifex_ast.Namespace, the inner
-# type can of course be a built-in simple type like str.  e.g. typing.List[str]
+# The inner type can be an instance of an AST node, in other words an instance
+# of a @dataclass like ifex_ast.Namespace, or it can be a built-in simple type
+# like str.  e.g. typing.List[str]
 #
-# Next, in the 'dataclasses' python module we find the function fields().
-# It returns a list that represents the fields (members) of the dataclass.
+# Next, in the 'dataclasses' python module we find the function: fields().
+# fields() returns a list of the fields (i.e member variables) of the dataclass.
 # Each field is represented by an object (an instance of the dataclasses.Field
-# class).  We name variables that refer to such Field() instances as `field`.
-# A field thus represents a member variable in the python (data)class.
+# class).  In the following function we name variables that refer to such
+# Field() instances as `field`.  A variable named field thus represents a
+# member variable in the python (data)class.
+#
 # A field object contains several informations such as the name of the member
 # variable (field.name), and the `.type` member, which gives us the
 # type_indicator as described above.
@@ -55,7 +59,7 @@
 # NOTE: Here in the descriptions we might refer to an object's "type" when we
 # strictly mean its Type Indicator.  Since typing in python is dynamic,
 # the actual type of an object could be different (and can be somewhat fungible
-# too in theory, but generally not in this code).
+# too in theory, but usually not in this code).
 
 def is_dataclass_type(cls):
     """Check if a class is a dataclass."""
@@ -142,6 +146,73 @@ def field_referenced_type(f):
     else:
         return field_actual_type(f)
 
+
+# --- End of generic functions --
+
+# ------------------------------------------------------------------------------------------------
+# Above thes line we had generic functions that give information about a AST-model built
+# from a number of @dataclasses and field typing information.
+#
+# We can notice that those only refers to *python* concepts such as @dataclasses and the typing module.
+# This means, those functions could also be used for other similarly described AST models, beyond
+# the IFEX Core IDL model.
+#
+# Below this line follows functions that are specific to IFEX. For example is_ifex_variant_typedef evaluates
+# an actual IFEX-specific concern about the IFEX variant type.  It is not a variant type of the python
+# typing concept (which is called typing.Union anyhow) - these functions are about IFEX-specific concerns.
+
+# Check if string is "variant<something>" where something can be empty
+p_shortform_variant0 = r'variant<\s*([^,]*\s*(?:,\s*[^,]*)*)\s*>'
+
+# ...and this for variant<at_least_one>
+p_shortform_variant1 = r'variant<\s*([^,]+(?:\s*,\s*[^,]+)*)\s*>'
+
+# ...and this is the one we actually need. A valid variant has at least 2
+# types listed or it would not make sense. This last pattern guarantees this:
+p_shortform_variant2 = r'variant<\s*[^,]+(?:\s*,\s*[^,]+)+\s*>'
+
+def is_ifex_variant_shortform(s):
+    """ Answer if a Typedef object has datatype defined to using the short form: variant<type1,type2,type3...> """
+
+    # Convert "truthy" result object to actual bool for nicer debugging
+    return bool(s and s != '' and re.match(p_shortform_variant2, s))
+
+def is_ifex_variant_typedef(f):
+    """ Answer if a Typedef object uses a variant type.
+    A variant type can be defined in either one of these two ways:
+    1. The field "datatypes" has a value, in other words there is a *list* of datatypes, as opposed to only one
+    or:
+    2. That the short form syntax is used in the datatype name:  variant<type1,type2,...>."""
+
+    # Convert "truthy" result object to actual bool for nicer debugging
+    return bool( isinstance(f, ifex_ast.Typedef) and (f.datatypes or is_ifex_variant_shortform(f.datatype)) )
+
+def is_ifex_invalid_typedef(f):
+    """Check if both a single and multiple datatypes are defined. That is invalid."""
+    return is_ifex_variant_typedef(f) and f.datatype and f.datatypes
+
+def get_variant_types(obj):
+    """Return a list of the types handled by the given variant type.  The function accepts either a Typedef object or a string with the type name. The string must then be the variant<a,b,c> fundamental type - it cannot be the name of a typedef."""
+    if isinstance(obj, ifex_ast.Typedef):
+        if is_ifex_invalid_typedef(obj):
+            raise TypeException('Provided variant object is misconfigured')
+        # Process datatypes list
+        if obj.datatypes:
+            return obj.datatypes
+        # or process single datatype (string)
+        else: 
+            return get_variant_types(obj.datatype)
+    elif type(obj) == str:
+        match = re.search(r'variant *<(.*?)>', obj)
+        if match:
+            types = match.group(1).split(',')
+            return [t.strip() for t in types]
+
+    # (else) Any other cases = error
+    raise Exception('Provided object is not a variant type: {obj=}')
+
+# ------------------------------------------------------------------------------------------------
+
 VERBOSE = False
 
 # Tree processing function:
@@ -210,3 +281,16 @@ def _simple_process(arg):
 if __name__ == "__main__":
     print("TEST: Note that already seen types are skipped, and this is a depth-first search =>  The structure of the tree is not easily seen from this output.")
     walk_type_tree(ifex_ast.Namespace, _simple_process)
+
+    x = ifex_ast.Typedef("name", datatype="variant<a,b>")
+    print(f"{is_ifex_variant_shortform(x.datatype)=}")
+    print(f"{is_ifex_variant_typedef(x)=}")
+
+    y = ifex_ast.Typedef("name", datatypes=["foo", "bar", "baz"])
+    print(f"{is_ifex_variant_shortform(y.datatype)=}")
+    print(f"{is_ifex_variant_typedef(y)=}")
+    print(f"The types of y are: {get_variant_types(y)}")
+    print(f"The types of x are: {get_variant_types(x)}")
+    s = "variant<this, that ,and,another >"
+    print(f"The types of {s} are: {get_variant_types(s)}")
+

tests/gen_test.py

@@ -11,6 +11,7 @@
 import yaml
 
 from ifex.model import ifex_ast, ifex_parser, ifex_generator
+import ifex.model.ifex_ast_introspect as introspect
 import dacite, pytest
 import os
 
@@ -55,6 +56,28 @@ def test_ast_gen():
     assert service.minor_version == 0
 
 
+# This does not assert anything -> but if printouts are captured they can be studied
+def test_print():
+    ast = ifex_parser.get_ast_from_yaml_file(os.path.join(TestPath, 'test.variant', 'input.yaml'))
+    print(yaml.dump(ast, sort_keys=False))
+    print(introspect.get_variant_types(ast.namespaces[0].typedefs[0]))
+
+# Test expectations and helper-functions on the variant type
+def test_variant():
+    ast = ifex_parser.get_ast_from_yaml_file(os.path.join(TestPath, 'test.variant', 'input.yaml'))
+    # Method argument
+    v0type = ast.namespaces[0].methods[0].input[0].datatype
+    assert not introspect.is_ifex_variant_typedef(v0type)
+    assert introspect.is_ifex_variant_shortform(v0type)
+
+    # Typdefs
+    v1 = ast.namespaces[0].typedefs[0]
+    v2 = ast.namespaces[0].typedefs[1]
+    assert introspect.is_ifex_variant_typedef(v1)
+    assert introspect.is_ifex_variant_typedef(v2)
+    assert introspect.is_ifex_variant_shortform(v2.datatype)
+
+
 def test_ast_manual():
     service = ifex_ast.AST(name='test', description='test', major_version=1, minor_version=0)