Merge pull request #226 from epics-containers/type-restore

Support Jinja in non str Args

.vscode/settings.json

@@ -1,8 +1,6 @@
 {
     "python.testing.pytestArgs": [
-        "--cov=ibek",
-        "--cov-report",
-        "xml:cov.xml"
+        "tests"
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,

docs/developer/explanations/entities.rst

@@ -149,7 +149,7 @@ Click the arrows to reveal the files.
         <details>
         <summary><a>all.ibek.ioc.yaml</a></summary>
 
-    .. include:: ../../../tests/samples/iocs/ibek-mo-ioc-01.yaml
+    .. include:: ../../../tests/samples/iocs/motorSim.ibek.ioc.yaml
         :literal:
 
     .. raw:: html

pyproject.toml

@@ -66,6 +66,7 @@ ignore_missing_imports = true # Ignore missing stubs in imported modules
 addopts = """
     --tb=native -vv --doctest-modules --doctest-glob="*.rst"
     --cov=ibek --cov-report term --cov-report xml:cov.xml
+    --ignore tests/samples
     """
 # https://iscinumpy.gitlab.io/post/bound-version-constraints/#watch-for-warnings
 filterwarnings = [

src/ibek/args.py

@@ -4,12 +4,33 @@
 
 from __future__ import annotations
 
+from enum import Enum
 from typing import Any, Dict, Optional
 
 from pydantic import Field
-from typing_extensions import Literal
+from typing_extensions import Annotated, Literal
 
-from .globals import BaseSettings
+from .globals import JINJA, BaseSettings
+
+JinjaString = Annotated[
+    str, Field(description="A Jinja2 template string", pattern=JINJA)
+]
+
+
+class ValueTypes(Enum):
+    """The type of a value"""
+
+    string = "str"
+    float = "float"
+    int = "int"
+    bool = "bool"
+    list = "list"
+
+    def __str__(self):
+        return str(self.value)
+
+    def __repr__(self):
+        return str(self.value)
 
 
 class Value(BaseSettings):
@@ -19,7 +40,10 @@ class Value(BaseSettings):
     description: str = Field(
         description="Description of what the value will be used for"
     )
-    value: str = Field(description="The contents of the value")
+    value: Any = Field(description="The contents of the value")
+    type: ValueTypes = Field(
+        description="The type of the value", default=ValueTypes.string
+    )
 
 
 class Arg(BaseSettings):
@@ -38,7 +62,7 @@ class FloatArg(Arg):
     """An argument with a float value"""
 
     type: Literal["float"] = "float"
-    default: Optional[float] = None
+    default: Optional[float | JinjaString] = None
 
 
 class StrArg(Arg):
@@ -52,14 +76,14 @@ class IntArg(Arg):
     """An argument with an int value"""
 
     type: Literal["int"] = "int"
-    default: Optional[int] = None
+    default: Optional[int | JinjaString] = None
 
 
 class BoolArg(Arg):
     """An argument with an bool value"""
 
     type: Literal["bool"] = "bool"
-    default: Optional[bool] = None
+    default: Optional[bool | JinjaString] = None
 
 
 class ObjectArg(Arg):

src/ibek/definition.py

@@ -5,7 +5,7 @@
 from __future__ import annotations
 
 from enum import Enum
-from typing import Any, Mapping, Optional, Sequence, Union
+from typing import Annotated, Any, Mapping, Optional, Sequence, Union
 
 from pydantic import Field, PydanticUndefinedAnnotation
 from typing_extensions import Literal
@@ -115,6 +115,12 @@ class EntityPVI(BaseSettings):
     pv_prefix: str = Field("", description='PV prefix for PVI PV - e.g. "$(P)"')
 
 
+discriminated = Annotated[  # type: ignore
+    Union[tuple(Arg.__subclasses__())],
+    Field(discriminator="type", description="union of arg types"),
+]
+
+
 class EntityDefinition(BaseSettings):
     """
     A single definition of a class of Entity that an IOC instance may instantiate
@@ -127,15 +133,17 @@ class EntityDefinition(BaseSettings):
         description="A description of the Support module defined here"
     )
     # declare Arg as Union of its subclasses for Pydantic to be able to deserialize
-    args: Sequence[Union[tuple(Arg.__subclasses__())]] = Field(  # type: ignore
-        description="The arguments IOC instance should supply", default=()
+
+    args: Sequence[discriminated] = Field(  # type: ignore
+        description="The arguments IOC instance should supply",
+        default=(),
     )
-    values: Sequence[Value] = Field(
+    post_defines: Sequence[Value] = Field(
         description="Calculated values to use as additional arguments "
         "With Jinja evaluation after all Args",
         default=(),
     )
-    pre_values: Sequence[Value] = Field(
+    pre_defines: Sequence[Value] = Field(
         description="Calculated values to use as additional arguments "
         "With Jinja evaluation before all Args",
         default=(),

src/ibek/entity_factory.py

@@ -6,14 +6,15 @@
 
 import builtins
 from pathlib import Path
-from typing import Any, Dict, List, Literal, Tuple, Type
+from typing import Annotated, Any, Dict, List, Literal, Sequence, Tuple, Type
 
-from pydantic import create_model, field_validator
-from pydantic.fields import FieldInfo
+from pydantic import Field, create_model, field_validator
 from pydantic_core import PydanticUndefined, ValidationError
 from ruamel.yaml.main import YAML
 
-from .args import EnumArg, IdArg, ObjectArg
+from ibek.globals import JINJA
+
+from .args import EnumArg, IdArg, ObjectArg, Value
 from .ioc import Entity, EnumVal, clear_entity_model_ids, get_entity_by_id
 from .support import EntityDefinition, Support
 from .utils import UTILS
@@ -61,13 +62,36 @@ def _make_entity_model(
         Create an Entity Model from a Definition instance and a Support instance.
         """
 
+        def add_defines(s: Sequence[Value]) -> None:
+            # add in the pre_defines or post_defines as Args in the Entity
+            for value in s:
+                typ = getattr(builtins, str(value.type))
+                add_arg(value.name, typ, value.description, value.value)
+
         def add_arg(name, typ, description, default):
             if default is None:
                 default = PydanticUndefined
-            args[name] = (
-                typ,
-                FieldInfo(description=description, default=default),
-            )
+            # cheesy check for enum, can this be improved?
+            if typ in [str, object] or "enum" in str(typ):
+                args[name] = (Annotated[typ, Field(description=description)], default)
+            else:
+                args[name] = (
+                    Annotated[
+                        Annotated[
+                            str,
+                            Field(
+                                description=f"jinja that renders to {typ}",
+                                pattern=JINJA,
+                            ),
+                        ]
+                        | Annotated[typ, Field(description=description)],
+                        Field(
+                            description=f"union of {typ} and jinja "
+                            "representation of {typ}"
+                        ),
+                    ],
+                    default,
+                )
 
         args: Dict[str, Tuple[type, Any]] = {}
         validators: Dict[str, Any] = {}
@@ -78,25 +102,26 @@ def add_arg(name, typ, description, default):
         # add in the calculated values Jinja Templates as Fields in the Entity
         # these are the pre_values that should be Jinja rendered before any
         # Args (or post values)
-        for value in definition.pre_values:
-            add_arg(value.name, str, value.description, value.value)
+        add_defines(definition.pre_defines)
 
         # add in each of the arguments as a Field in the Entity
         for arg in definition.args:
-            full_arg_name = f"{full_name}.{arg.name}"
-            arg_type: Any
+            # TODO - don't understand why I need type ignore here
+            # arg is 'discriminated' which is a Union of all the Arg subclasses
+            full_arg_name = f"{full_name}.{arg.name}"  # type: ignore
+            type: Any
 
             if isinstance(arg, ObjectArg):
-
-                @field_validator(arg.name, mode="after")
+                # TODO look into why arg.name requires type ignore
+                @field_validator(arg.name, mode="after")  # type: ignore
                 def lookup_instance(cls, id):
                     return get_entity_by_id(id)
 
                 validators[full_arg_name] = lookup_instance
-                arg_type = object
+                type = object
 
             elif isinstance(arg, IdArg):
-                arg_type = str
+                type = str
 
             elif isinstance(arg, EnumArg):
                 # Pydantic uses the values of the Enum as the options in the schema.
@@ -107,20 +132,20 @@ def lookup_instance(cls, id):
                     enum_swapped[str(v) if v else str(k)] = k
                 # TODO review enums especially with respect to Pydantic 2.7.1
                 val_enum = EnumVal(arg.name, enum_swapped)  # type: ignore
-                arg_type = val_enum
+                type = val_enum
 
             else:
                 # arg.type is str, int, float, etc.
-                arg_type = getattr(builtins, arg.type)
-            add_arg(arg.name, arg_type, arg.description, getattr(arg, "default"))
+                type = getattr(builtins, arg.type)
+            # TODO look into why arg.name requires type ignore
+            add_arg(arg.name, type, arg.description, getattr(arg, "default"))  # type: ignore
 
         # add in the calculated values Jinja Templates as Fields in the Entity
-        for value in definition.values:
-            add_arg(value.name, str, value.description, value.value)
+        add_defines(definition.post_defines)
 
         # add the type literal which discriminates between the different Entity classes
         typ = Literal[full_name]  # type: ignore
-        add_arg("type", typ, definition.description, full_name)
+        args["type"] = (typ, Field(description=definition.description))
 
         class_name = full_name.replace(".", "_")
         entity_cls = create_model(

src/ibek/gen_scripts.py

@@ -50,10 +50,8 @@ def create_boot_script(entities: Sequence[Entity]) -> str:
     renderer = Render()
 
     return template.render(
-        # old name for global context for backward compatibility
-        __utils__=UTILS,
-        # new short name for global context
-        _ctx_=UTILS,
+        # global context for all jinja renders
+        _global=UTILS,
         # put variables created with set/get directly in the context
         **UTILS.variables,
         env_var_elements=renderer.render_environment_variable_elements(entities),

src/ibek/globals.py

@@ -138,6 +138,8 @@ def RUNTIME_DEBS(self):
 
 GLOBALS = _Globals()
 
+JINJA = r".*\{\{.*\}\}.*"
+
 
 class BaseSettings(BaseModel):
     """A Base class for setting consistent Pydantic model configuration"""

src/ibek/ioc.py

@@ -5,6 +5,7 @@
 
 from __future__ import annotations
 
+import json
 from enum import Enum
 from typing import Any, Dict, List, Sequence
 
@@ -71,7 +72,24 @@ def add_ibek_attributes(self):
             if isinstance(value, str):
                 # Jinja expansion of any of the Entity's string args/values
                 value = UTILS.render(entity_dict, value)
-                setattr(self, arg, str(value))
+                # this is a cheesy test - any better ideas please let me know
+                if "Union" in str(model_field.annotation):
+                    # Args that were non strings and have been rendered by Jinja
+                    # must be coerced back into their original type
+                    try:
+                        # The following replace are to make the string json compatible
+                        # (maybe we should python decode instead of json.loads?)
+                        value = value.replace("'", '"')
+                        value = value.replace("True", "true")
+                        value = value.replace("False", "false")
+                        value = json.loads(value)
+                        # likewise for bools
+                    except:
+                        print(
+                            f"ERROR: fail to decode {value} as a {model_field.annotation}"
+                        )
+                        raise
+                setattr(self, arg, value)
                 # update the entity_dict with the rendered value
                 entity_dict[arg] = value
 

src/ibek/support.py

@@ -5,7 +5,7 @@
 from __future__ import annotations
 
 import json
-from typing import Sequence
+from typing import Any, Sequence
 
 from pydantic import Field
 
@@ -20,6 +20,11 @@ class Support(BaseSettings):
     Provides the deserialize entry point.
     """
 
+    shared: Sequence[Any] = Field(
+        description="A place to create any anchors required for repeating YAML",
+        default=(),
+    )
+
     module: str = Field(description="Support module name, normally the repo name")
     defs: Sequence[EntityDefinition] = Field(
         description="The definitions an IOC can create using this module"

src/ibek/templates/st.cmd.jinja

@@ -1,6 +1,6 @@
 # EPICS IOC Startup Script generated by https://github.com/epics-containers/ibek
 
-cd "{{ __utils__.get_env('IOC') }}"
+cd "{{ _global.get_env('IOC') }}"
 {% if env_var_elements %}
 {{ env_var_elements}}
 {% endif -%}
@@ -9,7 +9,7 @@ dbLoadDatabase dbd/ioc.dbd
 ioc_registerRecordDeviceDriver pdbbase
 
 {{ script_elements }}
-dbLoadRecords {{ __utils__.get_env('RUNTIME_DIR') }}/ioc.db
+dbLoadRecords {{ _global.get_env('RUNTIME_DIR') }}/ioc.db
 iocInit
 
 {% if post_ioc_init_elements %}

src/ibek/utils.py

@@ -2,7 +2,7 @@
 A class containing utility functions for passing into the Jinja context.
 
 This allows us to provide simple functions that can be called inside
-Jinja templates with {{ __utils__.function_name() }}. It also allows
+Jinja templates with {{ _global.function_name() }}. It also allows
 us to maintain state between calls to the Jinja templates because
 we pass a single instance of this class into all Jinja contexts.
 """
@@ -114,7 +114,7 @@ def counter(
 
     def render(self, context: Any, template_text: Any) -> str:
         """
-        Render a Jinja template with the global __utils__ object in the context
+        Render a Jinja template with the global _global object in the context
         """
         if not isinstance(template_text, str):
             # because this function is used to template arguments, it may
@@ -125,10 +125,8 @@ def render(self, context: Any, template_text: Any) -> str:
             jinja_template = Template(template_text, undefined=StrictUndefined)
             return jinja_template.render(
                 context,
-                # old name for global context for backward compatibility
-                __utils__=self,
-                # new short name for global context
-                _ctx_=self,
+                # global context for all jinja renders
+                _global=self,
                 # put variables created with set/get directly in the context
                 **self.variables,
                 ioc_yaml_file_name=self.file_name,