Merge pull request #229 from epics-containers/improve-jinja

Improve jinja capabilities further

src/ibek/args.py

@@ -90,7 +90,9 @@ class ObjectArg(Arg):
     """A reference to another entity defined in this IOC"""
 
     type: Literal["object"] = "object"
-    default: Optional[str] = None
+    # represented by an id string in YAML but converted to an Entity object
+    # during validation
+    default: Optional[str | object] = None
 
 
 class IdArg(Arg):

src/ibek/entity_factory.py

@@ -8,14 +8,14 @@
 from pathlib import Path
 from typing import Annotated, Any, Dict, List, Literal, Sequence, Tuple, Type
 
-from pydantic import Field, create_model, field_validator
+from pydantic import Field, create_model
 from pydantic_core import PydanticUndefined, ValidationError
 from ruamel.yaml.main import YAML
 
 from ibek.globals import JINJA
 
 from .args import EnumArg, IdArg, ObjectArg, Value
-from .ioc import Entity, EnumVal, clear_entity_model_ids, get_entity_by_id
+from .ioc import Entity, EnumVal, clear_entity_model_ids
 from .support import EntityDefinition, Support
 from .utils import UTILS
 
@@ -106,18 +106,10 @@ def add_arg(name, typ, description, default):
 
         # add in each of the arguments as a Field in the Entity
         for arg in definition.args:
-            # TODO - don't understand why I need type ignore here
-            # arg is 'discriminated' which is a Union of all the Arg subclasses
-            full_arg_name = f"{full_name}.{arg.name}"  # type: ignore
             type: Any
 
             if isinstance(arg, ObjectArg):
-                # TODO look into why arg.name requires type ignore
-                @field_validator(arg.name, mode="after")  # type: ignore
-                def lookup_instance(cls, id):
-                    return get_entity_by_id(id)
-
-                validators[full_arg_name] = lookup_instance
+                # we now defer the lookup of the object until whole model validation
                 type = object
 
             elif isinstance(arg, IdArg):

src/ibek/ioc.py

@@ -78,7 +78,7 @@ def add_ibek_attributes(self):
                     # must be coerced back into their original type
                     try:
                         # The following replace are to make the string json compatible
-                        # (maybe we should python decode instead of json.loads?)
+                        # (maybe we should python decode instead of json.loads)
                         value = value.replace("'", '"')
                         value = value.replace("True", "true")
                         value = value.replace("False", "false")
@@ -94,10 +94,12 @@ def add_ibek_attributes(self):
                 entity_dict[arg] = value
 
             if model_field.annotation == object:
-                # if the field is an object but the type is str then look up
-                # the actual object (this covers default values with obj ref)
+                # look up the actual object by it's id
                 if isinstance(value, str):
-                    setattr(self, arg, get_entity_by_id(value))
+                    value = get_entity_by_id(value)
+                    setattr(self, arg, value)
+                # update the entity_dict with looked up object
+                entity_dict[arg] = value
 
             if arg in ids:
                 # add this entity to the global id index

src/ibek/utils.py

@@ -8,37 +8,12 @@
 """
 
 import os
-from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Dict, Mapping
 
 from jinja2 import StrictUndefined, Template
 
 
-@dataclass
-class Counter:
-    """
-    Provides the ability to supply unique numbers to Jinja templates
-    """
-
-    start: int
-    current: int
-    stop: int
-
-    def increment(self, count: int):
-        self.current += count
-        if self.current > self.stop:
-            raise ValueError(
-                f"Counter {self.current} exceeded stop value of {self.stop}"
-            )
-
-    def __repr__(self) -> str:
-        return str(self.current)
-
-    def __str__(self) -> str:
-        return str(self.current)
-
-
 class Utils:
     """
     A Utility class for adding functions to the Jinja context
@@ -49,17 +24,12 @@ def __init__(self: "Utils"):
         self.ioc_name: str = ""
         self.__reset__()
 
-        # old names for backward compatibility
-        self.set_var = self.set
-        self.get_var = self.get
-
     def __reset__(self: "Utils"):
         """
         Reset all saved state. For use in testing where more than one
         IOC is rendered in a single session
         """
         self.variables: Dict[str, Any] = {}
-        self.counters: Dict[str, Counter] = {}
 
     def set_file_name(self: "Utils", file: Path):
         """
@@ -81,36 +51,37 @@ def get_env(self, key: str) -> str:
 
     def set(self, key: str, value: Any) -> Any:
         """create a global variable for our jinja context"""
-        self.variables[key] = value
+        s_key = str(key)
+        self.variables[s_key] = value
         return value
 
     def get(self, key: str, default="") -> Any:
         """get the value a global variable for our jinja context"""
         # default is used to set an initial value if the variable is not set
-        return self.variables.get(key, default)
+        s_key = str(key)
+        return self.variables.get(s_key, default)
 
-    def counter(
-        self, name: str, start: int = 0, stop: int = 65535, inc: int = 1
+    def incrementor(
+        self, name: str, start: int = 0, increment: int = 1, stop: int | None = None
     ) -> int:
         """
         get a named counter that increments by inc each time it is called
 
         creates a new counter if it does not yet exist
         """
-        counter = self.counters.get(name)
+        index = str(name)
+        counter = self.variables.get(index)
+
         if counter is None:
-            counter = Counter(start, start, stop)
-            self.counters[name] = counter
+            self.variables[index] = start
         else:
-            if counter.start != start or counter.stop != stop:
-                raise ValueError(
-                    f"Redefining counter {name} with different start/stop values"
-                )
-        result = counter.current
-        counter.increment(inc)
-        self.counters[name] = counter
-
-        return result
+            if not isinstance(counter, int):
+                raise ValueError(f"Variable {index} is not an integer")
+            self.variables[index] += increment
+            if stop is not None and self.variables[index] > stop:
+                raise ValueError(f"Counter {index} exceeded maximum value of {stop}")
+
+        return self.variables[index]
 
     def render(self, context: Any, template_text: Any) -> str:
         """

tests/samples/iocs/technosoft.ibek.ioc.yaml

@@ -29,7 +29,6 @@ entities:
     controllerName: TML
     P: "SPARC:TML"
     TTY: /tmp # /var/tmp/ttyV0
-    numAxes: 1
     hostid: 15
 
   - <<: *motor

tests/samples/outputs/technosoft/st.cmd

@@ -4,7 +4,7 @@ cd "/epics/ioc"
 dbLoadDatabase dbd/ioc.dbd
 ioc_registerRecordDeviceDriver pdbbase
 
-ndsCreateDevice "TechnosoftTML", "TML", "FILE=/tmp/,NAXIS=1,DEV_PATH=/tmp,HOST_ID=15,AXIS_SETUP_0=$(SUPPORT)/motorTechnosoft/tml_lib/config/star_vat_phs.t.zip,AXIS_ID_0=15,AXIS_HOMING_SW_0=LSN,AXIS_SETUP_1=$(SUPPORT)/motorTechnosoft/tml_lib/config/star_vat_phs.t.zip,AXIS_ID_1=17,AXIS_HOMING_SW_1=LSN"
+ndsCreateDevice "TechnosoftTML", "TML", "FILE=/tmp/,NAXIS=2,DEV_PATH=/tmp,HOST_ID=15,AXIS_SETUP_0=$(SUPPORT)/motorTechnosoft/tml_lib/config/star_vat_phs.t.zip,AXIS_ID_0=15,AXIS_HOMING_SW_0=LSN,AXIS_SETUP_1=$(SUPPORT)/motorTechnosoft/tml_lib/config/star_vat_phs.t.zip,AXIS_ID_1=17,AXIS_HOMING_SW_1=LSN"
 dbLoadRecords("$(SUPPORT)/motorTechnosoft/db/tmlAxis.template","PREFIX=SPARC:TML, CHANNEL_ID=MOT, CHANNEL_PREFIX=ax0, ASYN_PORT=TML, ASYN_ADDR=0, NSTEPS=200, NMICROSTEPS=256, VELO=20, VELO_MIN=0.1, VELO_MAX=50.0, ACCL=0.5, ACCL_MIN=0.01, ACCL_MAX=1.5, HAR=0.5, HVEL=10.0, JAR=1, JVEL=5, ENABLED=1, SLSP=0.8, EGU=ustep, TIMEOUT=0")
 dbLoadRecords("$(SUPPORT)/motorTechnosoft/db/tmlAxis.template","PREFIX=SPARC:TML, CHANNEL_ID=MOT, CHANNEL_PREFIX=ax1, ASYN_PORT=TML, ASYN_ADDR=0, NSTEPS=200, NMICROSTEPS=256, VELO=20, VELO_MIN=0.1, VELO_MAX=50.0, ACCL=0.5, ACCL_MIN=0.011, ACCL_MAX=1.5, HAR=0.5, HVEL=10.0, JAR=1, JVEL=5, ENABLED=1, SLSP=0.8, EGU=SPARC:TML, TIMEOUT=0")
 

tests/samples/schemas/fastVacuum.ibek.ioc.schema.json

@@ -108,7 +108,7 @@
               "type": "integer"
             }
           ],
-          "default": "{{ _global.counter(\"fvGaugeNum\", start=1) }}",
+          "default": "{{ _global.incrementor(master, start=1) }}",
           "description": "union of <class 'int'> and jinja representation of {typ}",
           "title": "Gaugenum"
         },
@@ -119,7 +119,7 @@
           "type": "string"
         },
         "mask": {
-          "default": "{{ _global.set('fvMask', _global.get('fvMask', 0) + 2**gaugeNum) }}",
+          "default": "{{ _global.incrementor(\"mask_{}\".format(master), 2, 2**gaugeNum) }}",
           "description": "mask for the channel",
           "title": "Mask",
           "type": "string"
@@ -135,22 +135,6 @@
           "description": "Gauge PV",
           "title": "Gaugepv",
           "type": "string"
-        },
-        "addr_offset": {
-          "anyOf": [
-            {
-              "description": "jinja that renders to <class 'int'>",
-              "pattern": ".*\\{\\{.*\\}\\}.*",
-              "type": "string"
-            },
-            {
-              "description": "waveform address offset for the first waveform for this channel",
-              "type": "integer"
-            }
-          ],
-          "default": "{{ (gaugeNum - 1) * master.combined_nelm }}",
-          "description": "union of <class 'int'> and jinja representation of {typ}",
-          "title": "Addr Offset"
         }
       },
       "required": [

tests/samples/schemas/ibek.support.schema.json

@@ -566,6 +566,7 @@
             {
               "type": "string"
             },
+            {},
             {
               "type": "null"
             }

tests/samples/schemas/ipac.ibek.ioc.schema.json

@@ -178,7 +178,7 @@
           "title": "Count"
         },
         "number": {
-          "default": "{{  _global.counter(\"InterruptVector\", start=192, stop=255, inc=count) }}",
+          "default": "{{  _global.incrementor(\"InterruptVector\", start=192, stop=255, increment=count) }}",
           "description": "Interrupt Vector number",
           "title": "Number",
           "type": "string"
@@ -537,7 +537,7 @@
           "title": "Interrupt Vector"
         },
         "card_id": {
-          "default": "{{ _global.counter(\"Carriers\", start=0) }}",
+          "default": "{{ _global.incrementor(\"Carriers\", start=0) }}",
           "description": "Carrier Card Identifier",
           "title": "Card Id",
           "type": "string"

tests/samples/schemas/technosoft.ibek.ioc.schema.json

@@ -41,22 +41,6 @@
           "title": "Tty",
           "type": "string"
         },
-        "numAxes": {
-          "anyOf": [
-            {
-              "description": "jinja that renders to <class 'int'>",
-              "pattern": ".*\\{\\{.*\\}\\}.*",
-              "type": "string"
-            },
-            {
-              "description": "The number of axes to create",
-              "type": "integer"
-            }
-          ],
-          "default": 1,
-          "description": "union of <class 'int'> and jinja representation of {typ}",
-          "title": "Numaxes"
-        },
         "hostid": {
           "anyOf": [
             {
@@ -73,15 +57,32 @@
           "title": "Hostid"
         },
         "CONFIG": {
-          "default": "FILE=/tmp/,NAXIS={{numAxes}},DEV_PATH={{TTY}},HOST_ID={{hostid}}",
+          "default": "DEV_PATH={{TTY}},HOST_ID={{hostid}}",
           "description": "TML Configuration",
           "title": "Config",
           "type": "string"
         },
         "axisConfiguration": {
-          "default": "{{ _global.set_var('motorTML.axisConfiguration',[]) }}",
-          "description": "collects the axis configuration from axis entities",
-          "title": "Axisconfiguration",
+          "anyOf": [
+            {
+              "description": "jinja that renders to <class 'list'>",
+              "pattern": ".*\\{\\{.*\\}\\}.*",
+              "type": "string"
+            },
+            {
+              "description": "collects the axis configuration from axis entities",
+              "items": {},
+              "type": "array"
+            }
+          ],
+          "default": [],
+          "description": "union of <class 'list'> and jinja representation of {typ}",
+          "title": "Axisconfiguration"
+        },
+        "axisNum": {
+          "default": "{{ \"axes_{}\".format(controllerName) }}",
+          "description": "A name for the axis counter (used to generate unique axis numbers)",
+          "title": "Axisnum",
           "type": "string"
         }
       },
@@ -113,16 +114,26 @@
           "title": "Entity Enabled",
           "type": "boolean"
         },
-        "num": {
-          "default": "{{ _global.counter(\"motorTML.axisCount\", start=0) }}",
-          "description": "The auto incrementing axis number",
-          "title": "Num",
-          "type": "string"
-        },
         "controller": {
           "description": "a reference to the motion controller",
           "title": "Controller"
         },
+        "num": {
+          "anyOf": [
+            {
+              "description": "jinja that renders to <class 'int'>",
+              "pattern": ".*\\{\\{.*\\}\\}.*",
+              "type": "string"
+            },
+            {
+              "description": "The axis number",
+              "type": "integer"
+            }
+          ],
+          "default": "{{ _global.incrementor(controller.axisNum) }}",
+          "description": "union of <class 'int'> and jinja representation of {typ}",
+          "title": "Num"
+        },
         "CHANNEL_PREFIX": {
           "default": "ax0",
           "description": "The axis prefix",
@@ -424,7 +435,7 @@
           "type": "string"
         },
         "axisConfiguration": {
-          "default": "{{ _global.get_var('motorTML.axisConfiguration').append(CONFIG) }}",
+          "default": "{{ controller.axisConfiguration.append(CONFIG) }}",
           "description": "Adds an axis configuration entry to the controller's list",
           "title": "Axisconfiguration",
           "type": "string"

tests/samples/schemas/utils.ibek.ioc.schema.json

@@ -24,13 +24,13 @@
           "type": "string"
         },
         "test_global_var": {
-          "default": "{{  _global.set_var(\"magic_global\", 42) }}",
+          "default": "{{  _global.set(\"magic_global\", 42) }}",
           "description": "test global variable setter",
           "title": "Test Global Var",
           "type": "string"
         },
         "get_global": {
-          "default": "{{  _global.get_var(\"magic_global\") }}",
+          "default": "{{  _global.get(\"magic_global\") }}",
           "description": "test global variable getter",
           "title": "Get Global",
           "type": "string"

tests/samples/support/epics.ibek.support.yaml

@@ -1,4 +1,4 @@
-# yaml-language-server: $schema=../schemas/bek.support.schema.json
+# yaml-language-server: $schema=../schemas/ibek.support.schema.json
 
 module: epics
 defs:
@@ -85,7 +85,7 @@ defs:
 
     post_defines:
       - name: number
-        value: '{{  _global.counter("InterruptVector", start=192, stop=255, inc=count) }}'
+        value: '{{  _global.incrementor("InterruptVector", start=192, stop=255, increment=count) }}'
         description: Interrupt Vector number
 
     env_vars: