From 07d6f62b2cdf87ff6221873b5b1d8e82ec6e3869 Mon Sep 17 00:00:00 2001 From: thomas-bc Date: Fri, 26 Apr 2024 15:35:49 -0700 Subject: [PATCH] Translate format strings at load-time --- .../common/loaders/ch_json_loader.py | 5 +- .../common/loaders/ch_xml_loader.py | 10 +- .../common/loaders/event_json_loader.py | 4 +- .../common/loaders/event_xml_loader.py | 16 ++- src/fprime_gds/common/loaders/json_loader.py | 34 ++--- src/fprime_gds/common/loaders/xml_loader.py | 40 ++++-- src/fprime_gds/common/utils/string_util.py | 128 ++++++++---------- .../common/utils/test_string_util.py | 115 +++++++++++++--- 8 files changed, 222 insertions(+), 130 deletions(-) diff --git a/src/fprime_gds/common/loaders/ch_json_loader.py b/src/fprime_gds/common/loaders/ch_json_loader.py index 7d8b7517..0c4cfe83 100644 --- a/src/fprime_gds/common/loaders/ch_json_loader.py +++ b/src/fprime_gds/common/loaders/ch_json_loader.py @@ -63,6 +63,9 @@ def construct_template_from_dict(self, channel_dict: dict): channel_name = channel_dict[self.NAME_FIELD].split(".")[1] channel_type = channel_dict.get("type") type_obj = self.parse_type(channel_type) + format_str = JsonLoader.preprocess_format_str( + channel_dict.get(self.FMT_STR_FIELD) + ) limit_field = channel_dict.get(self.LIMIT_FIELD) limit_low = limit_field.get(self.LIMIT_LOW) if limit_field else None @@ -81,7 +84,7 @@ def construct_template_from_dict(self, channel_dict: dict): channel_name, component_name, type_obj, - ch_fmt_str=channel_dict.get(self.FMT_STR_FIELD), + ch_fmt_str=format_str, ch_desc=channel_dict.get(self.DESC_FIELD), low_red=limit_low_red, low_orange=limit_low_orange, diff --git a/src/fprime_gds/common/loaders/ch_xml_loader.py b/src/fprime_gds/common/loaders/ch_xml_loader.py index 26a75a96..63b3dea6 100644 --- a/src/fprime_gds/common/loaders/ch_xml_loader.py +++ b/src/fprime_gds/common/loaders/ch_xml_loader.py @@ -49,15 +49,15 @@ def construct_dicts(self, path): respectively and the values are ChTemplate objects """ xml_tree = self.get_xml_tree(path) - versions = xml_tree.attrib.get("framework_version", "unknown"), xml_tree.attrib.get("project_version", "unknown") + versions = xml_tree.attrib.get( + "framework_version", "unknown" + ), xml_tree.attrib.get("project_version", "unknown") # Check if xml dict has channels section ch_section = self.get_xml_section(self.CH_SECT, xml_tree) if ch_section is None: msg = f"Xml dict did not have a {self.CH_SECT} section" - raise exceptions.GseControllerParsingException( - msg - ) + raise exceptions.GseControllerParsingException(msg) id_dict = {} name_dict = {} @@ -84,7 +84,7 @@ def construct_dicts(self, path): ch_desc = ch_dict[self.DESC_TAG] if self.FMT_STR_TAG in ch_dict: - ch_fmt_str = ch_dict[self.FMT_STR_TAG] + ch_fmt_str = XmlLoader.preprocess_format_str(ch_dict[self.FMT_STR_TAG]) # TODO we need to convert these into numbers, is this the best # way to do it? diff --git a/src/fprime_gds/common/loaders/event_json_loader.py b/src/fprime_gds/common/loaders/event_json_loader.py index 973505e0..08f793b8 100644 --- a/src/fprime_gds/common/loaders/event_json_loader.py +++ b/src/fprime_gds/common/loaders/event_json_loader.py @@ -59,7 +59,9 @@ def construct_dicts(self, path): event_id = event_dict[self.ID_TAG] event_severity = EventSeverity[event_dict[self.SEVERITY_TAG]] - event_fmt_str = event_dict.get(self.FMT_STR_TAG, "") + event_fmt_str = JsonLoader.preprocess_format_str( + event_dict.get(self.FMT_STR_TAG, "") + ) event_desc = event_dict.get(self.DESC_TAG) diff --git a/src/fprime_gds/common/loaders/event_xml_loader.py b/src/fprime_gds/common/loaders/event_xml_loader.py index 22b690fb..17e41b09 100644 --- a/src/fprime_gds/common/loaders/event_xml_loader.py +++ b/src/fprime_gds/common/loaders/event_xml_loader.py @@ -43,15 +43,15 @@ def construct_dicts(self, path): respectively and the values are ChTemplate objects """ xml_tree = self.get_xml_tree(path) - versions = xml_tree.attrib.get("framework_version", "unknown"), xml_tree.attrib.get("project_version", "unknown") + versions = xml_tree.attrib.get( + "framework_version", "unknown" + ), xml_tree.attrib.get("project_version", "unknown") # Check if xml dict has events section event_section = self.get_xml_section(self.EVENT_SECT, xml_tree) if event_section is None: msg = f"Xml dict did not have a {self.EVENT_SECT} section" - raise exceptions.GseControllerParsingException( - msg - ) + raise exceptions.GseControllerParsingException(msg) id_dict = {} name_dict = {} @@ -63,14 +63,18 @@ def construct_dicts(self, path): event_name = event_dict[self.NAME_TAG] event_id = int(event_dict[self.ID_TAG], base=16) event_severity = EventSeverity[event_dict[self.SEVERITY_TAG]] - event_fmt_str = event_dict[self.FMT_STR_TAG] + event_fmt_str = XmlLoader.preprocess_format_str( + event_dict[self.FMT_STR_TAG] + ) event_desc = None if self.DESC_TAG in event_dict: event_desc = event_dict[self.DESC_TAG] # Parse arguments - args = self.get_args_list(event, xml_tree, f"{ event_comp }::{ event_name }") + args = self.get_args_list( + event, xml_tree, f"{ event_comp }::{ event_name }" + ) event_temp = EventTemplate( event_id, diff --git a/src/fprime_gds/common/loaders/json_loader.py b/src/fprime_gds/common/loaders/json_loader.py index c7ebf0c0..90359f13 100644 --- a/src/fprime_gds/common/loaders/json_loader.py +++ b/src/fprime_gds/common/loaders/json_loader.py @@ -23,29 +23,16 @@ ) from fprime.common.models.serialize.serializable_type import SerializableType from fprime.common.models.serialize.string_type import StringType -from fprime.common.models.serialize.type_base import DictionaryType, BaseType +from fprime.common.models.serialize.type_base import BaseType +from typing import Optional # Custom Python Modules from . import dict_loader import json +from fprime_gds.common.utils.string_util import preprocess_fpp_format_str -# FORMAT_STR_MAP = { -# "U8": "%u", -# "I8": "%d", -# "U16": "%u", -# "I16": "%d", -# "U32": "%u", -# "I32": "%d", -# "U64": "%lu", -# "I64": "%ld", -# "F32": "%g", -# "F64": "%g", -# "bool": "%s", -# "string": "%s", -# "ENUM": "%d", -# } PRIMITIVE_TYPE_MAP = { "I8": I8Type, @@ -172,3 +159,18 @@ def parse_type(self, type_dict: dict) -> BaseType: raise ValueError( f"Channel entry in dictionary has unknown type {str(type_dict)}" ) + + @staticmethod + def preprocess_format_str(format_str: Optional[str]) -> str: + """Preprocess format strings before using them in Python format function + Internally, this converts FPP-style format strings to Python-style format strings + + Args: + format_str (str): FPP-style format string + + Returns: + str: Python-style format string + """ + if format_str is None: + return None + return preprocess_fpp_format_str(format_str) diff --git a/src/fprime_gds/common/loaders/xml_loader.py b/src/fprime_gds/common/loaders/xml_loader.py index 0fdc5259..8bff5365 100644 --- a/src/fprime_gds/common/loaders/xml_loader.py +++ b/src/fprime_gds/common/loaders/xml_loader.py @@ -13,6 +13,7 @@ @bug No known bugs """ + import os from fprime.common.models.serialize.array_type import ArrayType @@ -34,6 +35,7 @@ from fprime.common.models.serialize.string_type import StringType from lxml import etree +from fprime_gds.common.utils.string_util import preprocess_c_style_format_str from fprime_gds.common.data_types import exceptions from fprime_gds.version import ( MAXIMUM_SUPPORTED_FRAMEWORK_VERSION, @@ -258,18 +260,21 @@ def get_serializable_type(self, type_name, xml_obj): members = [] for memb in memb_section: name = memb.get(self.SER_MEMB_NAME_TAG) - fmt_str = memb.get(self.SER_MEMB_FMT_STR_TAG) + fmt_str = XmlLoader.preprocess_format_str( + memb.get(self.SER_MEMB_FMT_STR_TAG) + ) desc = memb.get(self.SER_MEMB_DESC_TAG) memb_type_name = memb.get(self.SER_MEMB_TYPE_TAG) memb_size = memb.get(self.SER_MEMB_SIZE_TAG) type_obj = self.parse_type(memb_type_name, memb, xml_obj) # memb_size is not None for member array - if(memb_size is not None): + if memb_size is not None: type_obj = ArrayType.construct_type( f"Array_{type_obj.__name__}_{memb_size}", type_obj, int(memb_size), - fmt_str) + fmt_str, + ) members.append((name, type_obj, fmt_str, desc)) @@ -319,10 +324,14 @@ def get_array_type(self, type_name, xml_obj): # Make config arr_type = arr_memb.get(self.ARR_TYPE_TAG) type_obj = self.parse_type(arr_type, arr_memb, xml_obj) - arr_format = arr_memb.get(self.ARR_FORMAT_TAG) + arr_format = XmlLoader.preprocess_format_str( + arr_memb.get(self.ARR_FORMAT_TAG) + ) arr_size = arr_memb.get(self.ARR_SIZE_TAG) - arr_obj = ArrayType.construct_type(type_name, type_obj, int(arr_size), arr_format) + arr_obj = ArrayType.construct_type( + type_name, type_obj, int(arr_size), arr_format + ) self.array_types[type_name] = arr_obj return arr_obj @@ -372,7 +381,9 @@ def parse_type(self, type_name, xml_item, xml_tree, context=None): return BoolType if type_name == "string": if self.STR_LEN_TAG not in xml_item.attrib: - print(f"Trying to parse string type, but found {self.STR_LEN_TAG} field") + print( + f"Trying to parse string type, but found {self.STR_LEN_TAG} field" + ) return None max_length = int(xml_item.get(self.STR_LEN_TAG, 0)) name = f"{context or ''}::{xml_item.get(self.ARG_NAME_TAG)}String" @@ -394,6 +405,17 @@ def parse_type(self, type_name, xml_item, xml_tree, context=None): # Abandon all hope msg = f"Could not find type {type_name}" - raise exceptions.GseControllerParsingException( - msg - ) + raise exceptions.GseControllerParsingException(msg) + + @staticmethod + def preprocess_format_str(format_str): + """Converts C-style format strings to Python-style format strings + For example "%x" -> "{:x}" or "%.2f" -> "{:.2f}" + + Args: + format_str (str): C-style format string + + Returns: + str: Python-style format string + """ + return preprocess_c_style_format_str(format_str) diff --git a/src/fprime_gds/common/utils/string_util.py b/src/fprime_gds/common/utils/string_util.py index b9d78b1a..343159df 100644 --- a/src/fprime_gds/common/utils/string_util.py +++ b/src/fprime_gds/common/utils/string_util.py @@ -7,28 +7,53 @@ Note: This function has an identical copy in fprime-gds """ +from typing import Any, Union import logging import re LOGGER = logging.getLogger("string_util_logger") -def format_string_template(format_str, given_values): - # Regular expression pattern to match format strings like "{.3f}" - pattern = r"{(\.?\d*[cdxoefg])}" +def format_string_template(template: str, value: Union[tuple, list, Any]) -> str: + """ + Function to format a string template with values. This function is a simple wrapper around the + format function. It accepts a tuple, list, or single value and passes it to the format function + + Args: + template (str): String template to be formatted + value (Union[tuple, list, Any]): Value(s) to be inserted into the template - # Replace the format string with the correct Python representation + Returns: + str: Formatted string + """ + if not isinstance(value, (tuple, list)): + value = (value,) try: - corrected_format_string = re.sub(pattern, r"{:\1}", format_str) - return corrected_format_string.format(*given_values) - except ValueError: - # TODO: This doesn't work correctly - needs rework - # Goal is to format either FPP-style of C-style strings - format_c_style_string_template(format_str, given_values) + return template.format(*value) + except (IndexError, ValueError) as e: + LOGGER.error( + f"Error formatting string template: {template} with value: {str(value)}" + ) + raise e + + +def preprocess_fpp_format_str(format_str: str) -> str: + """Preprocess a FPP-style format string and convert it to Python format string + FPP format strings are documented https://nasa.github.io/fpp/fpp-spec.html#Format-Strings + For example "{x}" -> "{:x}" or "{.2f}" -> "{:.2f}" + Args: + format_str (str): FPP-style format string -def format_c_style_string_template(format_str, given_values): - r""" + Returns: + str: Python-style format string + """ + pattern = r"{(\d*\.?\d*[cdxoefgCDXOEFG])}" + return re.sub(pattern, r"{:\1}", format_str) + + +def preprocess_c_style_format_str(format_str: str) -> str: + """ Function to convert C-string style to python format without using python interpolation Considered the following format for C-string: @@ -45,20 +70,26 @@ def format_c_style_string_template(format_str, given_values): This function will keep the flags, width, and .precision of C-string template. - It will keep f, d, x, o, and e flags and remove all other types. - Other types will be duck-typed by python - interpreter. + It will keep f, x, o, and e flags and remove all other types. + Other types will be duck-typed by python interpreter. lengths will also be removed since they are not meaningful to Python interpreter. `See: https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting` - `Regex Source: https://www.regexlib.com/REDetails.aspx?regexp_id=3363` + + For example "%x" -> "{:x}" or "%.2f" -> "{:.2f}" + + Args: + format_str (str): C-style format string + + Returns: + str: Python-style format string """ - def convert(match_obj, ignore_int): + def convert(match_obj: re.Match): if match_obj.group() is None: return match_obj - flags, width, precision, length, conversion_type = match_obj.groups() + flags, width, precision, _, conversion_type = match_obj.groups() format_template = "" if flags: format_template += f"{flags}" @@ -67,66 +98,13 @@ def convert(match_obj, ignore_int): if precision: format_template += f"{precision}" - if conversion_type: - if any( - [ - str(conversion_type).lower() == "f", - str(conversion_type).lower() == "x", - str(conversion_type).lower() == "o", - str(conversion_type).lower() == "e", - ] - ): - format_template += f"{conversion_type}" - elif all([not ignore_int, str(conversion_type).lower() == "d"]): - format_template += f"{conversion_type}" + if conversion_type and str(conversion_type).lower() in {"f", "x", "o", "e"}: + format_template += f"{conversion_type}" return "{}" if format_template == "" else "{:" + format_template + "}" - def convert_include_all(match_obj): - return convert(match_obj, ignore_int=False) - - def convert_ignore_int(match_obj): - return convert(match_obj, ignore_int=True) - - # Allowing single, list and tuple inputs - if not isinstance(given_values, (list, tuple)): - values = (given_values,) - elif isinstance(given_values, list): - values = tuple(given_values) - else: - values = given_values - - pattern = r"(?