added support for extend macro; added support for loading templates f…

…rom TTP_TEMPLATES_DIR directory

docs/source/Extend Tag/Extend Tag.rst

@@ -26,12 +26,14 @@ extended template, other tags (lookup, vars, input, output) are ignored. Nested
      - filter, comma separated list of lookup tag names to load
    * - `outputs`_
      - filter, comma separated list of output tag names to load
+   * - `macro`_
+     - name of macro function to pass template content through
 
 template
 --------
 ``template="path_string"``
 
-``path_string`` (mandatory) - OS path to template file or reference to template within TTP Templates repository in a form of ``ttp://path/to/template`` path.
+``path_string`` (mandatory) - relative current working directory or absolute OS path to template file or reference to template within TTP Templates repository in a form of ``ttp://path/to/template`` path. Alternatively, OS path to file within ``TTP_TEMPLATES_DIR`` directory, where ``TTP_TEMPLATES_DIR`` is an environment variable.
 
 **Example-1**
 
@@ -200,6 +202,28 @@ After parsing these results produced::
                       'peers': [{'asn': '65000', 'peer': '2.2.2.2'},
                                 {'asn': '65001', 'peer': '2.2.2.3'}]}}]]
 
+**Example-4**
+
+This example demonstrates how to use ``extend`` tag and ``TTP_TEMPLATES_DIR`` to load templates.
+
+Given this files structure::
+
+    C:
+    └───TTP_TEMPLATES
+        └───parse_vlans_template.txt
+
+And having ``TTP_TEMPLATES_DIR`` set to ``C:\TTP_TEMPLATES\`` value, we can use this template
+to refer to ``parse_vlans_template.txt`` file from within extend tag::
+
+    <extend template="parse_vlans_template.txt"/>
+
+Where ``parse_vlans_template.txt`` content is::
+
+    <group name="vlans.{{ vlan }}">
+    vlan {{ vlan }}
+     name {{ name }}
+    </group>
+
 inputs
 ------
 ``inputs="name1, name2, .. , nameN"``
@@ -229,3 +253,65 @@ outputs
 ``outputs="name1, name2, .. , nameN"``
 
 This filter allows to form a comma separated list of output tags to load from extended template, identified by output tag name attribute.
+
+macro
+-----
+``macro="macro_name"``
+
+Apply arbitrary Python function on template text content before embedding it into parent template.
+
+.. warning:: macro uses python ``exec`` function to parse code payload without imposing any restrictions, hence it is dangerous to
+   run templates from untrusted sources as they can have macro defined in them that can be used to execute any arbitrary code on the system.
+
+Macro function must accept single argument to hold embedded template string and must return string
+with resulted template content.
+
+**Example**
+
+In this example we define macro function to append 4 spaces to embedded template content.
+
+Template::
+
+    <macro>
+    def indent(template_text):
+        # macro to indent each line of original template with 4 space characters
+        return "\\n".join(f"    {line}" for line in template_text.splitlines())
+    </macro>
+
+    <extend template="./assets/extend_vlan.txt" macro="indent"/>
+
+Where file ``./assets/extend_vlan.txt`` content is::
+
+    <group name="vlans.{{ vlan }}">
+    vlan {{ vlan }}
+     name {{ name }}
+    </group>
+
+After passing through macro final template content will look like::
+
+    <macro>
+    def indent(data):
+        # macro to indent each line of original template with 4 space characters
+        return "\\n".join(f"    {line}" for line in data.splitlines())
+    </macro>
+
+        <group name="vlans.{{ vlan }}">
+        vlan {{ vlan }}
+         name {{ name }}
+        </group>
+
+For this sample data::
+
+    # this data indented with 4 spaces
+        vlan 1234
+         name some_vlan
+        !
+        vlan 910
+         name one_more
+        !
+
+Final template will produce results::
+
+    [
+        [{"vlans": {"1234": {"name": "some_vlan"}, "910": {"name": "one_more"}}}]
+    ]

test/pytest/assets/TTP_TEMPLATES_DIR_TEST/test_ttp_templates_dir_env_variable.txt

@@ -0,0 +1,4 @@
+<group name="vlans.{{ vlan }}">
+vlan {{ vlan }}
+ name {{ name }}
+</group>

test/pytest/test_extend_tag.py

@@ -1,4 +1,5 @@
 import sys
+import os
 
 sys.path.insert(0, "../..")
 import pprint
@@ -856,3 +857,56 @@ def test_extend_tag_groups_recursive_extend_load_several_top_groups():
 
 
 # test_extend_tag_groups_recursive_extend_load_several_top_groups()
+
+
+def test_extend_tag_from_file_with_macro():
+    template = """
+<macro>
+def indent(data):
+    # macro to indent each line of original template with 4 space characters
+    return "\\n".join(f"    {line}" for line in data.splitlines())
+</macro>
+
+<extend template="./assets/extend_vlan.txt" macro="indent"/>
+    """
+    data = """
+    vlan 1234
+     name some_vlan
+    !
+    vlan 910
+     name one_more
+    !
+    """
+    parser = ttp(data=data, template=template)
+    parser.parse()
+    res = parser.result()
+    pprint.pprint(res)
+    assert res == [
+        [{"vlans": {"1234": {"name": "some_vlan"}, "910": {"name": "one_more"}}}]
+    ]
+
+# test_extend_tag_from_file_with_macro()
+
+
+def test_ttp_templates_dir_env_variable_with_extend():
+    os.environ["TTP_TEMPLATES_DIR"] = os.path.join(
+        os.getcwd(), "assets", "TTP_TEMPLATES_DIR_TEST"
+    )
+    data = """
+vlan 1234
+ name some_vlan
+!
+vlan 910
+ name one_more
+!
+    """
+    template = """
+<extend template="test_ttp_templates_dir_env_variable.txt"/>
+    """
+    parser = ttp(data=data, template=template)
+    parser.parse()
+    res = parser.result()
+    pprint.pprint(res)
+    assert res == [[{'vlans': {'1234': {'name': 'some_vlan'}, '910': {'name': 'one_more'}}}]]
+
+# test_ttp_templates_dir_env_variable_with_extend()

test/pytest/test_misc.py

@@ -1135,3 +1135,22 @@ def worker():
 
 # test_in_threads_parsing()
 
+def test_ttp_templates_dir_env_variable():
+    os.environ["TTP_TEMPLATES_DIR"] = os.path.join(
+        os.getcwd(), "assets", "TTP_TEMPLATES_DIR_TEST"
+    )
+    data = """
+vlan 1234
+ name some_vlan
+!
+vlan 910
+ name one_more
+!
+    """
+    parser = ttp(data=data, template="test_ttp_templates_dir_env_variable.txt")
+    parser.parse()
+    res = parser.result()
+    pprint.pprint(res)
+    assert res == [[{'vlans': {'1234': {'name': 'some_vlan'}, '910': {'name': 'one_more'}}}]]
+
+# test_ttp_templates_dir_env_variable()

test/pytest/test_output_formatters.py

@@ -345,7 +345,7 @@ def test_excel_formatter_update():
     """
     # copy workbook to update it
     copyfile(
-        src="./Output/excel_out_test_excel_formatter_update_source.xlsx",
+        src="./assets/excel_out_test_excel_formatter_update_source.xlsx",
         dst="./Output/excel_out_test_excel_formatter_update.xlsx",
     )
     # run parsing
@@ -433,7 +433,7 @@ def test_excel_formatter_update_using_result_kwargs():
     """
     # copy workbook to update it
     copyfile(
-        src="./Output/excel_out_test_excel_formatter_update_source.xlsx",
+        src="./assets/excel_out_test_excel_formatter_update_source.xlsx",
         dst="./Output/excel_out_test_excel_formatter_update.xlsx",
     )
     # run parsing

ttp/ttp.py

@@ -190,7 +190,9 @@ class ttp:
     construct final results and run outputs.
 
     :param data: (obj) file object or OS path to text file or directory with text files with data to parse
-    :param template: (obj) file object or OS path to text file with template or template text string
+    :param template: (obj) one of: file object, OS path to template text file, OS path to directory with
+        templates text files, template text string, TTP Templates path (ttp://x/y/z), OS path to template
+        text file within ``TTP_TEMPLATES_DIR`` directory, where ``TTP_TEMPLATES_DIR`` is an environment variable.
     :param base_path: (str) base OS path prefix to load data from for template's inputs
     :param log_level: (str) level of logging "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
     :param log_file: (str) path where to save log file
@@ -204,6 +206,14 @@ class ttp:
         parser.parse()
         result = parser.result(format="json")
         print(result[0])
+
+    Or if ``TTP_TEMPLATES_DIR=/absolute/os/path/to/templates/dir/``::
+
+        from ttp import ttp
+        parser = ttp(data="/os/path/to/data/dir/", template="template.txt")
+        parser.parse()
+        result = parser.result(format="json")
+        print(result[0])
     """
 
     def __init__(
@@ -912,8 +922,8 @@ def __init__(
         self.macro = ttp_macro  # dictionary of macro name to function mapping
         self.results_method = "per_input"  # how to join results
         self.macro_text = (
-            []
-        )  # list to contain macro functions text to transfer into other processes
+            set()
+        )  # set to contain macro functions text to transfer into other processes
         self.filters = filters  # list that contains names of child templates to extract
         self.__doc__ = ""  # string to contain template doc/description
         self.template = template_text  # attribute to store template text
@@ -1153,7 +1163,7 @@ def parse_macro(self, element):
             )
             self.macro.update(funcs)
             # save macro text to be able to restore macro functions within another process
-            self.macro_text.append(element.text)
+            self.macro_text.add(element.text)
         except SyntaxError as e:
             log.error(
                 "template.parse_macro: syntax error, failed to load macro: \n{},\nError: {}".format(
@@ -1310,6 +1320,11 @@ def handle_extend(self, template_text=None, template_ET=None, top=True):
         # load template string to XML etree object
         if template_text:
             template_ET = self.construct_etree(template_text)
+        # load macro functions from top template into self.macro dict
+        if top == True:
+            for child in template_ET:
+                if child.tag == "macro":
+                    self.parse_macro(child)
         # recursively iterate over etree handling extend tags
         for index, child in enumerate(template_ET):
             if child.tag == "extend":
@@ -1321,7 +1336,12 @@ def handle_extend(self, template_text=None, template_ET=None, top=True):
                             path_to_template
                         )
                     )
-                extend_ET = self.construct_etree(content[0][1])
+                # run extend tag macro function if any
+                template_content = content[0][1]
+                if child.attrib.get("macro") in self.macro:
+                    template_content = self.macro[child.attrib["macro"]](content[0][1])
+                # construct etree out of template content
+                extend_ET = self.construct_etree(template_content)
                 # if extended template has _anonymous_ group as the only child, use only its text
                 if (
                     len(extend_ET) == 1

ttp/utils/loaders.py

@@ -4,6 +4,25 @@
 log = logging.getLogger(__name__)
 
 
+def _load_template_file(filepath, read):
+    """
+    Helper function to load template file content or return path to template
+    """
+    if read:
+        try:
+            if _ttp_["python_major_version"] == 2:
+                with open(filepath, "r") as file_obj:
+                    return [("text_data", file_obj.read())]
+            with open(filepath, "r", encoding="utf-8") as file_obj:
+                return [("text_data", file_obj.read())]
+        except UnicodeDecodeError:
+            log.warning(
+                'ttp_utils.load_files: Unicode read error, file "{}"'.format(filepath)
+            )
+    else:
+        return [("file_name", filepath)]
+
+
 def load_files(path, extensions=None, filters=None, read=False):
     """
     Method to load files from path, and filter file names with
@@ -20,6 +39,7 @@ def load_files(path, extensions=None, filters=None, read=False):
     extensions = extensions or []
     filters = filters or []
     files = []
+    ttp_templates_dir = os.getenv("TTP_TEMPLATES_DIR")
     # need to use path[:5000] cause if path is actually text of the template
     # and has length more then X symbols, os.path will choke with "path too long"
     # error, hence the safe-assumption that no os path exists longer then 5000 symbols
@@ -37,21 +57,14 @@ def load_files(path, extensions=None, filters=None, read=False):
         from ttp_templates import get_template
 
         return [("text_data", get_template(path=path.replace("ttp://", "")))]
-    # check if path is a path to file:
+    # check if path is a path to file
     elif os.path.isfile(path[:5000]):
-        if read:
-            try:
-                if _ttp_["python_major_version"] == 2:
-                    with open(path, "r") as file_obj:
-                        return [("text_data", file_obj.read())]
-                with open(path, "r", encoding="utf-8") as file_obj:
-                    return [("text_data", file_obj.read())]
-            except UnicodeDecodeError:
-                log.warning(
-                    'ttp_utils.load_files: Unicode read error, file "{}"'.format(path)
-                )
-        else:
-            return [("file_name", path)]
+        return _load_template_file(path, read)
+    # check if path is a path to file within ttp_templates_dir
+    elif ttp_templates_dir and os.path.isfile(
+        os.path.join(ttp_templates_dir, path[:5000])
+    ):
+        return _load_template_file(os.path.join(ttp_templates_dir, path[:5000]), read)
     # check if path is a directory:
     elif os.path.isdir(path[0:5000]):
         from re import search as re_search