CI(pyavd): Recompile schemas and templates on change (#4637)

.github/requirements-ci.txt

@@ -3,3 +3,5 @@
 ./python-avd
 # The -r path is relative to this file.
 -r ../ansible_collections/arista/avd/requirements.txt
+# Needed for molecule
+jsonschema-rs>=0.24

.gitignore

@@ -48,3 +48,7 @@ ansible_collections/arista/avd/tests/.mypy_cache/
 
 # Temporary files creating during CI workflow
 /tmp-requirements-minimum.txt
+
+# Hash files created when running pyavd from source
+python-avd/pyavd/*/j2templates/.hash
+python-avd/pyavd/*/schema/schema_fragments/.hash

.pre-commit-config.yaml

@@ -198,11 +198,11 @@ repos:
 
       - id: schemas
         name: Build AVD schemas and documentation.
-        entry: sh -c 'exec python-avd/scripts/build-schemas.py'
+        entry: sh -c 'exec python-avd/scripts/build_schemas.py'
         language: python
         files: python-avd/pyavd/[a-z_]+/schema
         pass_filenames: false
-        additional_dependencies: ['deepmerge>=1.1.0', 'PyYAML>=6.0.0', 'pydantic>=2.3.0', 'jsonschema-rs', 'referencing>=0.35.0']
+        additional_dependencies: ['deepmerge>=1.1.0', 'PyYAML>=6.0.0', 'pydantic>=2.3.0', 'jsonschema-rs>=0.24', 'referencing>=0.35.0']
 
       - id: templates
         name: Precompile Jinja2 templates

ansible_collections/arista/avd/docs/contribution/development-tooling.md

@@ -80,11 +80,23 @@ pip3 install -r ansible_collections/arista/avd/requirements-dev.txt -r ansible_c
     You may be required to set `ansible_python_interpreter` in your Ansible inventory.
     For more information consult with the [Ansible documentation](https://docs.ansible.com/ansible/latest/reference_appendices/python_3_support.html#using-python-3-on-the-managed-machines-with-commands-and-playbooks).
 
+## Running from source
+
+Ansible-AVD is able to detect when it is running from source, and will prepend the Python Path with the path to the PyAVD source. This ensures that running Ansible-AVD from source will also use PyAVD from the same source.
+
+The modification of the Python Path is logged as a warning in the Ansible output.
+
+TODO: Add picture.
+
+When running from source, the `verify_requirements` action will check if any schema fragment or templates has changed locally and if so, will recompile on the fly either or both as
+required for `eos_cli_config_gen` and `eos_designs`, allowing a seamless development workflow while using Ansible.
+When using pyavd, it is required to run pre-commit to achieve the same.
+
 ## Pre-commit
 
 - [pre-commit](https://github.com/aristanetworks/avd/blob/devel/.pre-commit-config.yaml) can run standard hooks on every commit to automatically point out issues in code such as missing semicolons, trailing whitespace, and debug statements.
 - Pointing these issues out before code review allows a code reviewer to focus on the architecture of a change while not wasting time with trivial style nitpicks.
-- Additionally, the AVD project leverages pre-commit hooks to build and update the AVD schemas and documentation artifacts.
+- Additionally, the AVD project leverages pre-commit hooks to build and update the AVD schemas, templates and documentation artifacts.
 
 ### Install pre-commit hook
 

ansible_collections/arista/avd/plugins/__init__.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2023-2024 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the LICENSE file.
+from __future__ import annotations
+
+from contextlib import suppress
+from pathlib import Path
+
+PYTHON_AVD_PATH = Path(__file__).parents[4] / "python-avd"
+RUNNING_FROM_SOURCE_PATH = PYTHON_AVD_PATH / "pyavd/running_from_src.txt"
+RUNNING_FROM_SOURCE = RUNNING_FROM_SOURCE_PATH.exists()
+
+if RUNNING_FROM_SOURCE:
+    import sys
+
+    # TODO: @gmuloc - once proper logging has been implemented for the collection, replace this with a log statement.
+    # Note that we can't output anything to stdout or stderr in normal mode or it breaks ansible-sanity
+    with suppress(ImportError):
+        from ansible.utils.display import Display
+
+        display = Display()
+
+        display.v(f"AVD detected it is running from source, prepending the path to the source of pyavd '{PYTHON_AVD_PATH}' to PATH to use it.")
+
+    sys.path = [str(PYTHON_AVD_PATH), *sys.path]

ansible_collections/arista/avd/plugins/action/verify_requirements.py

@@ -16,6 +16,8 @@
 from ansible.plugins.action import ActionBase, display
 from ansible.utils.collection_loader._collection_finder import _get_collection_metadata
 
+from ansible_collections.arista.avd.plugins import PYTHON_AVD_PATH, RUNNING_FROM_SOURCE
+
 try:
     # Relying on packaging installed by ansible
     from packaging.requirements import InvalidRequirement, Requirement
@@ -25,6 +27,16 @@
 except ImportError:
     HAS_PACKAGING = False
 
+try:
+    from ansible_collections.arista.avd.plugins.plugin_utils.utils.init_logging import init_pyavd_logging
+
+    HAS_INIT_PYAVD_LOGGING = True
+except ImportError:
+    HAS_INIT_PYAVD_LOGGING = False
+
+if HAS_INIT_PYAVD_LOGGING:
+    init_pyavd_logging()
+
 MIN_PYTHON_SUPPORTED_VERSION = (3, 10)
 DEPRECATE_MIN_PYTHON_SUPPORTED_VERSION = False
 
@@ -83,6 +95,7 @@ def _validate_python_requirements(requirements: list, info: dict) -> bool:
     return False if any python requirement is not valid
     """
     valid = True
+    pyavd_from_source = False
 
     requirements_dict = {
         "not_found": {},
@@ -100,6 +113,10 @@ def _validate_python_requirements(requirements: list, info: dict) -> bool:
             msg = f"Wrong format for requirement {raw_req}"
             raise AnsibleActionFail(msg) from exc
 
+        if RUNNING_FROM_SOURCE and req.name == "pyavd":
+            pyavd_from_source = True
+            display.vvv("AVD is running from source, checking pyavd version.", "Verify Requirements")
+
         try:
             installed_version = version(req.name)
             display.vvv(f"Found {req.name} {installed_version} installed!", "Verify Requirements")
@@ -109,8 +126,8 @@ def _validate_python_requirements(requirements: list, info: dict) -> bool:
             potential_dists = Distribution.discover(name=req.name)
             detected_versions = [dist.version for dist in potential_dists]
             valid_versions = [version for version in detected_versions if req.specifier.contains(version)]
-            if len(detected_versions) > 1:
-                display.v(f"Found {req.name} {detected_versions} metadata - this could mean legacy dist-info files are present in your site-packages folder")
+            if len(detected_versions) > 1 and not pyavd_from_source:
+                display.v(f"Found {req.name} {detected_versions} metadata - this could mean legacy dist-info files are present in your site-packages folder.")
         except PackageNotFoundError:
             requirements_dict["not_found"][req.name] = {
                 "installed": None,
@@ -122,13 +139,13 @@ def _validate_python_requirements(requirements: list, info: dict) -> bool:
 
         if req.specifier.contains(installed_version):
             requirements_dict["valid"][req.name] = {
-                "installed": installed_version,
+                "installed": f"{installed_version}{' (running from source)' if pyavd_from_source else ''}",
                 "required_version": str(req.specifier) if len(req.specifier) > 0 else None,
             }
         elif len(valid_versions) > 0:
             # More than one dist found and at least one was matching - output a warning
             requirements_dict["valid"][req.name] = {
-                "installed": installed_version,
+                "installed": f"{installed_version}{' (running from source)' if pyavd_from_source else ''}",
                 "detected_versions": detected_versions,
                 "valid_versions": valid_versions,
                 "required_version": str(req.specifier) if len(req.specifier) > 0 else None,
@@ -150,18 +167,20 @@ def _validate_python_requirements(requirements: list, info: dict) -> bool:
                 wrap_text=False,
             )
             requirements_dict["mismatched"][req.name] = {
-                "installed": installed_version,
+                "installed": f"{installed_version}{' (running from source)' if pyavd_from_source else ''}",
                 "detected_versions": detected_versions,
                 "valid_versions": None,
                 "required_version": str(req.specifier) if len(req.specifier) > 0 else None,
             }
+            display.error(f"Python library '{req.name}' version running {installed_version} - requirement is {req!s}", wrap_text=False)
         else:
             display.error(f"Python library '{req.name}' version running {installed_version} - requirement is {req!s}", wrap_text=False)
             requirements_dict["mismatched"][req.name] = {
-                "installed": installed_version,
+                "installed": f"{installed_version}{' (running from source)' if pyavd_from_source else ''}",
                 "required_version": str(req.specifier) if len(req.specifier) > 0 else None,
             }
             valid = False
+        pyavd_from_source = False
 
     info["python_requirements"] = requirements_dict
     return valid
@@ -324,6 +343,18 @@ def _get_running_collection_version(running_collection_name: str, result: dict)
     }
 
 
+def check_running_from_source() -> None:
+    """Check if running from sources, if so recompile schemas and templates as needed."""
+    if not RUNNING_FROM_SOURCE:
+        return
+    # if running from source, path to pyavd and schema_tools has already been prepended to Python Path
+    from schema_tools.check_schemas import check_schemas
+    from schema_tools.compile_templates import check_templates
+
+    check_schemas()
+    check_templates()
+
+
 class ActionModule(ActionBase):
     def run(self, tmp: Any = None, task_vars: dict | None = None) -> dict:
         if task_vars is None:
@@ -362,7 +393,11 @@ def run(self, tmp: Any = None, task_vars: dict | None = None) -> dict:
 
         _get_running_collection_version(running_collection_name, info["ansible"])
 
+        check_running_from_source()
+
         display.display(f"AVD version {info['ansible']['collection']['version']}", color=C.COLOR_OK)
+        if RUNNING_FROM_SOURCE:
+            display.display(f"AVD is running from source using PyAVD at '{PYTHON_AVD_PATH}'", color=C.COLOR_OK)
         if display.verbosity < 1:
             display.display("Use -v for details.")
 

ansible_collections/arista/avd/plugins/plugin_utils/utils/init_logging.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2023-2024 Arista Networks, Inc.
+# Use of this source code is governed by the Apache License 2.0
+# that can be found in the LICENSE file.
+from __future__ import annotations
+
+import logging
+
+from ansible.utils.display import Display
+
+from .python_to_ansible_logging_handler import PythonToAnsibleHandler
+
+
+def init_pyavd_logging() -> None:
+    """
+    Specify logger parameters for pyavd and schematools.
+
+    The verbosity depends on the verbosity level passed to Ansible
+    - Ansible verbosity 0 translate to a level of warning
+    - Ansible verbosity 1 to 2 (-v to -vv) translates to a level of info
+    - Ansible verbosity 3 and above translates to a level of debug
+    """
+    pyavd_logger = logging.getLogger("pyavd")
+    schema_tools_logger = logging.getLogger("schema_tools")
+    # Avoid duplicate logs
+    pyavd_logger.propagate = False
+    schema_tools_logger.propagate = False
+
+    pyavd_handler = PythonToAnsibleHandler(None)
+    pyavd_formatter = logging.Formatter("[pyavd] - %(message)s")
+    pyavd_handler.setFormatter(pyavd_formatter)
+
+    pyavd_logger.addHandler(pyavd_handler)
+    schema_tools_logger.addHandler(pyavd_handler)
+
+    verbosity = Display().verbosity
+    if verbosity >= 3:
+        pyavd_logger.setLevel(logging.DEBUG)
+        schema_tools_logger.setLevel(logging.DEBUG)
+    elif verbosity > 0:
+        pyavd_logger.setLevel(logging.INFO)
+        schema_tools_logger.setLevel(logging.INFO)
+    else:
+        pyavd_logger.setLevel(logging.WARNING)
+        schema_tools_logger.setLevel(logging.WARNING)

ansible_collections/arista/avd/plugins/plugin_utils/utils/python_to_ansible_logging_handler.py

@@ -7,11 +7,11 @@
 from logging import Filter, Handler
 from typing import TYPE_CHECKING
 
+from ansible.utils.display import Display
+
 if TYPE_CHECKING:
     from logging import LogRecord
 
-    from ansible.utils.display import Display
-
 
 class PythonToAnsibleHandler(Handler):
     """
@@ -25,20 +25,27 @@ class PythonToAnsibleHandler(Handler):
     * send DEBUG logs to display.vvv which can be visualized when running a playbook with `-vvv`
     """
 
-    def __init__(self, result: dict, display: Display) -> None:
+    def __init__(self, result: dict | None, display: Display | None = None) -> None:
         super().__init__()
-        self.display = display
+        # If no display object was given, retrieve the Singleton
+        self.display = display or Display()
         self.result = result
 
     def emit(self, record: LogRecord) -> None:
         """Custom emit function that reads the message level."""
         message = self._format_msg(record)
         if record.levelno in [logging.CRITICAL, logging.ERROR]:
-            self.result.setdefault("stderr_lines", []).append(message)
-            self.result["stderr"] = self.result.setdefault("stderr", "") + f"{message!s}\n"
-            self.result["failed"] = True
+            if self.result is not None:
+                self.result.setdefault("stderr_lines", []).append(message)
+                self.result["stderr"] = self.result.setdefault("stderr", "") + f"{message!s}\n"
+                self.result["failed"] = True
+            else:
+                self.display.error(str(message))
         elif record.levelno in [logging.WARNING, logging.WARNING]:
-            self.result.setdefault("warnings", []).append(message)
+            if self.result is not None:
+                self.result.setdefault("warnings", []).append(message)
+            else:
+                self.display.warning(str(message))
         elif record.levelno == logging.INFO:
             self.display.v(str(message))
         elif record.levelno == logging.DEBUG:

ansible_collections/arista/avd/requirements-dev.txt

@@ -11,7 +11,7 @@ codespell>=2.2.6
 deepmerge>=1.1.0
 docker>=7.1.0
 identify>=1.4.20
-jsonschema-rs
+jsonschema-rs>=0.24
 molecule>=6.0
 molecule-plugins[docker]>=23.4.0
 pydantic>=2.3.0

python-avd/Makefile

@@ -30,7 +30,7 @@ publish: ## Publish pyavd package to PyPI (build first)
 
 .PHONY: compile-schemas
 compile-schemas:
-	$(SCRIPTS_DIR)/build-schemas.py
+	$(SCRIPTS_DIR)/build_schemas.py
 
 .PHONY: compile-templates
 compile-templates:

python-avd/pyavd/templater.py

@@ -3,6 +3,7 @@
 # that can be found in the LICENSE file.
 from __future__ import annotations
 
+import logging
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -14,6 +15,8 @@
     import os
     from collections.abc import Sequence
 
+LOGGER = logging.getLogger(__name__)
+
 
 class Undefined(StrictUndefined):
     """
@@ -126,7 +129,7 @@ def compile_templates_in_paths(self, precompiled_templates_path: str, searchpath
         self.environment.loader = ExtensionFileSystemLoader(searchpaths)
         self.environment.compile_templates(
             zip=None,
-            log_function=print,
+            log_function=LOGGER.debug,
             target=precompiled_templates_path,
             ignore_errors=False,
         )

python-avd/pyproject.toml

@@ -51,7 +51,7 @@ requires = [
     "setuptools>=66.1",
     "wheel",
     "jinja2>=3.0",
-    "jsonschema-rs",
+    "jsonschema-rs>=0.24",
     "referencing>=0.35.0",
     "deepmerge>=1.1.0",
     "pyyaml>=6.0.0",