Refactor rendering module and improve template handling

Delete `rendering.py` and establish new modular structure with `expressions.py`, `templates.py`, and `environment.py`. Update import paths accordingly and add new test cases to cover the added functionality.

.pre-commit-config.yaml

@@ -1,15 +1,15 @@
 repos:
+  - repo: https://github.com/psf/black
+    rev: 24.8.0
+    hooks:
+      - id: black
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
     rev: 'v0.6.2'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
         exclude: test.*
-  - repo: https://github.com/psf/black
-    rev: 24.8.0
-    hooks:
-      - id: black
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
     hooks:
@@ -22,6 +22,7 @@ repos:
       - id: check-shebang-scripts-are-executable
       - id: check-symlinks
       - id: check-toml
+        exclude: test.*
       - id: check-yaml
         exclude: |
           (?x)^(

project_forge/context_builder/context.py

@@ -5,7 +5,7 @@
 
 from project_forge.configurations.composition import Composition
 from project_forge.context_builder.overlays import process_overlay
-from project_forge.rendering import render_expression
+from project_forge.rendering.expressions import render_expression
 
 
 def get_starting_context() -> dict:

project_forge/context_builder/overlays.py

@@ -5,7 +5,7 @@
 
 from project_forge.configurations.composition import Overlay
 from project_forge.context_builder.questions import answer_question
-from project_forge.rendering import render_expression
+from project_forge.rendering.expressions import render_expression
 
 
 def process_overlay(overlay: Overlay, running_context: dict, question_ui: Callable) -> dict:

project_forge/context_builder/questions.py

@@ -4,7 +4,7 @@
 
 from project_forge.configurations.pattern import Choice, Question
 from project_forge.core.validators import ExprValidator
-from project_forge.rendering import render_bool_expression, render_expression
+from project_forge.rendering.expressions import render_bool_expression, render_expression
 
 
 def filter_choices(choices: List[Choice], running_context: dict) -> dict:

project_forge/core/validators.py

@@ -2,7 +2,7 @@
 
 from typing import Any, TypeVar
 
-from project_forge.rendering import render_expression
+from project_forge.rendering.expressions import render_expression
 
 T = TypeVar("T")
 

project_forge/rendering/__init__.py

@@ -0,0 +1 @@
+"""Tools for rendering templates and template strings."""

project_forge/rendering/environment.py

@@ -0,0 +1,114 @@
+"""Tools and classes for managing the Jinja2 rendering environment."""
+
+import re
+from collections import ChainMap
+from pathlib import Path
+from typing import Any, Callable, List, Optional
+
+from jinja2 import BaseLoader, Environment, TemplateNotFound, Undefined
+
+
+class InheritanceMap(ChainMap[str, Path]):
+    """Provides convenience functions for managing template inheritance."""
+
+    @property
+    def is_empty(self) -> bool:
+        """The context has only one mapping and it is empty."""
+        return len(self.maps) == 1 and len(self.maps[0]) == 0
+
+    def inheritance(self, key: str) -> List[Path]:
+        """
+        Show all the values associated with a key, from most recent to least recent.
+
+        If the maps were added in the order `{"a": Path("1")}, {"a": Path("2")}, {"a": Path("3")}`,
+        The output for `inheritance("a")` would be `[Path("3"), Path("2"), Path("1")]`.
+
+        Args:
+            key: The key to look up
+
+        Returns:
+            The values for that key with the last value first.
+        """
+        return [mapping[key] for mapping in self.maps[::-1] if key in mapping]
+
+
+class SuperUndefined(Undefined):
+    """Let calls to super() work ok."""
+
+    def __getattr__(self, name: str) -> Any:
+        """Override the superclass' __getattr__ method to handle the `jinja_pass_arg` attribute."""
+        if name.startswith("__"):
+            raise AttributeError(name)
+        return False if name == "jinja_pass_arg" else self._fail_with_undefined_error()
+
+    def __call__(self) -> str:
+        """If the undefined is called (like super()) it outputs an empty string."""
+        return ""
+
+
+class InheritanceLoader(BaseLoader):
+    """Load templates from inherited templates of the same name."""
+
+    extends_re: str = "{block_start_string}\\s*extends\\s*[\"']([^\"']+)[\"']\\s*{block_end_string}"
+
+    def __init__(self, inheritance_map: InheritanceMap):
+        self.templates = inheritance_map
+
+    def get_source(self, environment: Environment, template: str) -> tuple[str, str | None, Callable[[], bool] | None]:
+        """Load the template."""
+        # Parse the name of the template
+        bits = template.split("/")
+        index = 0
+        if len(bits) == 2 and bits[0].isdigit():
+            index = int(bits[0])
+            template_name = bits[1]
+        else:
+            template_name = template
+
+        # Get template inheritance
+        inheritance = self.templates.inheritance(template_name)
+        inheritance_len = len(inheritance)
+
+        if not inheritance:
+            raise TemplateNotFound(template_name)
+
+        # Load the template from the index
+        if index >= inheritance_len:
+            raise TemplateNotFound(template)  # Maybe this wasn't one of our customized extended paths
+
+        path = inheritance[index]
+        source = path.read_text()
+
+        # look for an `extends` tag
+        block_start_string = environment.block_start_string
+        block_end_string = environment.block_end_string
+        regex = re.compile(
+            self.extends_re.format(block_start_string=block_start_string, block_end_string=block_end_string)
+        )
+        if match := regex.search(source):
+            if index == len(inheritance) - 1:
+                # we've reached our last template, so we must remove the `extends` tag completely
+                source = source.replace(match[0], "")
+            else:
+                # rewrite the `extends` tag to reference the next item in the inheritance
+                source = source.replace(match[1], f"{index + 1}/{match[1]}")
+
+        return source, None, lambda: True
+
+
+def load_environment(template_map: Optional[InheritanceMap] = None, extensions: Optional[list] = None) -> Environment:
+    """
+    Load the Jinja2 template environment.
+
+    Args:
+        template_map: The template inheritance used to load the templates
+        extensions: A list of Jinja extensions to load into the environment
+
+    Returns:
+        The Jinja environment
+    """
+    template_map = template_map or InheritanceMap()
+    extensions = extensions or []
+    return Environment(  # NOQA: S701
+        loader=InheritanceLoader(template_map), extensions=extensions, undefined=SuperUndefined
+    )

project_forge/rendering/templates.py

@@ -0,0 +1,56 @@
+"""Management of templates."""
+
+from pathlib import Path
+from typing import Dict, List
+
+from project_forge.rendering.environment import InheritanceMap
+
+
+def catalog_templates(template_dir: Path) -> Dict[str, Path]:
+    """
+    Catalog templates into a dictionary.
+
+    This creates a mapping of a relative file name to a full path.
+
+    For a file structure like:
+
+        /path-to-templates/
+            {{ repo_name }}/
+                file1.txt
+                subdir/
+                    file2.txt
+                empty-subdir/
+
+    A call to `catalog_templates(Path("/path-to-templates"))` would return:
+
+        {
+            "{{ repo_name }}": Path("/path-to-templates/{{ repo_name }}"),
+            "{{ repo_name }}/file1.txt": Path("/path-to-templates/{{ repo_name }}/file1.txt"),
+            "{{ repo_name }}/subdir": Path("/path-to-templates/{{ repo_name }}/subdir"),
+            "{{ repo_name }}/subdir/file2.txt": Path("/path-to-templates/{{ repo_name }}/subdir/file2.txt"),
+            "{{ repo_name }}/empty-subdir": Path("/path-to-templates/{{ repo_name }}/empty-subdir"),
+        }
+
+    Args:
+        template_dir: The directory to catalog
+
+    Returns:
+        A mapping of the relative path as a string to the full path
+    """
+    templates = {}
+    for root, dirs, files in template_dir.walk():
+        for file in files:
+            template_path = root / file
+            templates[str(template_path.relative_to(template_dir))] = template_path
+        for dir_ in dirs:
+            template_path = root / dir_
+            templates[str(template_path.relative_to(template_dir))] = template_path
+    return {key: templates[key] for key in sorted(templates)}
+
+
+def catalog_inheritance(template_paths: List[Path]) -> InheritanceMap:
+    """Create an InheritanceMap that reflects the inheritance of all the template paths."""
+    inheritance = InheritanceMap()
+    for template_path in template_paths:
+        inheritance = inheritance.new_child(catalog_templates(template_path))
+    return inheritance

tests/fixtures/python-boilerplate/{{ repo_name }}/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ repo_name }}"
+description = "{{ short_description }}"
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Natural Language :: English",
+    "Programming Language :: Python :: 3.12",
+]
+keywords = ["{{ repo_name }}"]
+dynamic = ["version"]
+license = { file = "LICENSE" }
+requires-python = ">=3.9"
+dependencies = [{% block dependencies %}{% for pkg, version in dependencies.items() %}
+    "{{ pkg }}{{ version }}",
+{%- endfor %}{% endblock dependencies %}
+]
+
+[project.optional-dependencies]
+{% block optional_dependencies %}
+dev = [{% block dev_dependencies %}{% for pkg, version in dev_requirements.items() %}
+    "{{ pkg }}{{ version }}",
+{%- endfor %}{% endblock dev_dependencies %}
+]
+test = [{% block test_dependencies %}{% for pkg, version in test_requirements.items() %}
+    "{{ pkg }}{{ version }}",
+{%- endfor %}{% endblock test_dependencies %}
+]
+docs = [{% block docs_dependencies %}{% for pkg, version in docs_requirements.items() %}
+    "{{ pkg }}{{ version }}",
+{%- endfor %}{% endblock docs_dependencies %}
+]
+
+[tool.hatch.version]
+path = "{{ repo_name }}/__init__.py"

tests/fixtures/python-package/{{ repo_name }}/pyproject.toml

@@ -0,0 +1,5 @@
+{% extends "pyproject.toml" %}
+
+{% block test_dependencies %}{{ super() }}{% for pkg, version in test_requirements.items() %}
+    "{{ pkg }}{{ version }}",
+{%- endfor %}{% endblock test_dependencies %}