From 76773517dad8fa5c6c1b94e44c6f19204d8387b9 Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Wed, 20 Sep 2023 13:43:06 -0700 Subject: [PATCH 01/11] Add documentation for `modify` command. --- src/web/BL_Python/web/scaffolding/__main__.py | 2 +- .../scaffolding/templates/base/README.md.j2 | 1 - src/web/README.md | 18 +++++++++++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/web/BL_Python/web/scaffolding/__main__.py b/src/web/BL_Python/web/scaffolding/__main__.py index c66d5669..c7e49be1 100644 --- a/src/web/BL_Python/web/scaffolding/__main__.py +++ b/src/web/BL_Python/web/scaffolding/__main__.py @@ -97,7 +97,7 @@ def _parse_args(): metavar="output directory", dest="output_directory", type=str, - help="The output directory. The default is a new directory sharing the name of the application.", + help="The output directory. The default is a directory sharing the name of the application.", ) modify_parser.set_defaults(mode_executor=partial(_run_modify_mode), mode="modify") diff --git a/src/web/BL_Python/web/scaffolding/templates/base/README.md.j2 b/src/web/BL_Python/web/scaffolding/templates/base/README.md.j2 index 9922161b..00cd0317 100644 --- a/src/web/BL_Python/web/scaffolding/templates/base/README.md.j2 +++ b/src/web/BL_Python/web/scaffolding/templates/base/README.md.j2 @@ -30,7 +30,6 @@ To make it easy to get started, a new Blueprint can be added with the scaffolder Existing Blueprints can be freely changed. Endpoints can be added, removed, and altered in any way necessary. The only requirement is that one variable with the suffix `_blueprint` whose value is a `Blueprint` instance must exist. ### Using OpenAPI - ... ## Configuration diff --git a/src/web/README.md b/src/web/README.md index b4b5931e..08eaa3be 100644 --- a/src/web/README.md +++ b/src/web/README.md @@ -26,7 +26,9 @@ The command used is `bl-python-scaffold create`. Please run `bl-python-scaffold **Note** The command `bl-python-scaffold` has two modes: `create` and `modify`. The former is used to create a new application, while the latter is used to modify an existing one.
- Scaffold help + Scaffold "create" help + +These options are for the `bl-python-scaffold create` command. | Option | Explanation | Required? | | --- | --- | --- | @@ -38,6 +40,20 @@ The command used is `bl-python-scaffold create`. Please run `bl-python-scaffold | `-o ` | Store the new application in a directory other than one that matches the application name. | No |
+
+ Scaffold "modify" help + +These options are for the `bl-python-scaffold modify` command. + +| Option | Explanation | Required? | +| --- | --- | --- | +| `-h` | Show the tool help text. | No | +| `-n ` | This is the name of your application. It is the name Flask will use to start up, and also acts as a default value for other options when they are not specified when running this tool. | Yes | +| `-e ` | An endpoint to create. By default, an endpoint sharing the name of your application is created. If `-e` is specified even once, the default is _not_ created. This option can be specified more than once to create multiple endpoints. | No | +| `-o ` | Modify the application in a directory other than one that matches the application name. | No | + +
+

To create an application with a single API endpoint, run `bl-python-scaffold create -n ` where `` is replaced with the desired name of your application. By default, the scaffolder will output into a directory matching the name of your application. **Existing files will be overwritten.** From 42ee0b554ac362975283862a27a3f55be6a65d8c Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Wed, 20 Sep 2023 15:38:11 -0700 Subject: [PATCH 02/11] Add hooks to scaffolder so modules can configure behavior. --- src/web/BL_Python/web/scaffolding/__main__.py | 8 +- .../BL_Python/web/scaffolding/scaffolder.py | 76 ++++++++++++++----- .../templates/base/pyproject.toml.j2 | 2 +- .../base/{{application_name}}/__init__.py.j2 | 2 +- .../templates/basic/config.toml.j2 | 4 +- .../templates/openapi/config.toml.j2 | 4 +- .../modules/database/__hook__.py | 19 +++++ .../__init__.py.j2} | 0 8 files changed, 87 insertions(+), 28 deletions(-) create mode 100644 src/web/BL_Python/web/scaffolding/templates/optional/{{application_name}}/modules/database/__hook__.py rename src/web/BL_Python/web/scaffolding/templates/optional/{{application_name}}/modules/{database.py.j2 => database/__init__.py.j2} (100%) diff --git a/src/web/BL_Python/web/scaffolding/__main__.py b/src/web/BL_Python/web/scaffolding/__main__.py index c7e49be1..65f7ffab 100644 --- a/src/web/BL_Python/web/scaffolding/__main__.py +++ b/src/web/BL_Python/web/scaffolding/__main__.py @@ -121,7 +121,9 @@ def _parse_args(): def _run_create_mode(args: Namespace): log.info("Running create mode.") - log.info(f"Creating new application from {args.template_type} template ...") + log.info( + f'Creating new application "{args.name}" from {args.template_type} template ...' + ) # TODO consider pulling licenses from GitHub # https://docs.github.com/en/rest/licenses/licenses?apiVersion=2022-11-28#get-all-commonly-used-licenses @@ -164,10 +166,10 @@ def _run_modify_mode(args: Namespace): def scaffold(): args = _parse_args() - log.info( + print( f'Scaffolding application named "{args.name}" under directory `{Path(args.output_directory).absolute()}`.' ) args.mode_executor(args) - log.info("Done.") + print("Done.") diff --git a/src/web/BL_Python/web/scaffolding/scaffolder.py b/src/web/BL_Python/web/scaffolding/scaffolder.py index 7759a642..9cba3cb1 100644 --- a/src/web/BL_Python/web/scaffolding/scaffolder.py +++ b/src/web/BL_Python/web/scaffolding/scaffolder.py @@ -1,10 +1,12 @@ import logging -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field +from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path from typing import Any, cast from BL_Python.programming.collections.dict import merge from jinja2 import BaseLoader, Environment, PackageLoader, Template +from pkg_resources import ResourceManager, get_provider @dataclass @@ -25,6 +27,7 @@ class ScaffoldConfig: application_name: str template_type: str | None = None modules: list[ScaffoldModule] | None = None + module: dict[str, Any] = field(default_factory=dict) endpoints: list[ScaffoldEndpoint] | None = None mode: str = "create" @@ -76,6 +79,7 @@ def _render_template( self, template_name: str, template_environment: Environment, + template_directory_prefix: str = "", template_config: dict[str, Any] | None = None, template: Template | None = None, overwrite_existing_files: bool = True, @@ -85,6 +89,7 @@ def _render_template( template_output_path = Path( self._config.output_directory, + template_directory_prefix, self._render_template_string(template_name).replace(".j2", ""), ) @@ -109,13 +114,45 @@ def _render_template( template.stream(**template_config).dump(str(template_output_path)) def _scaffold_directory( - self, env: Environment, overwrite_existing_files: bool = True + self, + env: Environment, + template_directory_prefix: str = "", + overwrite_existing_files: bool = True, ): # render the base templates - for template_name in cast(list[str], env.list_templates()): + for template_name in cast(list[str], env.list_templates(extensions=["j2"])): self._render_template( - template_name, env, overwrite_existing_files=overwrite_existing_files + template_name, + env, + template_directory_prefix=template_directory_prefix, + overwrite_existing_files=overwrite_existing_files, + ) + + _manager = ResourceManager() + _provider = get_provider("BL_Python.web") + + def _execute_module_hooks(self, module_template_directory: str): + module_hook_path = Path(module_template_directory, "__hook__.py") + if self._provider.has_resource(str(module_hook_path)): + # load the module from its path + # and execute it + spec = spec_from_file_location( + "__hook__", + self._provider.get_resource_filename( + self._manager, str(module_hook_path) + ), ) + if spec is None or spec.loader is None: + raise Exception( + f"Module cannot be created from path {module_hook_path}" + ) + module = module_from_spec(spec) + spec.loader.exec_module(module) + + for module_name, module_var in vars(module).items(): + if not module_name.startswith("on_"): + continue + module_var(self._config_dict, self._log) def _scaffold_modules( self, env: Environment, overwrite_existing_files: bool = True @@ -126,20 +163,18 @@ def _scaffold_modules( # render optional module templates for module in self._config.modules: - template = env.get_template( - f"{{{{application_name}}}}/modules/{module.module_name}.py.j2" - ) + module_template_directory = f"scaffolding/templates/optional/{{{{application_name}}}}/modules/{module.module_name}" + self._execute_module_hooks(module_template_directory) - if template.name is None: - self._log.error( - f"Could not find template `{{{{application_name}}}}/modules/{module.module_name}.py.j2`." - ) - continue + module_env = Environment( + trim_blocks=True, + lstrip_blocks=True, + loader=PackageLoader("BL_Python.web", str(module_template_directory)), + ) - self._render_template( - template.name, - env, - template=template, + self._scaffold_directory( + module_env, + template_directory_prefix=f"{self._config.application_name}/modules/{module.module_name}", overwrite_existing_files=overwrite_existing_files, ) @@ -181,8 +216,8 @@ def _scaffold_endpoints( self._render_template( rendered_template_name, env, - template_config, - template, + template_config=template_config, + template=template, overwrite_existing_files=overwrite_existing_files, ) @@ -206,10 +241,13 @@ def scaffold(self): ) if self._config.mode == "create": + # scaffold modules first so they can alter the config if necessary + self._scaffold_modules(optional_env) + self._scaffold_directory(base_env) # template type should go after base so it can override any base templates self._scaffold_directory(template_type_env) - self._scaffold_modules(optional_env) + self._scaffold_endpoints(optional_env) # other mode is "modify" else: diff --git a/src/web/BL_Python/web/scaffolding/templates/base/pyproject.toml.j2 b/src/web/BL_Python/web/scaffolding/templates/base/pyproject.toml.j2 index a49c9dad..2e5ee1cc 100644 --- a/src/web/BL_Python/web/scaffolding/templates/base/pyproject.toml.j2 +++ b/src/web/BL_Python/web/scaffolding/templates/base/pyproject.toml.j2 @@ -10,7 +10,7 @@ version = "0.0.1" dependencies = [ "bl-python-programming@ git+ssh://git@github.com/uclahs-cds/private-BL-python-libraries.git@main#subdirectory=src/programming", "bl-python-web@ git+ssh://git@github.com/uclahs-cds/private-BL-python-libraries.git@main#subdirectory=src/web", -{% if {'module_name': 'database'} in modules %} +{% if 'database' in module %} "bl-python-database@ git+ssh://git@github.com/uclahs-cds/private-BL-python-libraries.git@main#subdirectory=src/database", {% endif %} ] diff --git a/src/web/BL_Python/web/scaffolding/templates/base/{{application_name}}/__init__.py.j2 b/src/web/BL_Python/web/scaffolding/templates/base/{{application_name}}/__init__.py.j2 index b8f8e42f..7fb2407d 100644 --- a/src/web/BL_Python/web/scaffolding/templates/base/{{application_name}}/__init__.py.j2 +++ b/src/web/BL_Python/web/scaffolding/templates/base/{{application_name}}/__init__.py.j2 @@ -1,7 +1,7 @@ application_configs = [] application_modules = [] -{% if modules.database %} +{% if 'database' in module %} from BL_Python.database.config import DatabaseConfig from BL_Python.database.dependency_injection import ScopedSessionModule application_configs.append(DatabaseConfig) diff --git a/src/web/BL_Python/web/scaffolding/templates/basic/config.toml.j2 b/src/web/BL_Python/web/scaffolding/templates/basic/config.toml.j2 index cc9a2fe0..5802faba 100644 --- a/src/web/BL_Python/web/scaffolding/templates/basic/config.toml.j2 +++ b/src/web/BL_Python/web/scaffolding/templates/basic/config.toml.j2 @@ -28,8 +28,8 @@ secure = true samesite = 'None' secret_key = 'abc123' -{% if {'module_name': 'database'} in modules %} +{% if 'database' in module %} [database] -connection_string = 'sqlite:///:memory:' +connection_string = '{{module.database.connection_string}}' sqlalchemy_echo = false {% endif %} \ No newline at end of file diff --git a/src/web/BL_Python/web/scaffolding/templates/openapi/config.toml.j2 b/src/web/BL_Python/web/scaffolding/templates/openapi/config.toml.j2 index 573c589f..d72a7964 100644 --- a/src/web/BL_Python/web/scaffolding/templates/openapi/config.toml.j2 +++ b/src/web/BL_Python/web/scaffolding/templates/openapi/config.toml.j2 @@ -32,8 +32,8 @@ secure = true samesite = 'None' secret_key = '123abc' -{% if modules.database %} +{% if 'database' in module %} [database] -connection_string = 'sqlite:///:memory:' +connection_string = '{{module.database.connection_string}}' sqlalchemy_echo = false {% endif %} \ No newline at end of file diff --git a/src/web/BL_Python/web/scaffolding/templates/optional/{{application_name}}/modules/database/__hook__.py b/src/web/BL_Python/web/scaffolding/templates/optional/{{application_name}}/modules/database/__hook__.py new file mode 100644 index 00000000..67d686b1 --- /dev/null +++ b/src/web/BL_Python/web/scaffolding/templates/optional/{{application_name}}/modules/database/__hook__.py @@ -0,0 +1,19 @@ +from logging import Logger +from typing import Any + + +def on_create(config: dict[str, Any], log: Logger): + """Called when an application is created.""" + + config["module"]["database"] = {} + + connection_string = input( + "\nEnter a database connection string.\nBy default this is `sqlite:///:memory:`.\nRetain this default by pressing enter, or type something else.\n> " + ) + + config["module"]["database"]["connection_string"] = ( + connection_string if connection_string else "sqlite:///:memory:" + ) + log.info( + f"Using database connection string `{config['module']['database']['connection_string']}`" + ) diff --git a/src/web/BL_Python/web/scaffolding/templates/optional/{{application_name}}/modules/database.py.j2 b/src/web/BL_Python/web/scaffolding/templates/optional/{{application_name}}/modules/database/__init__.py.j2 similarity index 100% rename from src/web/BL_Python/web/scaffolding/templates/optional/{{application_name}}/modules/database.py.j2 rename to src/web/BL_Python/web/scaffolding/templates/optional/{{application_name}}/modules/database/__init__.py.j2 From 27dc774be7c13755f8699c8d23c13f82c53d28fb Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Thu, 21 Sep 2023 14:57:11 -0700 Subject: [PATCH 03/11] Add documentation and type directives. --- .../BL_Python/web/scaffolding/scaffolder.py | 129 ++++++++++++++++-- 1 file changed, 114 insertions(+), 15 deletions(-) diff --git a/src/web/BL_Python/web/scaffolding/scaffolder.py b/src/web/BL_Python/web/scaffolding/scaffolder.py index 9cba3cb1..c8e40b54 100644 --- a/src/web/BL_Python/web/scaffolding/scaffolder.py +++ b/src/web/BL_Python/web/scaffolding/scaffolder.py @@ -6,7 +6,12 @@ from BL_Python.programming.collections.dict import merge from jinja2 import BaseLoader, Environment, PackageLoader, Template -from pkg_resources import ResourceManager, get_provider +# fmt: off +from pkg_resources import \ + ResourceManager # pyright: ignore[reportUnknownVariableType,reportGeneralTypeIssues] +from pkg_resources import get_provider + +# fmt: on @dataclass @@ -38,11 +43,13 @@ def __init__(self, config: ScaffoldConfig, log: logging.Logger) -> None: self._config_dict = asdict(config) self._log = log self._checked_directories: set[Path] = set() + # BaseLoader is used for rendering strings only. It does not need additional functionality. self._base_env = Environment( loader=BaseLoader # pyright: ignore[reportGeneralTypeIssues] ) def _create_directory(self, directory: Path, overwrite_existing_files: bool = True): + """Create the directories that the rendered templates will be stored in.""" if directory in self._checked_directories: return @@ -58,6 +65,7 @@ def _create_directory(self, directory: Path, overwrite_existing_files: bool = Tr def _render_template_string( self, template_string: str, template_config: dict[str, Any] | None = None ): + """Render a string that may contain jinja2 directives.""" if template_config is None: template_config = self._config_dict @@ -65,7 +73,9 @@ def _render_template_string( f"Rendering template string `{template_string}` with config `{template_config}`." ) - rendered_string = self._base_env.from_string(template_string).render( + rendered_string = self._base_env.from_string( + template_string + ).render( # pyright: ignore[reportUnknownMemberType] **template_config ) @@ -84,12 +94,35 @@ def _render_template( template: Template | None = None, overwrite_existing_files: bool = True, ): + """ + Render a template that can either be found by name in + the provided `template_environment`, or render the provided `template`. + + :param template_name: The name of the template that might be discoverable in the `template_environment`. + :param template_environment: The jinja2 template environment used for rendering the template. + :param template_directory_prefix: If not provided, templates will be stored relative to their discovered root in the template environment. + For example, if the discovered template relative to the template environment exists at `./__init__.py`, the rendered template will be + stored at `/__init__.py`. If the template environment root points to a deeper directory in the template hierarchy, + for example, in `{{application_name}}/modules/database/`, and the rendered template is `./__init__.py`, this default behavior + is undesired. To resolve this, `template_directory_prefix` can be set in order to cause the rendered template to be stored under a + different directory. For example, with the `template_directory_prefix` of `optional/{{application_name}}/modules/database/`, if the + file `./__init__.py` is discovered at the template environment root, + the file will instead be stored at `/{{application_name}}/modules/database/__init__.py`. + :param template_config: A dictionary of values that the template can reference. + :param template: If provided, this template will be rendered instead of trying to discover the template with the name `template_name` + in the template environment. + :param overwrite_existing_files: If True, any existing files will be overwritten. If False, the template will not be rendered, and + the file at the output location will not be overwritten. + """ if template_config is None: template_config = self._config_dict template_output_path = Path( self._config.output_directory, template_directory_prefix, + # This causes paths with jinja2 directives to be rendered. + # For example, if `self._config_dict['application_name']` is `foo`, + # the path `base/{{application_name}}` is rendered as `base/foo`. self._render_template_string(template_name).replace(".j2", ""), ) @@ -108,10 +141,13 @@ def _render_template( f"Rendering template `{template_output_path}` with config `{template_config}`" ) + # if a template is not provided, find the template in the environment if not template: template = template_environment.get_template(template_name) - template.stream(**template_config).dump(str(template_output_path)) + template.stream( # pyright: ignore[reportUnknownMemberType] + **template_config + ).dump(str(template_output_path)) def _scaffold_directory( self, @@ -119,8 +155,19 @@ def _scaffold_directory( template_directory_prefix: str = "", overwrite_existing_files: bool = True, ): + """ + Render all templates discovered under the root directory of the provided template environment. + Rendered templates are output relative to their location in the template directory, prefixed + with `/template_directory_prefix`. + Only files ending with `.j2` are rendered. + """ # render the base templates - for template_name in cast(list[str], env.list_templates(extensions=["j2"])): + for template_name in cast( + list[str], + env.list_templates( # pyright: ignore[reportUnknownMemberType] + extensions=["j2"] + ), + ): self._render_template( template_name, env, @@ -128,17 +175,30 @@ def _scaffold_directory( overwrite_existing_files=overwrite_existing_files, ) - _manager = ResourceManager() + # These are used to aid in discovering files that are + # part of the BL_Python.web module. Relative path lookups + # do not work, so they are done using _provider. + _manager: Any = ResourceManager() _provider = get_provider("BL_Python.web") def _execute_module_hooks(self, module_template_directory: str): + """ + Modules may have a file named `__hook__.py`. If it exists, it + is executed as part of the scaffolding process. + + Hooks that can be configured in a module are: + Called when an application is being created: + `on_create(config: dict[str, Any], log: Logger) -> None` + """ module_hook_path = Path(module_template_directory, "__hook__.py") - if self._provider.has_resource(str(module_hook_path)): + if self._provider.has_resource( # pyright: ignore[reportUnknownMemberType] + str(module_hook_path) + ): # load the module from its path # and execute it spec = spec_from_file_location( "__hook__", - self._provider.get_resource_filename( + self._provider.get_resource_filename( # pyright: ignore[reportUnknownArgumentType,reportUnknownMemberType] self._manager, str(module_hook_path) ), ) @@ -152,18 +212,33 @@ def _execute_module_hooks(self, module_template_directory: str): for module_name, module_var in vars(module).items(): if not module_name.startswith("on_"): continue + # Execute methods starting with `on_`. + # Since `_execute_module_hooks` is currently + # only called when an application is being created, + # any such method is only called when an application + # is being created. Although the only documented + # method allowed is `on_create`, any such prefixed + # method will be called. module_var(self._config_dict, self._log) - def _scaffold_modules( - self, env: Environment, overwrite_existing_files: bool = True - ): + def _scaffold_modules(self, overwrite_existing_files: bool = True): + """ + Render any modules configured to render. + """ if self._config.modules is None: self._log.debug("No optional modules to scaffold.") return - # render optional module templates + # each module's configuration currently only includes + # the module's name. This name also matches the filesystem + # directory for the module's templates. for module in self._config.modules: + # module templates are stored under `optional/` + # because we don't always want to render any given module. module_template_directory = f"scaffolding/templates/optional/{{{{application_name}}}}/modules/{module.module_name}" + # executed module hooks before any rendering is done + # so the hooks can modify the config or do other + # work if it's needed. self._execute_module_hooks(module_template_directory) module_env = Environment( @@ -174,6 +249,9 @@ def _scaffold_modules( self._scaffold_directory( module_env, + # prefix the directory so that rendered templates + # are output _not_ relative to the template environment + # root, which would store the rendered templates in `./`. template_directory_prefix=f"{self._config.application_name}/modules/{module.module_name}", overwrite_existing_files=overwrite_existing_files, ) @@ -181,11 +259,23 @@ def _scaffold_modules( def _scaffold_endpoints( self, env: Environment, overwrite_existing_files: bool = True ): + """ + Render API endpoint templates. + """ + # technically, `self._config.endpoints` is always set in `__main__` + # but we do this check to satisfy the type checker. if self._config.endpoints is None: self._log.debug("No endpoints to scaffold.") return - # render optional API endpoint templates + # render optional API endpoint templates. + # these templates are stored under `optional/` because: + # 1. they are functionally optional. an application can be + # scaffolded without any endpoints, although `__main__` forces + # at least one endpoint. + # 2. more than one endpoint can be rendered, so a location + # for the templates that is not used to render an entire + # directory of templates (like the `base` templates) is needed. for endpoint in [asdict(dc) for dc in self._config.endpoints]: # {{endpoint_name}} is the file name - this is _not_ meant to be an interpolated string template = env.get_template( @@ -200,6 +290,8 @@ def _scaffold_endpoints( template_string_config = merge(self._config_dict, endpoint) + # render the template output path, replacing the jinja2 + # directives with their associated values from `self._config_dict`. rendered_template_name = self._render_template_string( template.name, template_string_config ) @@ -222,11 +314,15 @@ def _scaffold_endpoints( ) def scaffold(self): + # used for the primary set of templates that a + # scaffolded application is made up of. base_env = Environment( trim_blocks=True, lstrip_blocks=True, loader=PackageLoader("BL_Python.web", f"scaffolding/templates/base"), ) + # used for the selected template type templates + # that can replace files from the base templates. template_type_env = Environment( trim_blocks=True, lstrip_blocks=True, @@ -234,6 +330,7 @@ def scaffold(self): "BL_Python.web", f"scaffolding/templates/{self._config.template_type}" ), ) + # used for templates under `optional/` optional_env = Environment( trim_blocks=True, lstrip_blocks=True, @@ -241,11 +338,13 @@ def scaffold(self): ) if self._config.mode == "create": - # scaffold modules first so they can alter the config if necessary - self._scaffold_modules(optional_env) + # scaffold modules first so they can alter the config if necessary. + # a template environment is not passed in because `scaffold_modules` + # creates a new environment for each module. + self._scaffold_modules() self._scaffold_directory(base_env) - # template type should go after base so it can override any base templates + # template type should go after base so it can override any base templates. self._scaffold_directory(template_type_env) self._scaffold_endpoints(optional_env) From 4d1ec579fe5e95c8fa3fc02c8357d1dd7ce90cb6 Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Thu, 21 Sep 2023 16:07:13 -0700 Subject: [PATCH 04/11] Fix bug with adding unintended value to scaffold configuration. --- src/web/BL_Python/web/scaffolding/scaffolder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/BL_Python/web/scaffolding/scaffolder.py b/src/web/BL_Python/web/scaffolding/scaffolder.py index c8e40b54..a08fe989 100644 --- a/src/web/BL_Python/web/scaffolding/scaffolder.py +++ b/src/web/BL_Python/web/scaffolding/scaffolder.py @@ -288,7 +288,7 @@ def _scaffold_endpoints( ) continue - template_string_config = merge(self._config_dict, endpoint) + template_string_config = merge(self._config_dict.copy(), endpoint) # render the template output path, replacing the jinja2 # directives with their associated values from `self._config_dict`. From 0b70c033613a486a743f9ca38759f1cd8a235bca Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Thu, 21 Sep 2023 16:07:43 -0700 Subject: [PATCH 05/11] Add documentation explaining how the scaffolder works. --- src/web/BL_Python/web/scaffolding/README.md | 99 +++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 src/web/BL_Python/web/scaffolding/README.md diff --git a/src/web/BL_Python/web/scaffolding/README.md b/src/web/BL_Python/web/scaffolding/README.md new file mode 100644 index 00000000..470a319e --- /dev/null +++ b/src/web/BL_Python/web/scaffolding/README.md @@ -0,0 +1,99 @@ +# `BL_Python.web` Scaffolder + +This scaffolding tool exists to help people create a new `BL_Python.web` (or "BLApp") application by removing the tedium of setting up the initial requirements needed to start. + +This document aims to explain how the scaffolding tool works. + +# Templates + +The scaffolding tool relies on Jinja2 templates to render file contents, and file and directory paths. The templates can be found under the `templates/` directory. Any files that can be rendered end with the extension `.j2`. + +While Jinja2 is typically used to render HTML and related web files, this tool instead uses them to render Python, TOML, YAML, Markdown, etc. files as well as directory names. + +## Template Rendering + +Templates are broken up into several directories that each play a distinct role in the complete scaffolding of an application. Review Template Directories for more information. + +The following order is used when rendering: + +1. Modules +2. Base +3. Template Type +4. Endpoints + +Each subsequent set of templates can overwrite files rendered from the prior set. Modules are able to modify the configuration and behavior of rendering, and so are executed first to allow this. + +Note that directory names under each of the `templates/` directories can also contain Jinja2 directives as long as those directives also form a valid file name. For example, there are a couple of directories named `{{application_name}}/`. This is replaced with the name of the application and creates a directory with the name of the application under the output directory. For example, if `application_name` is "foo" then the directory will be named `foo/`. + +### Template Configuration + +Each template has access to the following configuration variables. + +| Name | Type | What it is | Example | +| --- | --- | --- | --- | +| **output_directory** | `str` | The root directory that rendered templates will be stored under | `foo` | +| **application_name** | `str` | The name of the `BL_Python.web` application being scaffolded | `foo` | +| **template_type** | `str` | The type of template being scaffolded - either "basic" or "openapi" | `basic` | +| **modules** | `list[dict[str, Any]]` | A list of the dictionary form of the `ScaffoldModule` type. Contains information on modules to be scaffolded. | `[{'module_name': 'database'}]` | +| **module** | `dict[str, Any]` | A dynamically configured set of values set from each module's `on_create` hook. | `{'database': {'connection_string': 'sqlite:///:memory:'}}` | +| **endpoints** | `list[dict[str, Any]]` | A list of the dictionary form of the `ScaffoldEndpoint` type. Contains information on endpoints to be scaffolded. | `[{'endpoint_name': 'foo', 'hostname': 'http://127.0.0.1:5000'}]` | +| **endpoint** | `dict[str, Any]` | A dictionary form of the `ScaffoldEndpoint` type. For each endpoint to be rendered, this is set to the values for that endpoint and is only available within the templates being rendered for a given endpoint. | `{'endpoint_name': 'foo', 'hostname': 'http://127.0.0.1:5000'}` | +| **mode** | `str` | The scaffolding mode. Can either be `create` or `modify`. | `create` | + +### Rendering Modes + +The scaffolder has two modes: "create" and "modify." Create will create a completely new application, running through each step in the process. Modify only supports adding new API endpoints, and so only executes the rendering of templates under Endpoints. + +## Template Directories + +### Base + +Templates under `base/` contain the essential files for a `BL_Python.web` application. This is the raw structure of such an application, and sets up the basics like: + +1. Python application dependencies +2. README and other documentation / non-code files +3. The bare minimum to run a `BL_Python.web` application + +All rendered files under `base/` can be replaced by rendered templates under `basic/` and `openapi/`. + +While the template `base/{{application_name}}/endpoints/application.py.j2` can also be replaced, it generally should not be. This template provides default endpoints used by infrastructure tooling to manage web applications. + +Templates under `base/` are rendered in a "glob" fashion, meaning all discovered templates are rendered and their structure is reflected in the output directory. + +### Basic + +Templates under `basic/` are used when rendering a "basic" `BL_Python.web` application. This is the default behavior of the scaffolder, and can also be set with the `-t basic` switch. + +A "basic" template uses auto-discovery of Flask Blueprints to create API endpoints. This can be seen in the `optional/{{application_name}}/endpoints/{{endpoint_name}}.py.j2` template, which makes a distinction between template types. + +Templates are `basic/` are also globbed. + +### OpenAPI + +Templates under `openapi/` are used when rendering an "openapi" `BL_Python.web` application. This can be done with the `-t openapi` switch. + +An "openapi" template uses an `openapi.yaml` file to define endpoints and their request and response details. + +OpenAPI applications do not use Flask Blueprints and so rely on a different structure of API template. As such, `base/{{application_name}}/endpoints/application.py.j2` is replaced to reflect this. You can also note the distinction in the `optional/{{application_name}}/endpoints/{{endpoint_name}}.py.j2` template. + +Templates are `openapi/` are also globbed. + +### Optional + +Templates under `optional/` are templates that are rendered in unique ways as compared to the Base, Basic, and OpenAPI templates. Unlike the others, these templates are not globbed, and they might not be used for every application that is scaffolded. + +#### Endpoints + +Templates under `optional/endpoints/` are rendered once for every API endpoint that should be scaffolded. This is done with the `-e ` switch, which can be specified multiple times. + +These templates are aware of the scaffold template type and must make distinctions between the Basic and OpenAPI template types. + +#### Modules + +Templates under `optional/modules/` are rendered for each module specified with the `-m ` switch. + +Modules are unique in that a callback can be specified that is executed when an application is being created. There is no callback available for when an application is modified. The available callbacks are: + +##### `on_create(config: dict[str, Any], log: Logger) -> None` + +This method can modify the configuration or do anything else necessary for the module and the other templates to render correctly. \ No newline at end of file From 7186324249d7a9cd44f47065368b8de26c2042e7 Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Fri, 22 Sep 2023 15:22:01 -0700 Subject: [PATCH 06/11] In templated applications, output all URLs at `/` when running in debug. --- .../endpoints/application.py.j2 | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/web/BL_Python/web/scaffolding/templates/base/{{application_name}}/endpoints/application.py.j2 b/src/web/BL_Python/web/scaffolding/templates/base/{{application_name}}/endpoints/application.py.j2 index 9359ccfb..5fcba9ac 100644 --- a/src/web/BL_Python/web/scaffolding/templates/base/{{application_name}}/endpoints/application.py.j2 +++ b/src/web/BL_Python/web/scaffolding/templates/base/{{application_name}}/endpoints/application.py.j2 @@ -3,9 +3,11 @@ Server blueprint Non-API specific endpoints for application management """ from logging import Logger +from typing import cast -from flask import Blueprint +from flask import Blueprint, Config, Flask, url_for from injector import inject +from werkzeug.routing import BuildError application_blueprint = Blueprint("", __name__) @@ -14,4 +16,33 @@ application_blueprint = Blueprint("", __name__) @application_blueprint.route("/healthcheck", methods=("GET",)) def healthcheck(log: Logger): log.info("Healthcheck") - return "healthcheck: '{{application_name}}' is running" \ No newline at end of file + return "healthcheck: '{{application_name}}' is running" + +@inject +@application_blueprint.route("/", methods=("GET",)) +def root(flask: Flask, config: Config, log: Logger): + # do not allow any requests to / in non-debug environments + if config['ENV'] != 'debug': + return "", 405 + + # in debug environments, print a table of all routes and their allowed methods + output = "" + for rule in flask.url_map.iter_rules(): + try: + url = url_for(rule.endpoint, **(rule.defaults or {})) + sep = "" + output += f"" + except BuildError as e: + log.warn(e) + + output += "
urlmethods
{url}" + for method in sorted(list(cast(set[str], rule.methods))): + output += sep + if method == "GET": + output += f"GET" + else: + output += method + sep = ", " + output += "
" + + return output \ No newline at end of file From 778f43e387769257bd6641f3545e4bd8bfc503aa Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Fri, 22 Sep 2023 15:41:49 -0700 Subject: [PATCH 07/11] Be more descriptive about the root URL and hostname in scaffolded applications. --- src/web/BL_Python/web/scaffolding/templates/base/README.md.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/web/BL_Python/web/scaffolding/templates/base/README.md.j2 b/src/web/BL_Python/web/scaffolding/templates/base/README.md.j2 index 00cd0317..ce314a48 100644 --- a/src/web/BL_Python/web/scaffolding/templates/base/README.md.j2 +++ b/src/web/BL_Python/web/scaffolding/templates/base/README.md.j2 @@ -7,7 +7,9 @@ 3. Install dependencies `pip install .` 4. Run the application `python {{application_name}}` -You can now access your endpoints in your browser, or with `curl`. +You can now access your endpoints in your browser, or with `curl`, using the address [http://127.0.0.1:5000/](http://127.0.0.1:5000/). + +By default, when running in debug mode, accessing [http://127.0.0.1:5000/](http://127.0.0.1:5000/) will display a table of available API endpoint URLs. ## Adding API endpoints From 4ca7c68efea4e0e66f64bf17228e34a37be4be82 Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Fri, 22 Sep 2023 15:42:22 -0700 Subject: [PATCH 08/11] Add some technical documentation to the `BL_Python.web` README. --- src/web/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/web/README.md b/src/web/README.md index 08eaa3be..99d1e749 100644 --- a/src/web/README.md +++ b/src/web/README.md @@ -60,4 +60,12 @@ To create an application with a single API endpoint, run `bl-python-scaffold cre ## Run Your Application -The scaffolder will have created several files and directories, including a README.md, under the output directory. Follow the instructions in your newly scaffolded application's README.md to run and configure your application. \ No newline at end of file +The scaffolder will have created several files and directories, including a README.md, under the output directory. Follow the instructions in your newly scaffolded application's README.md to run and configure your application. + +# About the Library + +`BL_Python.web` is intended to handle a lot of the boilerplate needed to create and run Flask applications. A primary component of that boilerplate is tying disparate pieces of functionality and other libraries together in a seemless way. For example, [SQLAlchemy](https://www.sqlalchemy.org/) is an ORM supported through `BL_Python.database` that this library integrates with to make database functionality simpler to make use of. + +## Flask + +`BL_Python.web` is based on [Flask 1.1.4](https://flask.palletsprojects.com/en/1.1.x/). Updating to Flask 2.x is not currently planned, but may happen in the future. \ No newline at end of file From 244934efd85f1df673e2471ae0177a1b8692e21d Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Fri, 22 Sep 2023 15:45:36 -0700 Subject: [PATCH 09/11] Make it clearer where information on template directories can be found. --- src/web/BL_Python/web/scaffolding/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/BL_Python/web/scaffolding/README.md b/src/web/BL_Python/web/scaffolding/README.md index 470a319e..4dafb5c9 100644 --- a/src/web/BL_Python/web/scaffolding/README.md +++ b/src/web/BL_Python/web/scaffolding/README.md @@ -12,7 +12,7 @@ While Jinja2 is typically used to render HTML and related web files, this tool i ## Template Rendering -Templates are broken up into several directories that each play a distinct role in the complete scaffolding of an application. Review Template Directories for more information. +Templates are broken up into several directories that each play a distinct role in the complete scaffolding of an application. Review [Template Directories](#template-directories) for more information. The following order is used when rendering: From 7fddcaaada8e8dad555d7316bd9c1dfc4d1eef0b Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Fri, 22 Sep 2023 16:00:36 -0700 Subject: [PATCH 10/11] Make the template rendering order and behavior clearer. --- src/web/BL_Python/web/scaffolding/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/web/BL_Python/web/scaffolding/README.md b/src/web/BL_Python/web/scaffolding/README.md index 4dafb5c9..7279e28b 100644 --- a/src/web/BL_Python/web/scaffolding/README.md +++ b/src/web/BL_Python/web/scaffolding/README.md @@ -21,7 +21,9 @@ The following order is used when rendering: 3. Template Type 4. Endpoints -Each subsequent set of templates can overwrite files rendered from the prior set. Modules are able to modify the configuration and behavior of rendering, and so are executed first to allow this. +Each subsequent set of templates can overwrite files rendered from the previous sets. This means, for example, that a rendered template for Template Type, e.g. `openapi/{{application_name}}/endpoints/application.py.j2`, can overwrite the like-named rendered template `base/{{application_name}}/endpoints/application.py.j2` because the Template Type templates under `openapi/` are rendered after the Base templates under `base/`. This behavior is not inherent to Jinja2 and is a conscious decision regarding the behavior of the scaffolding tool. + +Modules are able to modify the configuration and behavior of rendering, and so are executed first to allow this. Note that directory names under each of the `templates/` directories can also contain Jinja2 directives as long as those directives also form a valid file name. For example, there are a couple of directories named `{{application_name}}/`. This is replaced with the name of the application and creates a directory with the name of the application under the output directory. For example, if `application_name` is "foo" then the directory will be named `foo/`. From 02dd3502b4d6051fe50eb6f18098b83353112295 Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Fri, 22 Sep 2023 16:13:48 -0700 Subject: [PATCH 11/11] Make the module hook documentation clearer. --- src/web/BL_Python/web/scaffolding/README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/web/BL_Python/web/scaffolding/README.md b/src/web/BL_Python/web/scaffolding/README.md index 7279e28b..a0b6930e 100644 --- a/src/web/BL_Python/web/scaffolding/README.md +++ b/src/web/BL_Python/web/scaffolding/README.md @@ -86,16 +86,20 @@ Templates under `optional/` are templates that are rendered in unique ways as co #### Endpoints -Templates under `optional/endpoints/` are rendered once for every API endpoint that should be scaffolded. This is done with the `-e ` switch, which can be specified multiple times. +Templates under `optional/{{application_name}}/endpoints/` are rendered once for every API endpoint that should be scaffolded. This is done with the `-e ` switch, which can be specified multiple times. These templates are aware of the scaffold template type and must make distinctions between the Basic and OpenAPI template types. #### Modules -Templates under `optional/modules/` are rendered for each module specified with the `-m ` switch. +Templates under `optional/{{application_name}}/modules/` are rendered for each module specified with the `-m ` switch. -Modules are unique in that a callback can be specified that is executed when an application is being created. There is no callback available for when an application is modified. The available callbacks are: +Modules are unique in that a callback can be specified that is executed when an application is being created. There is no callback available for when an application is modified. These callbacks are defined in `__hook__.py` at the root of the module. + +There is currently only one such callback: ##### `on_create(config: dict[str, Any], log: Logger) -> None` -This method can modify the configuration or do anything else necessary for the module and the other templates to render correctly. \ No newline at end of file +This method can modify the scaffold configuration or do anything else necessary for the module and the other templates to render correctly. + +For an example, take a look at `modules/database/__hook__.py`. This module prompts the user for a database connection string and adds the value to the scaffold configuration key `module.database`. This is then utilized in the templates `basic/config.toml.j2` and `openapi/config.toml.j2`. \ No newline at end of file