Add Alembic migration support to database libraries

.github/workflows/CICD.yaml

@@ -163,10 +163,7 @@ jobs:
         with:
           name: pytest-and-coverage-report
           path: |
-            pytest.xml
-            cov.xml
-            .coverage
-            coverage/
+            reports/pytest/
           retention-days: 1
           if-no-files-found: error
 
@@ -219,14 +216,14 @@ jobs:
         with:
           name: bandit-sast-report
           path: |
-            bandit.sarif
+            reports/bandit.sarif
           retention-days: 1
           if-no-files-found: error
 
       - name: Upload bandit report to CodeQL
         uses: github/codeql-action/upload-sarif@v3
         with:
-          sarif_file: bandit.sarif
+          sarif_file: reports/bandit.sarif
 
   Style:
     name: Style and formatting

Makefile

@@ -23,9 +23,16 @@ GITHUB_REF ?= 00000000-0000-0000-0000-000000000000
 # Can be overridden.
 GITHUB_WORKSPACE ?= $(CURDIR)
 
+# What repository to publish packages to.
+# `testpypi` and `pypi` are valid values.
 PYPI_REPO ?= testpypi
 
+# The directory to write ephermal reports to,
+# such as pytest coverage reports.
+REPORTS_DIR ?= reports
 BANDIT_REPORT := bandit.sarif
+PYTEST_REPORT := pytest
+
 
 # Can be overridden. This is used to change the prereqs
 # of some supporting targets, like `format-ruff`.
@@ -70,8 +77,7 @@ PYPROJECT_FILES=./pyproject.toml $(wildcard src/*/pyproject.toml)
 PACKAGE_PATHS=$(subst /pyproject.toml,,$(PYPROJECT_FILES))
 PACKAGES=$(subst /pyproject.toml,,$(subst src/,BL_Python.,$(wildcard src/*/pyproject.toml)))
 
-# Rather than duplicating BL_Python.all,
-# just prereq it.
+.PHONY: dev
 dev : $(VENV) $(SETUP_DEPENDENCIES)
 	$(MAKE) _dev_build DEFAULT_TARGET=dev
 _dev_configure : $(VENV) $(PYPROJECT_FILES)
@@ -108,6 +114,7 @@ _cicd_build : _cicd_configure
 
 	@$(REPORT_VENV_USAGE)
 
+BL_Python.all: $(DEFAULT_TARGET)
 $(PACKAGES) : BL_Python.%: src/%/pyproject.toml $(VENV) $(CONFIGURE_TARGET) $(PYPROJECT_FILES)
 	@if [ -d $(call package_to_dist,$*) ]; then
 		@echo "Package $@ is already built, skipping..."
@@ -165,6 +172,7 @@ format-ruff : $(VENV) $(BUILD_TARGET)
 
 	ruff format --preview --respect-gitignore
 
+.PHONY: format format-ruff format-isort
 format : $(VENV) $(BUILD_TARGET) format-isort format-ruff
 
 
@@ -201,49 +209,64 @@ test-bandit : $(VENV) $(BUILD_TARGET)
 #	while testing bandit.
 	-bandit -c pyproject.toml \
 		--format sarif \
-		--output $(BANDIT_REPORT) \
+		--output $(REPORTS_DIR)/$(BANDIT_REPORT) \
 		-r .
 
 test-pytest : $(VENV) $(BUILD_TARGET)
 	$(ACTIVATE_VENV)
 
-	pytest $(PYTEST_FLAGS)
+	pytest $(PYTEST_FLAGS) \
+		&& PYTEST_EXIT_CODE=0 \
+		|| PYTEST_EXIT_CODE=$$?
+
+	-coverage html --data-file=$(REPORTS_DIR)/$(PYTEST_REPORT)/.coverage
+	-junit2html $(REPORTS_DIR)/$(PYTEST_REPORT)/pytest.xml $(REPORTS_DIR)/$(PYTEST_REPORT)/pytest.html
 
-	coverage html -d coverage
+	exit $$PYTEST_EXIT_CODE
 
+.PHONY: test test-pytest test-bandit test-pyright test-ruff test-isort
+_test : $(VENV) $(BUILD_TARGET) test-isort test-ruff test-pyright test-bandit test-pytest
 test : CMD_PREFIX=@
-test : $(VENV) $(BUILD_TARGET) clean-test test-isort test-ruff test-pyright test-bandit test-pytest
+test : clean-test
+	$(MAKE) -j --keep-going _test
 
 
+.PHONY: publish-all
 # Publishing should use a real install, which `cicd` fulfills
 publish-all : REWRITE_DEPENDENCIES=false
+# Publishing should use a real install. Reset the build env.
 publish-all : reset $(VENV)
 	$(ACTIVATE_VENV)
 
 	./publish_all.sh $(PYPI_REPO)
 
 
 clean-build :
-	find . -type d \( \
+	find . -type d \
+	\( \
+		-path ./$(VENV) \
+		-o -path ./.git \
+	\) -prune -false \
+	-o \( \
 		-name build \
+		-o -name dist \
 		-o -name __pycache__ \
 		-o -name \*.egg-info \
 		-o -name .pytest-cache \
 	\) -prune -exec rm -rf {} \;
 
 clean-test :
-	$(CMD_PREFIX)rm -rf cov.xml \
-		pytest.xml \
-		coverage \
-		.coverage \
-		$(BANDIT_REPORT)
-
+	$(CMD_PREFIX)rm -rf \
+		$(REPORTS_DIR)/$(PYTEST_REPORT) \
+		$(REPORTS_DIR)/$(BANDIT_REPORT)
 
+.PHONY: clean clean-test clean-build
 clean : clean-build clean-test
 	rm -rf $(VENV)
 
 	@echo '\nDeactivate your venv with `deactivate`'
 
+.PHONY: remake
 remake :
 	$(MAKE) clean
 	$(MAKE)
@@ -253,5 +276,6 @@ reset-check:
 	@echo -n "This will make destructive changes! Considering stashing changes first.\n"
 	@( read -p "Are you sure? [y/N]: " response && case "$$response" in [yY]) true;; *) false;; esac )
 
+.PHONY: reset reset-check
 reset : reset-check clean
 	git checkout -- $(PYPROJECT_FILES)

pyproject.toml

@@ -83,6 +83,8 @@ dev-dependencies = [
     "pytest-mock",
     "mock",
     "pytest-cov ~= 4.1",
+    "coverage ~= 7.4",
+    "junit2html ~= 30.1",
     "pyright ~= 1.1",
     "isort ~= 5.13",
     "ruff ~= 0.3",
@@ -138,6 +140,7 @@ reportUninitializedInstanceVariable = "information"
 reportUnnecessaryTypeIgnoreComment = "information"
 reportUnusedCallResult = "information"
 reportMissingTypeStubs = "information"
+reportWildcardImportFromLibrary = "warning"
 
 [tool.pytest.ini_options]
 pythonpath = [
@@ -173,9 +176,9 @@ addopts = [
     # and
     # https://github.com/microsoft/vscode-python/issues/21845
     "--cov=.",
-    "--junitxml=pytest.xml",
+    "--junitxml=reports/pytest/pytest.xml",
     "-o=junit_family=xunit2",
-    "--cov-report=xml:cov.xml",
+    "--cov-report=xml:reports/pytest/cov.xml",
     "--cov-report=term-missing",
 ]
 
@@ -187,9 +190,11 @@ norecursedirs = "__pycache__ build .pytest_cache *.egg-info .venv .github-venv"
 include_namespace_packages = true
 
 [tool.coverage.html]
+directory = "reports/pytest/coverage"
 show_contexts = true
 
 [tool.coverage.run]
+data_file = "reports/pytest/.coverage"
 dynamic_context = "test_function"
 relative_files = true
 omit = [

reports/.gitignore

@@ -0,0 +1,6 @@
+# https://stackoverflow.com/a/932982
+
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

src/database/BL_Python/database/config.py

@@ -1,13 +1,47 @@
+from typing import Any
+
 from BL_Python.programming.config import AbstractConfig
 from pydantic import BaseModel
+from pydantic.config import ConfigDict
 
 
 class DatabaseConnectArgsConfig(BaseModel):
+    # allow any values, as this type is not
+    # specifically the type to be used elsewhere
+    model_config = ConfigDict(extra="allow")
+
+
+class PostgreSQLDatabaseConnectArgsConfig(DatabaseConnectArgsConfig):
+    # ignore anything that DatabaseConnectArgsConfig
+    # allowed to be set, except for any other attributes
+    # of this class, which will end up assigned through
+    # the instatiation of the __init__ override of DatabaseConfig
+    model_config = ConfigDict(extra="ignore")
+
     sslmode: str = ""
     options: str = ""
 
 
+class SQLiteDatabaseConnectArgsConfig(DatabaseConnectArgsConfig):
+    model_config = ConfigDict(extra="ignore")
+
+
 class DatabaseConfig(BaseModel, AbstractConfig):
+    def __init__(self, **data: Any):
+        super().__init__(**data)
+
+        model_data = self.connect_args.model_dump() if self.connect_args else {}
+        if self.connection_string.startswith("sqlite://"):
+            self.connect_args = SQLiteDatabaseConnectArgsConfig(**model_data)
+        elif self.connection_string.startswith("postgresql://"):
+            self.connect_args = PostgreSQLDatabaseConnectArgsConfig(**model_data)
+
     connection_string: str = "sqlite:///:memory:"
     sqlalchemy_echo: bool = False
+    # the static field allows Pydantic to store
+    # values from a dictionary
     connect_args: DatabaseConnectArgsConfig | None = None
+
+
+class Config(BaseModel, AbstractConfig):
+    database: DatabaseConfig

src/database/BL_Python/database/migrations/__init__.py

@@ -1,115 +0,0 @@
-from typing import TYPE_CHECKING, List, Optional, Protocol, cast, final
-
-from sqlalchemy.orm import DeclarativeMeta
-
-MetaBaseType = Type[DeclarativeMeta]
-
-if TYPE_CHECKING:
-    from typing import Dict, Protocol, Type, TypeVar, Union
-
-    from sqlalchemy.engine import Dialect
-
-    TBase = TypeVar("TBase")
-
-    class TableNameCallback(Protocol):
-        def __call__(
-            self,
-            dialect_schema: "Union[str, None]",
-            full_table_name: str,
-            base_table: str,
-            meta_base: MetaBaseType,
-        ) -> None: ...
-
-    class Connection(Protocol):
-        dialect: Dialect
-
-    class Op(Protocol):
-        @staticmethod
-        def get_bind() -> Connection: ...
-
-
-@final
-class DialectHelper:
-    """
-    Utilities to get database schema and table names
-    for different SQL dialects and database engines.
-
-    For example, PostgreSQL supports schemas. This means:
-        * get_dialect_schema(meta) returns a schema name, if there is one, e.g. "cap"
-        * get_full_table_name(table_name, meta) returns the schema name, followed by the table name, e.g. " cap.assay_plate "
-
-    SQLite does not support schemas. This means:
-        * get_dialect_schema(meta) returns None
-        * get_full_table_name(table_name, meta) returns the table name, with the schema name prepended to it, e.g. " 'cap.assay_plate' "
-                                                The key difference is that there is no schema, and the table name comes from the SQLite
-                                                engine instantiation, which prepends the "schema" to the table name.
-    """
-
-    dialect: "Dialect"
-    dialect_supports_schemas: bool
-
-    def __init__(self, dialect: "Dialect"):
-        self.dialect = dialect
-        # right now we only care about SQLite and PSQL,
-        # so if the dialect is PSQL, then we consider the
-        # dialect to support schemas, otherwise it does not.
-        self.dialect_supports_schemas = dialect.name == "postgresql"
-
-    @staticmethod
-    def get_schema(meta: "MetaBaseType"):
-        table_args = cast(
-            Optional[dict[str, str]], getattr(meta, "__table_args__", None)
-        )
-        if table_args is None:
-            return None
-        return table_args.get("schema")
-
-    def get_dialect_schema(self, meta: "MetaBaseType"):
-        """Get the database schema as a string, or None if the dialect does not support schemas."""
-        if not self.dialect_supports_schemas:
-            return None
-        return DialectHelper.get_schema(meta)
-
-    def get_full_table_name(self, table_name: str, meta: "MetaBaseType"):
-        """
-        If the dialect supports schemas, then the table name does not have the schema prepended.
-        In dialects that don't support schemas, e.g., SQLite, the table name has the schema prepended.
-        This is because, when schemas are supported, the dialect automatically handles which schema
-        to use, while non-schema dialects do not reference any schemas.
-        """
-        if self.get_dialect_schema(meta):
-            return table_name
-        else:
-            return f"{DialectHelper.get_schema(meta)}.{table_name}"
-
-    def get_timestamp_sql(self):
-        timestamp_default_sql = "now()"
-        if self.dialect.name == "sqlite":
-            timestamp_default_sql = "CURRENT_TIMESTAMP"
-        return timestamp_default_sql
-
-    @staticmethod
-    def iterate_table_names(
-        op: "Op",
-        schema_tables: "Dict[MetaBaseType, List[str]]",
-        table_name_callback: "TableNameCallback",
-    ):
-        """
-        Call `table_name_callback` once for every table in every Base.
-
-        op: The `op` object from Alembic.
-        schema_tables: A dictionary of the tables this call applies to for every Base.
-        table_name_callback: A callback executed for every table in `schema_tables`.
-        """
-        dialect: Dialect = op.get_bind().dialect
-        schema = DialectHelper(dialect)
-        get_full_table_name = schema.get_full_table_name
-        get_dialect_schema = schema.get_dialect_schema
-
-        for meta_base, schema_base_tables in schema_tables.items():
-            dialect_schema = get_dialect_schema(meta_base)
-            for base_table in schema_base_tables:
-                full_table_name = get_full_table_name(base_table, meta_base)
-                table_name_callback(
-                    dialect_schema, full_table_name, base_table, meta_base
-                )

src/database/BL_Python/database/migrations/alembic/__main__.py

@@ -0,0 +1,31 @@
+import logging
+from os import environ
+
+# this is Alembic's main entry point
+from .bl_alembic import BLAlembic
+
+
+def bl_alembic(
+    argv: list[str] | None = None,
+    log_level: int | str | None = None,
+    allow_overwrite: bool | None = None,
+) -> None:
+    """
+    A method to support the `bl-alembic` command, which replaces `alembic.
+
+    :param list[str] | None argv: CLI arguments, defaults to None
+    :param int | str | None log_level: An integer log level to configure logging verbosity, defaults to None
+    """
+    logging.basicConfig(level=logging.INFO)
+    if not log_level:
+        log_level = environ.get(BLAlembic.LOG_LEVEL_NAME)
+        log_level = int(log_level) if log_level else logging.INFO
+
+    logger = logging.getLogger()
+    logger.setLevel(log_level)
+
+    BLAlembic(argv, logger).run()
+
+
+if __name__ == "__main__":
+    bl_alembic()