From 8fbac1fca7dee91f4d62c3c5f9330e9726992ff8 Mon Sep 17 00:00:00 2001 From: Colin Swaney Date: Tue, 17 Sep 2024 10:00:31 -0400 Subject: [PATCH 1/3] Update database config Move migrations to `db/migrations` and move database to home directory. Try to upgrade database on start-up. --- src/app/__init__.py | 11 ++- src/app/cli/__main__.py | 84 ++++++------------- src/{ => app/db}/migrations/__init__.py | 0 src/{ => app/db}/migrations/alembic.ini | 0 src/{ => app/db}/migrations/env.py | 0 src/{ => app/db}/migrations/script.py.mako | 0 ...024-04-19_initial_revision_57e64d3a65ea.py | 0 .../db}/migrations/versions/__init__.py | 0 8 files changed, 33 insertions(+), 62 deletions(-) rename src/{ => app/db}/migrations/__init__.py (100%) rename src/{ => app/db}/migrations/alembic.ini (100%) rename src/{ => app/db}/migrations/env.py (100%) rename src/{ => app/db}/migrations/script.py.mako (100%) rename src/{ => app/db}/migrations/versions/2024-04-19_initial_revision_57e64d3a65ea.py (100%) rename src/{ => app/db}/migrations/versions/__init__.py (100%) diff --git a/src/app/__init__.py b/src/app/__init__.py index d4a2ea4..61b6eaa 100644 --- a/src/app/__init__.py +++ b/src/app/__init__.py @@ -13,6 +13,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from litestar import Litestar, get, post, put, delete +from litestar.utils.module_loader import module_to_os_path from litestar.datastructures import State from litestar.dto import DTOConfig, DataclassDTO from advanced_alchemy.extensions.litestar import ( @@ -401,14 +402,18 @@ async def get_image_details(): session_config = AsyncSessionConfig(expire_on_commit=False) +BASE_DIR = module_to_os_path("app") + db_config = SQLAlchemyAsyncConfig( - connection_string="sqlite+aiosqlite:///app.sqlite", + connection_string=( + f"sqlite+aiosqlite:///{blackfish_config.BLACKFISH_HOME_DIR}/app.sqlite" + ), metadata=UUIDAuditBase.metadata, create_all=True, alembic_config=AlembicAsyncConfig( version_table_name="ddl_version", - script_config="migrations/alembic.ini", - script_location="migrations", + script_config=f"{BASE_DIR}/db/migrations/alembic.ini", + script_location=f"{BASE_DIR}/db/migrations", ), ) diff --git a/src/app/cli/__main__.py b/src/app/cli/__main__.py index 579e7bc..0faba77 100644 --- a/src/app/cli/__main__.py +++ b/src/app/cli/__main__.py @@ -14,7 +14,8 @@ update_profile, delete_profile, ) -from app.config import config, SlurmRemote +from app.config import config +from app.logger import logger """ The CLI serves as a client to access the API from as well as performing @@ -91,71 +92,36 @@ def start(reload: bool, profile: str) -> None: # pragma: no cover "Start the blackfish app." import uvicorn + from advanced_alchemy.extensions.litestar.alembic import AlembicCommands + from sqlalchemy.exc import OperationalError + from app import __file__ + from app import app if not os.path.isdir(config.BLACKFISH_HOME_DIR): click.echo("Home directory not found. Have you run `blackfish init`?") return - if profile is None: - # TODO: migrate_db() - # TODO: update models table - uvicorn.run( - "app:app", - host=config.BLACKFISH_HOST, - port=config.BLACKFISH_PORT, - log_level="info", - app_dir=os.path.abspath(os.path.join(__file__, "..", "..")), - reload_dirs=os.path.abspath(os.path.join(__file__, "..")), - reload=reload, - ) - # _ = subprocess.check_output( - # [ - # sys.executable, - # "-m", - # "uvicorn", - # "--host", - # config.BLACKFISH_HOST, - # "--port", - # str(config.BLACKFISH_PORT), - # "--log-level", - # "info", - # "--reload", - # "--app-dir", - # os.path.abspath(os.path.join(app.__file__, "..", "..")), - # "app:app", - # ] - # ) - # _ = subprocess.check_output( - # [ - # "litestar", - # "--app-dir", - # os.path.abspath(os.path.join(app.__file__, "..", "..")), - # "run", - # "--reload" - # ] - # ) - else: - # TODO: running Blackfish remotely - profile = config.BLACKFISH_PROFILES[profile] - if isinstance(profile, SlurmRemote): - raise NotImplementedError - # _ = subprocess.check_output( - # [ - # "ssh", - # f"{profile['user']}@{profile['host']}", - # "uvicorn", - # "--port", - # profile['port'], - # "--log-level", - # "info", - # "--reload", - # reload, - # ] - # ) + alembic_commands = AlembicCommands(app=app) + + try: + logger.info("Upgrading database...") + alembic_commands.upgrade() + except OperationalError as e: + if e.args == ("(sqlite3.OperationalError) table service already exists",): + logger.info("Database is already up-to-date. Skipping.") else: - raise NotImplementedError - raise NotImplementedError + logger.error(f"Failed to upgrade database: {e}") + + uvicorn.run( + "app:app", + host=config.BLACKFISH_HOST, + port=config.BLACKFISH_PORT, + log_level="info", + app_dir=os.path.abspath(os.path.join(__file__, "..", "..")), + reload_dirs=os.path.abspath(os.path.join(__file__, "..")), + reload=reload, + ) # blackfish run [OPTIONS] COMMAND diff --git a/src/migrations/__init__.py b/src/app/db/migrations/__init__.py similarity index 100% rename from src/migrations/__init__.py rename to src/app/db/migrations/__init__.py diff --git a/src/migrations/alembic.ini b/src/app/db/migrations/alembic.ini similarity index 100% rename from src/migrations/alembic.ini rename to src/app/db/migrations/alembic.ini diff --git a/src/migrations/env.py b/src/app/db/migrations/env.py similarity index 100% rename from src/migrations/env.py rename to src/app/db/migrations/env.py diff --git a/src/migrations/script.py.mako b/src/app/db/migrations/script.py.mako similarity index 100% rename from src/migrations/script.py.mako rename to src/app/db/migrations/script.py.mako diff --git a/src/migrations/versions/2024-04-19_initial_revision_57e64d3a65ea.py b/src/app/db/migrations/versions/2024-04-19_initial_revision_57e64d3a65ea.py similarity index 100% rename from src/migrations/versions/2024-04-19_initial_revision_57e64d3a65ea.py rename to src/app/db/migrations/versions/2024-04-19_initial_revision_57e64d3a65ea.py diff --git a/src/migrations/versions/__init__.py b/src/app/db/migrations/versions/__init__.py similarity index 100% rename from src/migrations/versions/__init__.py rename to src/app/db/migrations/versions/__init__.py From 7a8f7b9d19438464866ae439836141cffab3d1d5 Mon Sep 17 00:00:00 2001 From: Colin Swaney Date: Tue, 17 Sep 2024 10:38:51 -0400 Subject: [PATCH 2/3] Do not ignore migrations --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 42605ab..35d05a7 100644 --- a/.gitignore +++ b/.gitignore @@ -89,7 +89,6 @@ target/ # Database *.sqlite *.sqlite3 -src/migrations # GitHub pages site/ From 215a9ac2e95323a00ad313ce5c62011672a94d8f Mon Sep 17 00:00:00 2001 From: Colin Swaney Date: Tue, 17 Sep 2024 11:07:06 -0400 Subject: [PATCH 3/3] Create v0.1.0 revision --- .../2024-09-17_v0.1.0_6f8469490c8e.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 src/app/db/migrations/versions/2024-09-17_v0.1.0_6f8469490c8e.py diff --git a/src/app/db/migrations/versions/2024-09-17_v0.1.0_6f8469490c8e.py b/src/app/db/migrations/versions/2024-09-17_v0.1.0_6f8469490c8e.py new file mode 100644 index 0000000..2739e01 --- /dev/null +++ b/src/app/db/migrations/versions/2024-09-17_v0.1.0_6f8469490c8e.py @@ -0,0 +1,90 @@ +# type: ignore +"""v0.1.0 + +Revision ID: 6f8469490c8e +Revises: 57e64d3a65ea +Create Date: 2024-09-17 14:57:00.346660+00:00 + +""" +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING + +import sqlalchemy as sa +from alembic import op +from advanced_alchemy.types import ( + EncryptedString, + EncryptedText, + GUID, + ORA_JSONB, + DateTimeUTC, +) + +if TYPE_CHECKING: + pass + +__all__ = [ + "downgrade", + "upgrade", + "schema_upgrades", + "schema_downgrades", + "data_upgrades", + "data_downgrades", +] + +sa.GUID = GUID +sa.DateTimeUTC = DateTimeUTC +sa.ORA_JSONB = ORA_JSONB +sa.EncryptedString = EncryptedString +sa.EncryptedText = EncryptedText + +# revision identifiers, used by Alembic. +revision = "6f8469490c8e" +down_revision = "57e64d3a65ea" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + with op.get_context().autocommit_block(): + schema_upgrades() + data_upgrades() + + +def downgrade() -> None: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + with op.get_context().autocommit_block(): + data_downgrades() + schema_downgrades() + + +def schema_upgrades() -> None: + """schema upgrade migrations go here.""" + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("service", schema=None) as batch_op: + batch_op.alter_column("user", existing_type=sa.VARCHAR(), nullable=True) + batch_op.alter_column("port", existing_type=sa.INTEGER(), nullable=True) + + # ### end Alembic commands ### + + +def schema_downgrades() -> None: + """schema downgrade migrations go here.""" + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("service", schema=None) as batch_op: + batch_op.alter_column("port", existing_type=sa.INTEGER(), nullable=False) + batch_op.alter_column("user", existing_type=sa.VARCHAR(), nullable=False) + + # ### end Alembic commands ### + + +def data_upgrades() -> None: + """Add any optional data upgrade migrations here!""" + + +def data_downgrades() -> None: + """Add any optional data downgrade migrations here!"""