Flatten artefact.source and remove store from artefact unique constraint

backend/migrations/versions/2023_10_25_1139-cb121ad343dd_add_track_store_pocket_repo_to_artefact.py

@@ -0,0 +1,29 @@
+"""Add track, store, series, repo to Artefact
+
+Revision ID: cb121ad343dd
+Revises: ce0b50f657e9
+Create Date: 2023-10-25 11:39:46.208427+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "cb121ad343dd"
+down_revision = "ce0b50f657e9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column("artefact", sa.Column("track", sa.String(), nullable=True))
+    op.add_column("artefact", sa.Column("store", sa.String(), nullable=True))
+    op.add_column("artefact", sa.Column("series", sa.String(), nullable=True))
+    op.add_column("artefact", sa.Column("repo", sa.String(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("artefact", "repo")
+    op.drop_column("artefact", "series")
+    op.drop_column("artefact", "store")
+    op.drop_column("artefact", "track")

backend/migrations/versions/2023_10_25_1233-38807948d269_copy_artefact_source_into_track_store_.py

@@ -0,0 +1,61 @@
+"""Copy Artefact.source into track, store, series & repo
+
+Revision ID: 38807948d269
+Revises: cb121ad343dd
+Create Date: 2023-10-25 12:33:03.206702+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "38807948d269"
+down_revision = "cb121ad343dd"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    artefact_table = sa.table(
+        "artefact",
+        sa.column("id", sa.Integer()),
+        sa.column("source", sa.JSON()),
+        sa.column("track", sa.String()),
+        sa.column("store", sa.String()),
+        sa.column("series", sa.String()),
+        sa.column("repo", sa.String()),
+    )
+
+    conn = op.get_bind()
+    get_artefacts_id_source = sa.select(artefact_table.c.id, artefact_table.c.source)
+    for id, source in conn.execute(get_artefacts_id_source):
+        conn.execute(
+            sa.update(artefact_table)
+            .where(artefact_table.c.id == id)
+            .values(
+                track=source.get("track"),
+                store=source.get("store"),
+                series=source.get("series"),
+                repo=source.get("repo"),
+            )
+        )
+
+
+def downgrade() -> None:
+    artefact_table = sa.table(
+        "artefact",
+        sa.column("track", sa.String()),
+        sa.column("store", sa.String()),
+        sa.column("series", sa.String()),
+        sa.column("repo", sa.String()),
+    )
+
+    conn = op.get_bind()
+    conn.execute(
+        sa.update(artefact_table).values(
+            track=None,
+            store=None,
+            series=None,
+            repo=None,
+        )
+    )

backend/migrations/versions/2023_10_25_1240-0d23bbbe9ec8_add_delete_cascades_to_foreign_keys.py

@@ -0,0 +1,88 @@
+"""Add delete cascades to foreign keys
+
+Revision ID: 0d23bbbe9ec8
+Revises: 38807948d269
+Create Date: 2023-10-25 12:40:53.807999+00:00
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0d23bbbe9ec8"
+down_revision = "38807948d269"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_constraint("artefact_stage_id_fkey", "artefact", type_="foreignkey")
+    op.create_foreign_key(
+        "artefact_stage_id_fkey",
+        "artefact",
+        "stage",
+        ["stage_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    op.drop_constraint(
+        "artefact_build_artefact_id_fkey", "artefact_build", type_="foreignkey"
+    )
+    op.create_foreign_key(
+        "artefact_build_artefact_id_fkey",
+        "artefact_build",
+        "artefact",
+        ["artefact_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    op.drop_constraint("stage_family_id_fkey", "stage", type_="foreignkey")
+    op.create_foreign_key(
+        "stage_family_id_fkey",
+        "stage",
+        "family",
+        ["family_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+    op.drop_constraint(
+        "test_execution_artefact_build_id_fkey", "test_execution", type_="foreignkey"
+    )
+    op.create_foreign_key(
+        "test_execution_artefact_build_id_fkey",
+        "test_execution",
+        "artefact_build",
+        ["artefact_build_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint(
+        "test_execution_artefact_build_id_fkey", "test_execution", type_="foreignkey"
+    )
+    op.create_foreign_key(
+        "test_execution_artefact_build_id_fkey",
+        "test_execution",
+        "artefact_build",
+        ["artefact_build_id"],
+        ["id"],
+    )
+    op.drop_constraint("stage_family_id_fkey", "stage", type_="foreignkey")
+    op.create_foreign_key(
+        "stage_family_id_fkey", "stage", "family", ["family_id"], ["id"]
+    )
+    op.drop_constraint(
+        "artefact_build_artefact_id_fkey", "artefact_build", type_="foreignkey"
+    )
+    op.create_foreign_key(
+        "artefact_build_artefact_id_fkey",
+        "artefact_build",
+        "artefact",
+        ["artefact_id"],
+        ["id"],
+    )
+    op.drop_constraint("artefact_stage_id_fkey", "artefact", type_="foreignkey")
+    op.create_foreign_key(
+        "artefact_stage_id_fkey", "artefact", "stage", ["stage_id"], ["id"]
+    )

backend/migrations/versions/2023_10_25_1247-16043e3ffdd9_delete_snaps_due_to_corrupted_store.py

@@ -0,0 +1,42 @@
+"""Delete snaps due to corrupted store
+
+Revision ID: 16043e3ffdd9
+Revises: 0d23bbbe9ec8
+Create Date: 2023-10-25 12:47:01.958805+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "16043e3ffdd9"
+down_revision = "0d23bbbe9ec8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # This is a distructive migration it will remove all snaps from the database.
+    # The reason is that store should not be part of the unique constraint,
+    # and merging the fragmented snap artefacts will be challenging. Especially
+    # given that we don't know what the appropriate store should be.
+
+    artefact_table = sa.table("artefact", sa.column("stage_id"))
+    stage_table = sa.table("stage", sa.column("id"), sa.column("family_id"))
+    family_table = sa.table("family", sa.column("id"), sa.column("name"))
+
+    delete_stmt = (
+        sa.delete(artefact_table)
+        .where(artefact_table.c.stage_id == stage_table.c.id)
+        .where(stage_table.c.family_id == family_table.c.id)
+        .where(family_table.c.name == "snap")
+        .compile(dialect=postgresql.dialect(), compile_kwargs={"literal_binds": True})
+    )
+
+    conn = op.get_bind()
+    conn.execute(delete_stmt)
+
+
+def downgrade() -> None:
+    pass

backend/migrations/versions/2023_10_25_1345-d646a347472e_fix_artefact_uniqueness_constraint_to_.py

@@ -0,0 +1,32 @@
+"""Fix artefact uniqueness constraint to exclude store
+
+Revision ID: d646a347472e
+Revises: 16043e3ffdd9
+Create Date: 2023-10-25 13:45:11.298029+00:00
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d646a347472e"
+down_revision = "16043e3ffdd9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_constraint("unique_artefact", "artefact")
+    op.create_unique_constraint(
+        "unique_artefact",
+        "artefact",
+        ["name", "version", "track", "series", "repo"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint("unique_artefact", "artefact")
+    op.create_unique_constraint(
+        "unique_artefact",
+        "artefact",
+        ["name", "version", "source"],
+    )

backend/migrations/versions/2023_10_26_0755-49221114815a_drop_artefact_source_field.py

@@ -0,0 +1,53 @@
+"""Drop Artefact source field
+
+Revision ID: 49221114815a
+Revises: d646a347472e
+Create Date: 2023-10-26 07:55:27.449425+00:00
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "49221114815a"
+down_revision = "d646a347472e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_column("artefact", "source")
+
+
+def downgrade() -> None:
+    op.add_column(
+        "artefact",
+        sa.Column("source", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    )
+
+    artefact_table = sa.table(
+        "artefact",
+        sa.column("id", sa.Integer()),
+        sa.column("track", sa.String()),
+        sa.column("store", sa.String()),
+        sa.column("series", sa.String()),
+        sa.column("repo", sa.String()),
+        sa.column("source", postgresql.JSONB(astext_type=sa.Text())),
+    )
+
+    conn = op.get_bind()
+    for artefact in conn.execute(sa.select(artefact_table)):
+        source = {
+            k: getattr(artefact, k)
+            for k in ["track", "store", "series", "repo"]
+            if getattr(artefact, k) is not None
+        }
+
+        conn.execute(
+            sa.update(artefact_table)
+            .where(artefact_table.c.id == artefact.id)
+            .values(source=source)
+        )
+
+    op.alter_column("artefact", "source", nullable=False)

backend/test_observer/data_access/models.py

@@ -23,7 +23,6 @@
 from typing import TypeVar
 
 from sqlalchemy import ForeignKey, Index, String, UniqueConstraint, column
-from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import (
     DeclarativeBase,
     Mapped,
@@ -65,7 +64,7 @@ class Family(Base):
     name: Mapped[str] = mapped_column(String(100), unique=True, index=True)
     # Relationships
     stages: Mapped[list["Stage"]] = relationship(
-        back_populates="family", cascade="all, delete-orphan", order_by="Stage.position"
+        back_populates="family", cascade="all, delete", order_by="Stage.position"
     )
 
     def __repr__(self) -> str:
@@ -80,10 +79,10 @@ class Stage(Base):
     name: Mapped[str] = mapped_column(String(100), index=True)
     position: Mapped[int] = mapped_column()
     # Relationships
-    family_id: Mapped[int] = mapped_column(ForeignKey("family.id"))
+    family_id: Mapped[int] = mapped_column(ForeignKey("family.id", ondelete="CASCADE"))
     family: Mapped[Family] = relationship(back_populates="stages")
     artefacts: Mapped[list["Artefact"]] = relationship(
-        back_populates="stage", cascade="all, delete-orphan"
+        back_populates="stage", cascade="all, delete"
     )
 
     @property
@@ -106,19 +105,24 @@ class Artefact(Base):
 
     name: Mapped[str] = mapped_column(String(200), index=True)
     version: Mapped[str]
-    source: Mapped[dict] = mapped_column(JSONB)
+    track: Mapped[str | None]
+    store: Mapped[str | None]
+    series: Mapped[str | None]
+    repo: Mapped[str | None]
     # Relationships
-    stage_id: Mapped[int] = mapped_column(ForeignKey("stage.id"))
+    stage_id: Mapped[int] = mapped_column(ForeignKey("stage.id", ondelete="CASCADE"))
     stage: Mapped[Stage] = relationship(back_populates="artefacts")
     builds: Mapped[list["ArtefactBuild"]] = relationship(
-        back_populates="artefact", cascade="all, delete-orphan"
+        back_populates="artefact", cascade="all, delete"
     )
     # Default fields
     due_date: Mapped[date | None]
     status: Mapped[ArtefactStatus | None]
 
     __table_args__ = (
-        UniqueConstraint("name", "version", "source", name="unique_artefact"),
+        UniqueConstraint(
+            "name", "version", "track", "series", "repo", name="unique_artefact"
+        ),
     )
 
     def __repr__(self) -> str:
@@ -135,12 +139,14 @@ class ArtefactBuild(Base):
     architecture: Mapped[str] = mapped_column(String(100), index=True)
     revision: Mapped[int | None]
     # Relationships
-    artefact_id: Mapped[int] = mapped_column(ForeignKey("artefact.id"))
+    artefact_id: Mapped[int] = mapped_column(
+        ForeignKey("artefact.id", ondelete="CASCADE")
+    )
     artefact: Mapped[Artefact] = relationship(
         back_populates="builds", foreign_keys=[artefact_id]
     )
     test_executions: Mapped[list["TestExecution"]] = relationship(
-        back_populates="artefact_build", cascade="all, delete-orphan"
+        back_populates="artefact_build", cascade="all, delete"
     )
 
     __table_args__ = (
@@ -198,7 +204,9 @@ class TestExecution(Base):
     jenkins_link: Mapped[str] = mapped_column(String(200), nullable=True)
     c3_link: Mapped[str] = mapped_column(String(200), nullable=True)
     # Relationships
-    artefact_build_id: Mapped[int] = mapped_column(ForeignKey("artefact_build.id"))
+    artefact_build_id: Mapped[int] = mapped_column(
+        ForeignKey("artefact_build.id", ondelete="CASCADE")
+    )
     artefact_build: Mapped["ArtefactBuild"] = relationship(
         back_populates="test_executions"
     )