Restart test executions functionality (#161)

* Add dummy script to be used by jenkins

* Add dummy restart endpoint

* Remove unused import

* Fix OCI image not building by recreating poetry.lock file

* Fix tests

* Rename restart endpoint to reruns to be more RESTFull

* Basic implementation of test execution reruns

* Add another test case

* Rename some db types due to alembic/sqlalchemy upgrade

* Return family with test execution pending reruns

* Add test_execution_rerunner.py script to be used by jenkins

* Return is_rerun_requested with test execution objects

* Fix migration merge conflict

* Add a rerun example to seed data script

* Delete rerun request when starting a rerun

* More reruns in seed script

* Add frontend button to rerun test executions

* Small refactor

backend/migrations/versions/2024_04_30_0926-08bc88dcb1e1_create_test_execution_rerun_requests_.py

@@ -0,0 +1,40 @@
+"""Create test execution rerun requests table
+
+Revision ID: 08bc88dcb1e1
+Revises: 5d36de5a8a48
+Create Date: 2024-04-30 09:26:48.766175+00:00
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "08bc88dcb1e1"
+down_revision = "5d36de5a8a48"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "test_execution_rerun_request",
+        sa.Column("test_execution_id", sa.Integer(), nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["test_execution_id"],
+            ["test_execution.id"],
+            name=op.f("test_execution_rerun_request_test_execution_id_fkey"),
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("test_execution_rerun_request_pkey")),
+        sa.UniqueConstraint(
+            "test_execution_id",
+            name=op.f("test_execution_rerun_request_test_execution_id_key"),
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("test_execution_rerun_request")

backend/migrations/versions/2024_04_30_1029-33c0383ea9ca_rename_types_due_to_sqlalchemy_alembic_.py

@@ -0,0 +1,25 @@
+"""Rename types due to sqlalchemy/alembic upgrade
+
+Revision ID: 33c0383ea9ca
+Revises: 08bc88dcb1e1
+Create Date: 2024-04-30 10:29:23.440961+00:00
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "33c0383ea9ca"
+down_revision = "08bc88dcb1e1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute("ALTER TYPE artefact_status_enum RENAME TO artefactstatus")
+    op.execute("ALTER TYPE test_status_enum RENAME TO testexecutionstatus")
+
+
+def downgrade() -> None:
+    op.execute("ALTER TYPE artefactstatus RENAME TO artefact_status_enum")
+    op.execute("ALTER TYPE testexecutionstatus RENAME TO test_status_enum")

backend/scripts/seed_data.py

@@ -24,6 +24,7 @@
 BASE_URL = "http://localhost:30000/v1"
 START_TEST_EXECUTION_URL = f"{BASE_URL}/test-executions/start-test"
 END_TEST_EXECUTION_URL = f"{BASE_URL}/test-executions/end-test"
+RERUN_TEST_EXECUTION_URL = f"{BASE_URL}/test-executions/reruns"
 
 START_TEST_EXECUTION_REQUESTS = [
     StartTestExecutionRequest(
@@ -293,19 +294,34 @@ def seed_data(client: TestClient | requests.Session, session: Session | None = N
     ):
         add_user(email, session)
 
+    test_executions = []
     for start_request in START_TEST_EXECUTION_REQUESTS:
-        client.put(
+        response = client.put(
             START_TEST_EXECUTION_URL, json=start_request.model_dump(mode="json")
-        ).raise_for_status()
+        )
+        response.raise_for_status()
+        test_executions.append(response.json())
 
     for end_request in END_TEST_EXECUTION_REQUESTS:
         client.put(
             END_TEST_EXECUTION_URL, json=end_request.model_dump(mode="json")
         ).raise_for_status()
 
+    _rerun_some_test_executions(client, test_executions)
+
     _add_bugurl_and_duedate(session)
 
 
+def _rerun_some_test_executions(
+    client: TestClient | requests.Session, test_executions: list[dict]
+) -> None:
+    for te in test_executions[::2]:
+        client.post(
+            RERUN_TEST_EXECUTION_URL,
+            json={"test_execution_id": te["id"]},
+        ).raise_for_status()
+
+
 def _add_bugurl_and_duedate(session: Session) -> None:
     artefact = session.scalar(select(Artefact).limit(1))
 

backend/scripts/test_executions_rerunner.py

@@ -0,0 +1,65 @@
+"""
+This script will be used by jenkins to rerun test executions as requested
+by test observer. Please note that jenkins will download this file then run it.
+Therefore, this script can't import from the rest of the codebase and shouldn't be
+renamed. Dependencies used by this script must be installed on jenkins.
+"""
+
+import logging
+import re
+from os import environ
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+reruns_link = "https://test-observer.canonical.com/v1/test-executions/reruns"
+
+
+class Main:
+    def __init__(self, jenkins_api_token: str | None = None):
+        self.jenkins_auth = HTTPBasicAuth(
+            "admin", jenkins_api_token or environ["JENKINS_API_TOKEN"]
+        )
+
+    def run(self):
+        self._load_rerun_requests()
+        self._submit_rerun_requests()
+
+    def _load_rerun_requests(self) -> None:
+        response = requests.get(reruns_link)
+        self.rerun_requests = response.json()
+        logging.info(f"Received the following rerun requests:\n{self.rerun_requests}")
+
+    def _submit_rerun_requests(self) -> None:
+        for rerun_request in self.rerun_requests:
+            self._submit_rerun(rerun_request)
+
+    def _submit_rerun(self, rerun_request: dict) -> None:
+        base_job_link = self._extract_base_job_link_from_ci_link(
+            rerun_request["ci_link"]
+        )
+        if base_job_link:
+            family = rerun_request["family"]
+            if family == "deb":
+                self._submit_deb_rerun(base_job_link)
+            elif family == "snap":
+                self._submit_snap_rerun(base_job_link)
+            else:
+                logging.error(f"Invalid family name {family}")
+
+    def _extract_base_job_link_from_ci_link(self, ci_link: str) -> str | None:
+        matching = re.match(r"(.+/)\d+/", ci_link)
+        if matching:
+            return matching.group(1)
+        return None
+
+    def _submit_snap_rerun(self, base_job_link: str) -> None:
+        rerun_link = f"{base_job_link}/build"
+        logging.info(f"POST {rerun_link}")
+        requests.post(rerun_link, auth=self.jenkins_auth)
+
+    def _submit_deb_rerun(self, base_job_link: str) -> None:
+        rerun_link = f"{base_job_link}/buildWithParameters"
+        data = {"TESTPLAN": "full"}
+        logging.info(f"POST {rerun_link} {data}")
+        requests.post(rerun_link, auth=self.jenkins_auth, json=data)

backend/test_observer/controllers/artefacts/artefacts.py

@@ -21,7 +21,7 @@
 from sqlalchemy.orm import Session, joinedload
 
 from test_observer.data_access import queries
-from test_observer.data_access.models import Artefact, ArtefactBuild
+from test_observer.data_access.models import Artefact, ArtefactBuild, TestExecution
 from test_observer.data_access.models_enums import ArtefactStatus, FamilyName
 from test_observer.data_access.repository import get_artefacts_by_family
 from test_observer.data_access.setup import get_db
@@ -123,7 +123,11 @@ def get_artefact_builds(artefact_id: int, db: Session = Depends(get_db)):
         db.scalars(
             queries.latest_artefact_builds.where(
                 ArtefactBuild.artefact_id == artefact_id
-            ).options(joinedload(ArtefactBuild.test_executions))
+            ).options(
+                joinedload(ArtefactBuild.test_executions).joinedload(
+                    TestExecution.rerun_request
+                )
+            )
         ).unique()
     )
 

backend/test_observer/controllers/artefacts/models.py

@@ -18,8 +18,9 @@
 #        Omar Selo <[email protected]>
 #        Nadzeya Hutsko <[email protected]>
 from datetime import date
+from typing import Any
 
-from pydantic import AliasPath, BaseModel, ConfigDict, Field
+from pydantic import AliasPath, BaseModel, ConfigDict, Field, computed_field
 
 from test_observer.data_access.models_enums import (
     ArtefactStatus,
@@ -74,6 +75,11 @@ class TestExecutionDTO(BaseModel):
     # reasons, we allow multiple reasons to be picked for the approval
     review_decision: set[TestExecutionReviewDecision]
     review_comment: str
+    rerun_request: Any = Field(exclude=True)
+
+    @computed_field
+    def is_rerun_requested(self) -> bool:
+        return bool(self.rerun_request)
 
 
 class ArtefactBuildDTO(BaseModel):

backend/test_observer/controllers/test_executions/__init__.py

@@ -16,10 +16,11 @@
 
 from fastapi import APIRouter
 
-from . import end_test, get_test_results, patch, start_test
+from . import end_test, get_test_results, patch, reruns, start_test
 
 router = APIRouter(tags=["test-executions"])
 router.include_router(start_test.router)
 router.include_router(get_test_results.router)
 router.include_router(end_test.router)
 router.include_router(patch.router)
+router.include_router(reruns.router)

backend/test_observer/controllers/test_executions/logic.py

@@ -40,6 +40,8 @@ def reset_test_execution(
     test_execution.c3_link = None
     test_execution.review_decision = []
     test_execution.review_comment = ""
+    if test_execution.rerun_request:
+        db.delete(test_execution.rerun_request)
     db.commit()
 
     delete_previous_results(db, test_execution)

backend/test_observer/controllers/test_executions/models.py

@@ -154,3 +154,17 @@ class TestResultDTO(BaseModel):
             "the last one is the oldest one."
         ),
     )
+
+
+class RerunRequest(BaseModel):
+    test_execution_id: int
+
+
+class PendingRerun(BaseModel):
+    test_execution_id: int
+    ci_link: str = Field(validation_alias=AliasPath("test_execution", "ci_link"))
+    family: FamilyName = Field(
+        validation_alias=AliasPath(
+            "test_execution", "artefact_build", "artefact", "stage", "family", "name"
+        )
+    )

backend/test_observer/controllers/test_executions/reruns.py

@@ -0,0 +1,40 @@
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy import select
+from sqlalchemy.orm import Session, joinedload
+
+from test_observer.data_access.models import (
+    Artefact,
+    ArtefactBuild,
+    Stage,
+    TestExecution,
+    TestExecutionRerunRequest,
+)
+from test_observer.data_access.repository import get_or_create
+from test_observer.data_access.setup import get_db
+
+from .models import PendingRerun, RerunRequest
+
+router = APIRouter()
+
+
+@router.post("/reruns")
+def create_a_rerun_request(request: RerunRequest, db: Session = Depends(get_db)):
+    te = db.get(TestExecution, request.test_execution_id)
+    if not te:
+        msg = f"No test execution with id {request.test_execution_id} found"
+        raise HTTPException(status_code=404, detail=msg)
+
+    get_or_create(db, TestExecutionRerunRequest, {"test_execution_id": te.id})
+
+
+@router.get("/reruns", response_model=list[PendingRerun])
+def get_rerun_requests(db: Session = Depends(get_db)):
+    return db.scalars(
+        select(TestExecutionRerunRequest).options(
+            joinedload(TestExecutionRerunRequest.test_execution)
+            .joinedload(TestExecution.artefact_build)
+            .joinedload(ArtefactBuild.artefact)
+            .joinedload(Artefact.stage)
+            .joinedload(Stage.family)
+        )
+    )

backend/test_observer/data_access/models.py

@@ -266,6 +266,26 @@ def __repr__(self) -> str:
         return data_model_repr(self, "name", "architecture")
 
 
+class TestExecutionRerunRequest(Base):
+    """
+    Stores requests to rerun test executions.
+
+    Reason for this being a separate table is to make fetching all such
+    requests fast. Had we stored this as a column in TestExecution table then
+    we'd need to scan all test executions to get this list.
+    """
+
+    __test__ = False
+    __tablename__ = "test_execution_rerun_request"
+
+    test_execution_id: Mapped[int] = mapped_column(
+        ForeignKey("test_execution.id"), unique=True
+    )
+    test_execution: Mapped["TestExecution"] = relationship(
+        back_populates="rerun_request"
+    )
+
+
 class TestExecution(Base):
     """
     A table to represent the result of test execution.
@@ -289,6 +309,9 @@ class TestExecution(Base):
     test_results: Mapped[list["TestResult"]] = relationship(
         back_populates="test_execution", cascade="all, delete"
     )
+    rerun_request: Mapped[TestExecutionRerunRequest | None] = relationship(
+        back_populates="test_execution", cascade="all, delete"
+    )
     # Default fields
     status: Mapped[TestExecutionStatus] = mapped_column(
         default=TestExecutionStatus.NOT_STARTED

backend/test_observer/data_access/repository.py

@@ -69,7 +69,7 @@ def get_artefacts_by_family(
     :return: list of Artefacts
     """
     if latest_only:
-        subquery = (
+        base_query = (
             session.query(
                 Artefact.stage_id,
                 Artefact.name,
@@ -83,7 +83,9 @@ def get_artefacts_by_family(
 
         if family_name == FamilyName.SNAP:
             subquery = (
-                subquery.add_columns(Artefact.track).group_by(Artefact.track).subquery()
+                base_query.add_columns(Artefact.track)
+                .group_by(Artefact.track)
+                .subquery()
             )
 
             query = session.query(Artefact).join(
@@ -97,7 +99,7 @@ def get_artefacts_by_family(
             )
         else:
             subquery = (
-                subquery.add_columns(Artefact.repo, Artefact.series)
+                base_query.add_columns(Artefact.repo, Artefact.series)
                 .group_by(Artefact.repo, Artefact.series)
                 .subquery()
             )