From a48a351b24c1235013c9106452a3c79f6e6466fc Mon Sep 17 00:00:00 2001 From: Omar Abou Selo Date: Fri, 10 May 2024 09:42:35 +0300 Subject: [PATCH] Promote celery task (#180) * Make promoter a function instead of an endpoint * Add celery task to run promote script regularly --- backend/tasks/celery.py | 7 ++++++ backend/test_observer/controllers/router.py | 2 -- .../promoter => promotion}/__init__.py | 0 .../promoter => promotion}/promoter.py | 22 ++++++------------- .../tests/controllers/promoter/__init__.py | 0 .../promoter => promotion}/test_promoter.py | 8 +++---- 6 files changed, 17 insertions(+), 22 deletions(-) rename backend/test_observer/{controllers/promoter => promotion}/__init__.py (100%) rename backend/test_observer/{controllers/promoter => promotion}/promoter.py (92%) delete mode 100644 backend/tests/controllers/promoter/__init__.py rename backend/tests/{controllers/promoter => promotion}/test_promoter.py (96%) diff --git a/backend/tasks/celery.py b/backend/tasks/celery.py index a94ce499..4ae2016a 100644 --- a/backend/tasks/celery.py +++ b/backend/tasks/celery.py @@ -8,6 +8,7 @@ update_artefacts_with_tracker_info, ) from test_observer.kernel_swm_integration.swm_reader import get_artefacts_swm_info +from test_observer.promotion.promoter import promote_artefacts DEVELOPMENT_BROKER_URL = "redis://test-observer-redis" broker_url = environ.get("CELERY_BROKER_URL", DEVELOPMENT_BROKER_URL) @@ -20,6 +21,7 @@ @app.on_after_configure.connect def setup_periodic_tasks(sender, **kwargs): # noqa sender.add_periodic_task(300, integrate_with_kernel_swm.s()) + sender.add_periodic_task(600, run_promote_artefacts.s()) @app.task @@ -29,5 +31,10 @@ def integrate_with_kernel_swm(): update_artefacts_with_tracker_info(db, swm_info) +@app.task +def run_promote_artefacts(): + promote_artefacts(SessionLocal()) + + if __name__ == "__main__": app.start() diff --git a/backend/test_observer/controllers/router.py b/backend/test_observer/controllers/router.py index b811fa25..1fa473ea 100644 --- a/backend/test_observer/controllers/router.py +++ b/backend/test_observer/controllers/router.py @@ -26,11 +26,9 @@ from . import test_executions from .application import version from .artefacts import artefacts -from .promoter import promoter from .reports import reports router = APIRouter() -router.include_router(promoter.router) router.include_router(version.router, prefix="/v1/version") router.include_router(test_executions.router, prefix="/v1/test-executions") router.include_router(artefacts.router, prefix="/v1/artefacts") diff --git a/backend/test_observer/controllers/promoter/__init__.py b/backend/test_observer/promotion/__init__.py similarity index 100% rename from backend/test_observer/controllers/promoter/__init__.py rename to backend/test_observer/promotion/__init__.py diff --git a/backend/test_observer/controllers/promoter/promoter.py b/backend/test_observer/promotion/promoter.py similarity index 92% rename from backend/test_observer/controllers/promoter/promoter.py rename to backend/test_observer/promotion/promoter.py index 645c3c41..23beb7fe 100644 --- a/backend/test_observer/controllers/promoter/promoter.py +++ b/backend/test_observer/promotion/promoter.py @@ -21,8 +21,6 @@ import logging -from fastapi import APIRouter, Depends -from fastapi.responses import JSONResponse from sqlalchemy.orm import Session from test_observer.data_access import queries @@ -32,14 +30,11 @@ get_artefacts_by_family, get_stage_by_name, ) -from test_observer.data_access.setup import get_db from test_observer.external_apis.archive import ArchiveManager from test_observer.external_apis.snapcraft import ( get_channel_map_from_snapcraft, ) -router = APIRouter() - logger = logging.getLogger("test-observer-backend") CHANNEL_PROMOTION_MAP = { @@ -57,8 +52,7 @@ } -@router.put("/v0/artefacts/promote") -def promote_artefacts(db: Session = Depends(get_db)): +def promote_artefacts(db: Session): """ Promote all the artefacts in all the families if it has been updated on the external source @@ -70,23 +64,21 @@ def promote_artefacts(db: Session = Depends(get_db)): ) = promoter_controller(db) logger.info("INFO: Processed artefacts %s", processed_artefacts_status) if False in processed_artefacts_status.values(): - return JSONResponse( - status_code=500, - content={ + logger.error( + { artefact_key: processed_artefacts_error_messages[artefact_key] for ( artefact_key, artefact_status, ) in processed_artefacts_status.items() if artefact_status is False - }, + } ) - return JSONResponse( - status_code=200, - content={"detail": "All the artefacts have been processed successfully"}, + return logger.info( + {"detail": "All the artefacts have been processed successfully"} ) except Exception as exc: - return JSONResponse(status_code=500, content={"detail": str(exc)}) + return logger.error({"detail": str(exc)}) def promoter_controller(session: Session) -> tuple[dict, dict]: diff --git a/backend/tests/controllers/promoter/__init__.py b/backend/tests/controllers/promoter/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/backend/tests/controllers/promoter/test_promoter.py b/backend/tests/promotion/test_promoter.py similarity index 96% rename from backend/tests/controllers/promoter/test_promoter.py rename to backend/tests/promotion/test_promoter.py index e4a5e129..edffb5ad 100644 --- a/backend/tests/controllers/promoter/test_promoter.py +++ b/backend/tests/promotion/test_promoter.py @@ -22,17 +22,16 @@ from datetime import datetime, timedelta -from fastapi.testclient import TestClient from requests_mock import Mocker from sqlalchemy.orm import Session from test_observer.data_access.models import ArtefactBuild +from test_observer.promotion.promoter import promote_artefacts from tests.data_generator import DataGenerator def test_run_to_move_artefact_snap( db_session: Session, - test_client: TestClient, requests_mock: Mocker, generator: DataGenerator, ): @@ -92,7 +91,7 @@ def test_run_to_move_artefact_snap( ) # Act - test_client.put("/v0/artefacts/promote") + promote_artefacts(db_session) db_session.refresh(artefact) @@ -102,7 +101,6 @@ def test_run_to_move_artefact_snap( def test_run_to_move_artefact_deb( db_session: Session, - test_client: TestClient, requests_mock: Mocker, generator: DataGenerator, ): @@ -161,7 +159,7 @@ def test_run_to_move_artefact_deb( ) # Act - test_client.put("/v0/artefacts/promote") + promote_artefacts(db_session) db_session.refresh(artefact1)