From 3bd4e3b7872a84c9af94b3e0841c4ac3a520965f Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Sun, 1 Dec 2024 15:16:32 +0100 Subject: [PATCH 01/26] refactor RUT to use new transactional context --- .../api/rest/dependencies.py | 14 - .../api/rpc/_resource_tracker.py | 27 +- ...ackground_task_periodic_heartbeat_check.py | 37 +- .../services/credit_transactions.py | 23 +- .../modules/db/credit_transactions_db.py | 162 ++ .../services/modules/db/pricing_plans_db.py | 668 ++++++++ .../modules/db/repositories/__init__.py | 3 - .../services/modules/db/repositories/_base.py | 12 - .../db/repositories/resource_tracker.py | 1382 ----------------- .../services/modules/db/service_runs_db.py | 621 ++++++++ .../services/pricing_plans.py | 91 +- .../services/pricing_units.py | 61 +- .../process_message_running_service.py | 63 +- .../services/service_runs.py | 55 +- .../services/utils.py | 36 +- ...i_resource_tracker_service_runs__export.py | 2 +- ...ackground_task_periodic_heartbeat_check.py | 6 - .../with_dbs/test_process_rabbitmq_message.py | 12 +- ...t_process_rabbitmq_message_with_billing.py | 13 +- ...ss_rabbitmq_message_with_billing_cost_0.py | 13 +- 20 files changed, 1664 insertions(+), 1637 deletions(-) create mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/credit_transactions_db.py create mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/pricing_plans_db.py delete mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/__init__.py delete mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/_base.py delete mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py create mode 100644 services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/service_runs_db.py diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/dependencies.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/dependencies.py index 49ce9523cfe..dacf0ff08b5 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/dependencies.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rest/dependencies.py @@ -4,16 +4,11 @@ # import logging -from collections.abc import AsyncGenerator, Callable -from typing import Annotated -from fastapi import Depends from fastapi.requests import Request from servicelib.fastapi.dependencies import get_app, get_reverse_url_mapper from sqlalchemy.ext.asyncio import AsyncEngine -from ...services.modules.db.repositories._base import BaseRepository - logger = logging.getLogger(__name__) @@ -23,15 +18,6 @@ def get_resource_tracker_db_engine(request: Request) -> AsyncEngine: return engine -def get_repository(repo_type: type[BaseRepository]) -> Callable: - async def _get_repo( - engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], - ) -> AsyncGenerator[BaseRepository, None]: - yield repo_type(db_engine=engine) - - return _get_repo - - assert get_reverse_url_mapper # nosec assert get_app # nosec diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py index d7e9a5ca74d..5a382782f9d 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py @@ -29,9 +29,6 @@ from ...core.settings import ApplicationSettings from ...services import pricing_plans, pricing_units, service_runs -from ...services.modules.db.repositories.resource_tracker import ( - ResourceTrackerRepository, -) from ...services.modules.s3 import get_s3_client router = RPCRouter() @@ -56,7 +53,7 @@ async def get_service_run_page( return await service_runs.list_service_runs( user_id=user_id, product_name=product_name, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, limit=limit, offset=offset, wallet_id=wallet_id, @@ -87,7 +84,7 @@ async def export_service_runs( s3_region=s3_settings.S3_REGION, user_id=user_id, product_name=product_name, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, wallet_id=wallet_id, access_all_wallet_usage=access_all_wallet_usage, order_by=order_by, @@ -111,7 +108,7 @@ async def get_osparc_credits_aggregated_usages_page( return await service_runs.get_osparc_credits_aggregated_usages_page( user_id=user_id, product_name=product_name, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, aggregated_by=aggregated_by, time_period=time_period, limit=limit, @@ -134,7 +131,7 @@ async def get_pricing_plan( return await pricing_plans.get_pricing_plan( product_name=product_name, pricing_plan_id=pricing_plan_id, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, ) @@ -146,7 +143,7 @@ async def list_pricing_plans( ) -> list[PricingPlanGet]: return await pricing_plans.list_pricing_plans_by_product( product_name=product_name, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, ) @@ -158,7 +155,7 @@ async def create_pricing_plan( ) -> PricingPlanGet: return await pricing_plans.create_pricing_plan( data=data, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, ) @@ -172,7 +169,7 @@ async def update_pricing_plan( return await pricing_plans.update_pricing_plan( product_name=product_name, data=data, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, ) @@ -191,7 +188,7 @@ async def get_pricing_unit( product_name=product_name, pricing_plan_id=pricing_plan_id, pricing_unit_id=pricing_unit_id, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, ) @@ -205,7 +202,7 @@ async def create_pricing_unit( return await pricing_units.create_pricing_unit( product_name=product_name, data=data, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, ) @@ -219,7 +216,7 @@ async def update_pricing_unit( return await pricing_units.update_pricing_unit( product_name=product_name, data=data, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, ) @@ -238,7 +235,7 @@ async def list_connected_services_to_pricing_plan_by_pricing_plan( ] = await pricing_plans.list_connected_services_to_pricing_plan_by_pricing_plan( product_name=product_name, pricing_plan_id=pricing_plan_id, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, ) return output @@ -257,5 +254,5 @@ async def connect_service_to_pricing_plan( pricing_plan_id=pricing_plan_id, service_key=service_key, service_version=service_version, - resource_tracker_repo=ResourceTrackerRepository(db_engine=app.state.engine), + db_engine=app.state.engine, ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check.py index 256b737d479..fba9332502e 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/background_task_periodic_heartbeat_check.py @@ -10,11 +10,12 @@ ServiceRunStatus, ) from pydantic import NonNegativeInt, PositiveInt +from sqlalchemy.ext.asyncio import AsyncEngine from ..core.settings import ApplicationSettings from ..models.credit_transactions import CreditTransactionCreditsAndStatusUpdate from ..models.service_runs import ServiceRunStoppedAtUpdate -from .modules.db.repositories.resource_tracker import ResourceTrackerRepository +from .modules.db import credit_transactions_db, service_runs_db from .utils import compute_service_run_credit_costs, make_negative _logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ async def _check_service_heartbeat( - resource_tracker_repo: ResourceTrackerRepository, + db_engine: AsyncEngine, base_start_timestamp: datetime, resource_usage_tracker_missed_heartbeat_interval: timedelta, resource_usage_tracker_missed_heartbeat_counter_fail: NonNegativeInt, @@ -55,7 +56,7 @@ async def _check_service_heartbeat( missed_heartbeat_counter, ) await _close_unhealthy_service( - resource_tracker_repo, service_run_id, base_start_timestamp + db_engine, service_run_id, base_start_timestamp ) else: _logger.warning( @@ -63,13 +64,16 @@ async def _check_service_heartbeat( service_run_id, missed_heartbeat_counter, ) - await resource_tracker_repo.update_service_missed_heartbeat_counter( - service_run_id, last_heartbeat_at, missed_heartbeat_counter + await service_runs_db.update_service_missed_heartbeat_counter( + db_engine, + service_run_id=service_run_id, + last_heartbeat_at=last_heartbeat_at, + missed_heartbeat_counter=missed_heartbeat_counter, ) async def _close_unhealthy_service( - resource_tracker_repo: ResourceTrackerRepository, + db_engine: AsyncEngine, service_run_id: ServiceRunId, base_start_timestamp: datetime, ): @@ -80,8 +84,8 @@ async def _close_unhealthy_service( service_run_status=ServiceRunStatus.ERROR, service_run_status_msg="Service missed more heartbeats. It's considered unhealthy.", ) - running_service = await resource_tracker_repo.update_service_run_stopped_at( - update_service_run_stopped_at + running_service = await service_runs_db.update_service_run_stopped_at( + db_engine, data=update_service_run_stopped_at ) if running_service is None: @@ -108,8 +112,8 @@ async def _close_unhealthy_service( else CreditTransactionStatus.BILLED ), ) - await resource_tracker_repo.update_credit_transaction_credits_and_status( - update_credit_transaction + await credit_transactions_db.update_credit_transaction_credits_and_status( + db_engine, data=update_credit_transaction ) @@ -118,19 +122,18 @@ async def periodic_check_of_running_services_task(app: FastAPI) -> None: # This check runs across all products app_settings: ApplicationSettings = app.state.settings - resource_tracker_repo: ResourceTrackerRepository = ResourceTrackerRepository( - db_engine=app.state.engine - ) + _db_engine = app.state.engine base_start_timestamp = datetime.now(tz=timezone.utc) # Get all current running services (across all products) - total_count: PositiveInt = ( - await resource_tracker_repo.total_service_runs_with_running_status_across_all_products() + total_count: PositiveInt = await service_runs_db.total_service_runs_with_running_status_across_all_products( + _db_engine ) for offset in range(0, total_count, _BATCH_SIZE): - batch_check_services = await resource_tracker_repo.list_service_runs_with_running_status_across_all_products( + batch_check_services = await service_runs_db.list_service_runs_with_running_status_across_all_products( + _db_engine, offset=offset, limit=_BATCH_SIZE, ) @@ -138,7 +141,7 @@ async def periodic_check_of_running_services_task(app: FastAPI) -> None: await asyncio.gather( *( _check_service_heartbeat( - resource_tracker_repo=resource_tracker_repo, + db_engine=_db_engine, base_start_timestamp=base_start_timestamp, resource_usage_tracker_missed_heartbeat_interval=app_settings.RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC, resource_usage_tracker_missed_heartbeat_counter_fail=app_settings.RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/credit_transactions.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/credit_transactions.py index 0d4362e9748..c58eb76be8a 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/credit_transactions.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/credit_transactions.py @@ -13,19 +13,18 @@ ) from models_library.wallets import WalletID from servicelib.rabbitmq import RabbitMQClient +from sqlalchemy.ext.asyncio import AsyncEngine -from ..api.rest.dependencies import get_repository +from ..api.rest.dependencies import get_resource_tracker_db_engine from ..models.credit_transactions import CreditTransactionCreate -from .modules.db.repositories.resource_tracker import ResourceTrackerRepository +from .modules.db import credit_transactions_db from .modules.rabbitmq import get_rabbitmq_client_from_request from .utils import sum_credit_transactions_and_publish_to_rabbitmq async def create_credit_transaction( credit_transaction_create_body: CreditTransactionCreateBody, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], rabbitmq_client: Annotated[ RabbitMQClient, Depends(get_rabbitmq_client_from_request) ], @@ -47,12 +46,12 @@ async def create_credit_transaction( created_at=credit_transaction_create_body.created_at, last_heartbeat_at=credit_transaction_create_body.created_at, ) - transaction_id = await resource_tracker_repo.create_credit_transaction( - transaction_create + transaction_id = await credit_transactions_db.create_credit_transaction( + db_engine, data=transaction_create ) await sum_credit_transactions_and_publish_to_rabbitmq( - resource_tracker_repo, + db_engine, rabbitmq_client, credit_transaction_create_body.product_name, credit_transaction_create_body.wallet_id, @@ -64,10 +63,8 @@ async def create_credit_transaction( async def sum_credit_transactions_by_product_and_wallet( product_name: ProductName, wallet_id: WalletID, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], ) -> WalletTotalCredits: - return await resource_tracker_repo.sum_credit_transactions_by_product_and_wallet( - product_name, wallet_id + return await credit_transactions_db.sum_credit_transactions_by_product_and_wallet( + db_engine, product_name=product_name, wallet_id=wallet_id ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/credit_transactions_db.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/credit_transactions_db.py new file mode 100644 index 00000000000..76a8e9f1dfe --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/credit_transactions_db.py @@ -0,0 +1,162 @@ +import logging +from decimal import Decimal +from typing import cast + +import sqlalchemy as sa +from models_library.api_schemas_resource_usage_tracker.credit_transactions import ( + WalletTotalCredits, +) +from models_library.products import ProductName +from models_library.resource_tracker import CreditTransactionId, CreditTransactionStatus +from models_library.wallets import WalletID +from simcore_postgres_database.models.resource_tracker_credit_transactions import ( + resource_tracker_credit_transactions, +) +from simcore_postgres_database.utils_repos import transaction_context +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine + +from ....exceptions.errors import CreditTransactionNotCreatedDBError +from ....models.credit_transactions import ( + CreditTransactionCreate, + CreditTransactionCreditsAndStatusUpdate, + CreditTransactionCreditsUpdate, +) + +_logger = logging.getLogger(__name__) + + +async def create_credit_transaction( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + data: CreditTransactionCreate +) -> CreditTransactionId: + async with transaction_context(engine, connection) as conn: + insert_stmt = ( + resource_tracker_credit_transactions.insert() + .values( + product_name=data.product_name, + wallet_id=data.wallet_id, + wallet_name=data.wallet_name, + pricing_plan_id=data.pricing_plan_id, + pricing_unit_id=data.pricing_unit_id, + pricing_unit_cost_id=data.pricing_unit_cost_id, + user_id=data.user_id, + user_email=data.user_email, + osparc_credits=data.osparc_credits, + transaction_status=data.transaction_status, + transaction_classification=data.transaction_classification, + service_run_id=data.service_run_id, + payment_transaction_id=data.payment_transaction_id, + created=data.created_at, + last_heartbeat_at=data.last_heartbeat_at, + modified=sa.func.now(), + ) + .returning(resource_tracker_credit_transactions.c.transaction_id) + ) + result = await conn.execute(insert_stmt) + row = result.first() + if row is None: + raise CreditTransactionNotCreatedDBError(data=data) + return cast(CreditTransactionId, row[0]) + + +async def update_credit_transaction_credits( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + data: CreditTransactionCreditsUpdate +) -> CreditTransactionId | None: + async with transaction_context(engine, connection) as conn: + update_stmt = ( + resource_tracker_credit_transactions.update() + .values( + modified=sa.func.now(), + osparc_credits=data.osparc_credits, + last_heartbeat_at=data.last_heartbeat_at, + ) + .where( + ( + resource_tracker_credit_transactions.c.service_run_id + == data.service_run_id + ) + & ( + resource_tracker_credit_transactions.c.transaction_status + == CreditTransactionStatus.PENDING + ) + & ( + resource_tracker_credit_transactions.c.last_heartbeat_at + <= data.last_heartbeat_at + ) + ) + .returning(resource_tracker_credit_transactions.c.service_run_id) + ) + result = await conn.execute(update_stmt) + row = result.first() + if row is None: + return None + return cast(CreditTransactionId | None, row[0]) + + +async def update_credit_transaction_credits_and_status( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + data: CreditTransactionCreditsAndStatusUpdate +) -> CreditTransactionId | None: + async with transaction_context(engine, connection) as conn: + update_stmt = ( + resource_tracker_credit_transactions.update() + .values( + modified=sa.func.now(), + osparc_credits=data.osparc_credits, + transaction_status=data.transaction_status, + ) + .where( + ( + resource_tracker_credit_transactions.c.service_run_id + == data.service_run_id + ) + & ( + resource_tracker_credit_transactions.c.transaction_status + == CreditTransactionStatus.PENDING + ) + ) + .returning(resource_tracker_credit_transactions.c.service_run_id) + ) + result = await conn.execute(update_stmt) + row = result.first() + if row is None: + return None + return cast(CreditTransactionId | None, row[0]) + + +async def sum_credit_transactions_by_product_and_wallet( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + wallet_id: WalletID +) -> WalletTotalCredits: + async with transaction_context(engine, connection) as conn: + sum_stmt = sa.select( + sa.func.sum(resource_tracker_credit_transactions.c.osparc_credits) + ).where( + (resource_tracker_credit_transactions.c.product_name == product_name) + & (resource_tracker_credit_transactions.c.wallet_id == wallet_id) + & ( + resource_tracker_credit_transactions.c.transaction_status.in_( + [ + CreditTransactionStatus.BILLED, + CreditTransactionStatus.PENDING, + ] + ) + ) + ) + result = await conn.execute(sum_stmt) + row = result.first() + if row is None or row[0] is None: + return WalletTotalCredits( + wallet_id=wallet_id, available_osparc_credits=Decimal(0) + ) + return WalletTotalCredits(wallet_id=wallet_id, available_osparc_credits=row[0]) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/pricing_plans_db.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/pricing_plans_db.py new file mode 100644 index 00000000000..ea6376cc15b --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/pricing_plans_db.py @@ -0,0 +1,668 @@ +import logging + +import sqlalchemy as sa +from models_library.products import ProductName +from models_library.resource_tracker import ( + PricingPlanCreate, + PricingPlanId, + PricingPlanUpdate, + PricingUnitCostId, + PricingUnitId, + PricingUnitWithCostCreate, + PricingUnitWithCostUpdate, +) +from models_library.services import ServiceKey, ServiceVersion +from simcore_postgres_database.models.resource_tracker_pricing_plan_to_service import ( + resource_tracker_pricing_plan_to_service, +) +from simcore_postgres_database.models.resource_tracker_pricing_plans import ( + resource_tracker_pricing_plans, +) +from simcore_postgres_database.models.resource_tracker_pricing_unit_costs import ( + resource_tracker_pricing_unit_costs, +) +from simcore_postgres_database.models.resource_tracker_pricing_units import ( + resource_tracker_pricing_units, +) +from simcore_postgres_database.utils_repos import transaction_context +from sqlalchemy.dialects.postgresql import ARRAY, INTEGER +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine + +from ....exceptions.errors import ( + PricingPlanAndPricingUnitCombinationDoesNotExistsDBError, + PricingPlanDoesNotExistsDBError, + PricingPlanNotCreatedDBError, + PricingPlanToServiceNotCreatedDBError, + PricingUnitCostDoesNotExistsDBError, + PricingUnitCostNotCreatedDBError, + PricingUnitNotCreatedDBError, +) +from ....models.pricing_plans import ( + PricingPlansDB, + PricingPlansWithServiceDefaultPlanDB, + PricingPlanToServiceDB, +) +from ....models.pricing_unit_costs import PricingUnitCostsDB +from ....models.pricing_units import PricingUnitsDB + +_logger = logging.getLogger(__name__) + + +################################# +# Pricing plans +################################# + + +async def list_active_service_pricing_plans_by_product_and_service( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + service_key: ServiceKey, + service_version: ServiceVersion, +) -> list[PricingPlansWithServiceDefaultPlanDB]: + # NOTE: consilidate with utils_services_environmnets.py + def _version(column_or_value): + # converts version value string to array[integer] that can be compared + return sa.func.string_to_array(column_or_value, ".").cast(ARRAY(INTEGER)) + + async with transaction_context(engine, connection) as conn: + # Firstly find the correct service version + query = ( + sa.select( + resource_tracker_pricing_plan_to_service.c.service_key, + resource_tracker_pricing_plan_to_service.c.service_version, + ) + .select_from( + resource_tracker_pricing_plan_to_service.join( + resource_tracker_pricing_plans, + ( + resource_tracker_pricing_plan_to_service.c.pricing_plan_id + == resource_tracker_pricing_plans.c.pricing_plan_id + ), + ) + ) + .where( + ( + _version(resource_tracker_pricing_plan_to_service.c.service_version) + <= _version(service_version) + ) + & ( + resource_tracker_pricing_plan_to_service.c.service_key + == service_key + ) + & (resource_tracker_pricing_plans.c.product_name == product_name) + & (resource_tracker_pricing_plans.c.is_active.is_(True)) + ) + .order_by( + _version( + resource_tracker_pricing_plan_to_service.c.service_version + ).desc() + ) + .limit(1) + ) + result = await conn.execute(query) + row = result.first() + if row is None: + return [] + latest_service_key, latest_service_version = row + # Now choose all pricing plans connected to this service + query = ( + sa.select( + resource_tracker_pricing_plans.c.pricing_plan_id, + resource_tracker_pricing_plans.c.display_name, + resource_tracker_pricing_plans.c.description, + resource_tracker_pricing_plans.c.classification, + resource_tracker_pricing_plans.c.is_active, + resource_tracker_pricing_plans.c.created, + resource_tracker_pricing_plans.c.pricing_plan_key, + resource_tracker_pricing_plan_to_service.c.service_default_plan, + ) + .select_from( + resource_tracker_pricing_plan_to_service.join( + resource_tracker_pricing_plans, + ( + resource_tracker_pricing_plan_to_service.c.pricing_plan_id + == resource_tracker_pricing_plans.c.pricing_plan_id + ), + ) + ) + .where( + ( + _version(resource_tracker_pricing_plan_to_service.c.service_version) + == _version(latest_service_version) + ) + & ( + resource_tracker_pricing_plan_to_service.c.service_key + == latest_service_key + ) + & (resource_tracker_pricing_plans.c.product_name == product_name) + & (resource_tracker_pricing_plans.c.is_active.is_(True)) + ) + .order_by(resource_tracker_pricing_plan_to_service.c.pricing_plan_id.desc()) + ) + result = await conn.execute(query) + + return [ + PricingPlansWithServiceDefaultPlanDB.model_validate(row) + for row in result.fetchall() + ] + + +async def get_pricing_plan( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + pricing_plan_id: PricingPlanId, +) -> PricingPlansDB: + async with transaction_context(engine, connection) as conn: + select_stmt = sa.select( + resource_tracker_pricing_plans.c.pricing_plan_id, + resource_tracker_pricing_plans.c.display_name, + resource_tracker_pricing_plans.c.description, + resource_tracker_pricing_plans.c.classification, + resource_tracker_pricing_plans.c.is_active, + resource_tracker_pricing_plans.c.created, + resource_tracker_pricing_plans.c.pricing_plan_key, + ).where( + (resource_tracker_pricing_plans.c.pricing_plan_id == pricing_plan_id) + & (resource_tracker_pricing_plans.c.product_name == product_name) + ) + result = await conn.execute(select_stmt) + row = result.first() + if row is None: + raise PricingPlanDoesNotExistsDBError(pricing_plan_id=pricing_plan_id) + return PricingPlansDB.model_validate(row) + + +async def list_pricing_plans_by_product( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, +) -> list[PricingPlansDB]: + async with transaction_context(engine, connection) as conn: + select_stmt = sa.select( + resource_tracker_pricing_plans.c.pricing_plan_id, + resource_tracker_pricing_plans.c.display_name, + resource_tracker_pricing_plans.c.description, + resource_tracker_pricing_plans.c.classification, + resource_tracker_pricing_plans.c.is_active, + resource_tracker_pricing_plans.c.created, + resource_tracker_pricing_plans.c.pricing_plan_key, + ).where(resource_tracker_pricing_plans.c.product_name == product_name) + result = await conn.execute(select_stmt) + + return [PricingPlansDB.model_validate(row) for row in result.fetchall()] + + +async def create_pricing_plan( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + data: PricingPlanCreate, +) -> PricingPlansDB: + async with transaction_context(engine, connection) as conn: + insert_stmt = ( + resource_tracker_pricing_plans.insert() + .values( + product_name=data.product_name, + display_name=data.display_name, + description=data.description, + classification=data.classification, + is_active=True, + created=sa.func.now(), + modified=sa.func.now(), + pricing_plan_key=data.pricing_plan_key, + ) + .returning( + *[ + resource_tracker_pricing_plans.c.pricing_plan_id, + resource_tracker_pricing_plans.c.display_name, + resource_tracker_pricing_plans.c.description, + resource_tracker_pricing_plans.c.classification, + resource_tracker_pricing_plans.c.is_active, + resource_tracker_pricing_plans.c.created, + resource_tracker_pricing_plans.c.pricing_plan_key, + ] + ) + ) + result = await conn.execute(insert_stmt) + row = result.first() + if row is None: + raise PricingPlanNotCreatedDBError(data=data) + return PricingPlansDB.model_validate(row) + + +async def update_pricing_plan( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + data: PricingPlanUpdate, +) -> PricingPlansDB | None: + async with transaction_context(engine, connection) as conn: + update_stmt = ( + resource_tracker_pricing_plans.update() + .values( + display_name=data.display_name, + description=data.description, + is_active=data.is_active, + modified=sa.func.now(), + ) + .where( + ( + resource_tracker_pricing_plans.c.pricing_plan_id + == data.pricing_plan_id + ) + & (resource_tracker_pricing_plans.c.product_name == product_name) + ) + .returning( + *[ + resource_tracker_pricing_plans.c.pricing_plan_id, + resource_tracker_pricing_plans.c.display_name, + resource_tracker_pricing_plans.c.description, + resource_tracker_pricing_plans.c.classification, + resource_tracker_pricing_plans.c.is_active, + resource_tracker_pricing_plans.c.created, + resource_tracker_pricing_plans.c.pricing_plan_key, + ] + ) + ) + result = await conn.execute(update_stmt) + row = result.first() + if row is None: + return None + return PricingPlansDB.model_validate(row) + + +################################# +# Pricing plan to service +################################# + + +async def list_connected_services_to_pricing_plan_by_pricing_plan( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + pricing_plan_id: PricingPlanId, +) -> list[PricingPlanToServiceDB]: + async with transaction_context(engine, connection) as conn: + query = ( + sa.select( + resource_tracker_pricing_plan_to_service.c.pricing_plan_id, + resource_tracker_pricing_plan_to_service.c.service_key, + resource_tracker_pricing_plan_to_service.c.service_version, + resource_tracker_pricing_plan_to_service.c.created, + ) + .select_from( + resource_tracker_pricing_plan_to_service.join( + resource_tracker_pricing_plans, + ( + resource_tracker_pricing_plan_to_service.c.pricing_plan_id + == resource_tracker_pricing_plans.c.pricing_plan_id + ), + ) + ) + .where( + (resource_tracker_pricing_plans.c.product_name == product_name) + & (resource_tracker_pricing_plans.c.pricing_plan_id == pricing_plan_id) + ) + .order_by(resource_tracker_pricing_plan_to_service.c.pricing_plan_id.desc()) + ) + result = await conn.execute(query) + + return [PricingPlanToServiceDB.model_validate(row) for row in result.fetchall()] + + +async def upsert_service_to_pricing_plan( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + pricing_plan_id: PricingPlanId, + service_key: ServiceKey, + service_version: ServiceVersion, +) -> PricingPlanToServiceDB: + async with transaction_context(engine, connection) as conn: + query = ( + sa.select( + resource_tracker_pricing_plan_to_service.c.pricing_plan_id, + resource_tracker_pricing_plan_to_service.c.service_key, + resource_tracker_pricing_plan_to_service.c.service_version, + resource_tracker_pricing_plan_to_service.c.created, + ) + .select_from( + resource_tracker_pricing_plan_to_service.join( + resource_tracker_pricing_plans, + ( + resource_tracker_pricing_plan_to_service.c.pricing_plan_id + == resource_tracker_pricing_plans.c.pricing_plan_id + ), + ) + ) + .where( + (resource_tracker_pricing_plans.c.product_name == product_name) + & (resource_tracker_pricing_plans.c.pricing_plan_id == pricing_plan_id) + & ( + resource_tracker_pricing_plan_to_service.c.service_key + == service_key + ) + & ( + resource_tracker_pricing_plan_to_service.c.service_version + == service_version + ) + ) + ) + result = await conn.execute(query) + row = result.first() + + if row is not None: + delete_stmt = resource_tracker_pricing_plan_to_service.delete().where( + (resource_tracker_pricing_plans.c.pricing_plan_id == pricing_plan_id) + & ( + resource_tracker_pricing_plan_to_service.c.service_key + == service_key + ) + & ( + resource_tracker_pricing_plan_to_service.c.service_version + == service_version + ) + ) + await conn.execute(delete_stmt) + + insert_stmt = ( + resource_tracker_pricing_plan_to_service.insert() + .values( + pricing_plan_id=pricing_plan_id, + service_key=service_key, + service_version=service_version, + created=sa.func.now(), + modified=sa.func.now(), + service_default_plan=True, + ) + .returning( + *[ + resource_tracker_pricing_plan_to_service.c.pricing_plan_id, + resource_tracker_pricing_plan_to_service.c.service_key, + resource_tracker_pricing_plan_to_service.c.service_version, + resource_tracker_pricing_plan_to_service.c.created, + ] + ) + ) + result = await conn.execute(insert_stmt) + row = result.first() + if row is None: + raise PricingPlanToServiceNotCreatedDBError( + data=f"pricing_plan_id {pricing_plan_id}, service_key {service_key}, service_version {service_version}" + ) + return PricingPlanToServiceDB.model_validate(row) + + +################################# +# Pricing units +################################# + + +def _pricing_units_select_stmt(): + return sa.select( + resource_tracker_pricing_units.c.pricing_unit_id, + resource_tracker_pricing_units.c.pricing_plan_id, + resource_tracker_pricing_units.c.unit_name, + resource_tracker_pricing_units.c.unit_extra_info, + resource_tracker_pricing_units.c.default, + resource_tracker_pricing_units.c.specific_info, + resource_tracker_pricing_units.c.created, + resource_tracker_pricing_units.c.modified, + resource_tracker_pricing_unit_costs.c.cost_per_unit.label( + "current_cost_per_unit" + ), + resource_tracker_pricing_unit_costs.c.pricing_unit_cost_id.label( + "current_cost_per_unit_id" + ), + ) + + +async def list_pricing_units_by_pricing_plan( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + pricing_plan_id: PricingPlanId, +) -> list[PricingUnitsDB]: + async with transaction_context(engine, connection) as conn: + query = ( + _pricing_units_select_stmt() + .select_from( + resource_tracker_pricing_units.join( + resource_tracker_pricing_unit_costs, + ( + ( + resource_tracker_pricing_units.c.pricing_plan_id + == resource_tracker_pricing_unit_costs.c.pricing_plan_id + ) + & ( + resource_tracker_pricing_units.c.pricing_unit_id + == resource_tracker_pricing_unit_costs.c.pricing_unit_id + ) + ), + ) + ) + .where( + (resource_tracker_pricing_units.c.pricing_plan_id == pricing_plan_id) + & (resource_tracker_pricing_unit_costs.c.valid_to.is_(None)) + ) + .order_by(resource_tracker_pricing_unit_costs.c.cost_per_unit.asc()) + ) + result = await conn.execute(query) + + return [PricingUnitsDB.model_validate(row) for row in result.fetchall()] + + +async def get_valid_pricing_unit( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + pricing_plan_id: PricingPlanId, + pricing_unit_id: PricingUnitId, +) -> PricingUnitsDB: + async with transaction_context(engine, connection) as conn: + query = ( + _pricing_units_select_stmt() + .select_from( + resource_tracker_pricing_units.join( + resource_tracker_pricing_unit_costs, + ( + ( + resource_tracker_pricing_units.c.pricing_plan_id + == resource_tracker_pricing_unit_costs.c.pricing_plan_id + ) + & ( + resource_tracker_pricing_units.c.pricing_unit_id + == resource_tracker_pricing_unit_costs.c.pricing_unit_id + ) + ), + ).join( + resource_tracker_pricing_plans, + ( + resource_tracker_pricing_plans.c.pricing_plan_id + == resource_tracker_pricing_units.c.pricing_plan_id + ), + ) + ) + .where( + (resource_tracker_pricing_units.c.pricing_plan_id == pricing_plan_id) + & (resource_tracker_pricing_units.c.pricing_unit_id == pricing_unit_id) + & (resource_tracker_pricing_unit_costs.c.valid_to.is_(None)) + & (resource_tracker_pricing_plans.c.product_name == product_name) + ) + ) + result = await conn.execute(query) + + row = result.first() + if row is None: + raise PricingPlanAndPricingUnitCombinationDoesNotExistsDBError( + pricing_plan_id=pricing_plan_id, + pricing_unit_id=pricing_unit_id, + product_name=product_name, + ) + return PricingUnitsDB.model_validate(row) + + +async def create_pricing_unit_with_cost( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + data: PricingUnitWithCostCreate, + pricing_plan_key: str, +) -> tuple[PricingUnitId, PricingUnitCostId]: + async with transaction_context(engine, connection) as conn: + # pricing units table + insert_stmt = ( + resource_tracker_pricing_units.insert() + .values( + pricing_plan_id=data.pricing_plan_id, + unit_name=data.unit_name, + unit_extra_info=data.unit_extra_info.model_dump(), + default=data.default, + specific_info=data.specific_info.model_dump(), + created=sa.func.now(), + modified=sa.func.now(), + ) + .returning(resource_tracker_pricing_units.c.pricing_unit_id) + ) + result = await conn.execute(insert_stmt) + row = result.first() + if row is None: + raise PricingUnitNotCreatedDBError(data=data) + _pricing_unit_id = row[0] + + # pricing unit cost table + insert_stmt = ( + resource_tracker_pricing_unit_costs.insert() + .values( + pricing_plan_id=data.pricing_plan_id, + pricing_plan_key=pricing_plan_key, + pricing_unit_id=_pricing_unit_id, + pricing_unit_name=data.unit_name, + cost_per_unit=data.cost_per_unit, + valid_from=sa.func.now(), + valid_to=None, + created=sa.func.now(), + comment=data.comment, + modified=sa.func.now(), + ) + .returning(resource_tracker_pricing_unit_costs.c.pricing_unit_cost_id) + ) + result = await conn.execute(insert_stmt) + row = result.first() + if row is None: + raise PricingUnitCostNotCreatedDBError(data=data) + _pricing_unit_cost_id = row[0] + + return (_pricing_unit_id, _pricing_unit_cost_id) + + +async def update_pricing_unit_with_cost( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + data: PricingUnitWithCostUpdate, + pricing_plan_key: str, +) -> None: + async with transaction_context(engine, connection) as conn: + # pricing units table + update_stmt = ( + resource_tracker_pricing_units.update() + .values( + unit_name=data.unit_name, + unit_extra_info=data.unit_extra_info.model_dump(), + default=data.default, + specific_info=data.specific_info.model_dump(), + modified=sa.func.now(), + ) + .where( + resource_tracker_pricing_units.c.pricing_unit_id == data.pricing_unit_id + ) + .returning(resource_tracker_pricing_units.c.pricing_unit_id) + ) + await conn.execute(update_stmt) + + # If price change, then we update pricing unit cost table + if data.pricing_unit_cost_update: + # Firstly we close previous price + update_stmt = ( + resource_tracker_pricing_unit_costs.update() + .values( + valid_to=sa.func.now(), # <-- Closing previous price + modified=sa.func.now(), + ) + .where( + resource_tracker_pricing_unit_costs.c.pricing_unit_id + == data.pricing_unit_id + ) + .returning(resource_tracker_pricing_unit_costs.c.pricing_unit_id) + ) + result = await conn.execute(update_stmt) + + # Then we create a new price + insert_stmt = ( + resource_tracker_pricing_unit_costs.insert() + .values( + pricing_plan_id=data.pricing_plan_id, + pricing_plan_key=pricing_plan_key, + pricing_unit_id=data.pricing_unit_id, + pricing_unit_name=data.unit_name, + cost_per_unit=data.pricing_unit_cost_update.cost_per_unit, + valid_from=sa.func.now(), + valid_to=None, # <-- New price is valid + created=sa.func.now(), + comment=data.pricing_unit_cost_update.comment, + modified=sa.func.now(), + ) + .returning(resource_tracker_pricing_unit_costs.c.pricing_unit_cost_id) + ) + result = await conn.execute(insert_stmt) + row = result.first() + if row is None: + raise PricingUnitCostNotCreatedDBError(data=data) + + +################################# +# Pricing unit-costs +################################# + + +async def get_pricing_unit_cost_by_id( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + pricing_unit_cost_id: PricingUnitCostId, +) -> PricingUnitCostsDB: + async with transaction_context(engine, connection) as conn: + query = sa.select( + resource_tracker_pricing_unit_costs.c.pricing_unit_cost_id, + resource_tracker_pricing_unit_costs.c.pricing_plan_id, + resource_tracker_pricing_unit_costs.c.pricing_plan_key, + resource_tracker_pricing_unit_costs.c.pricing_unit_id, + resource_tracker_pricing_unit_costs.c.pricing_unit_name, + resource_tracker_pricing_unit_costs.c.cost_per_unit, + resource_tracker_pricing_unit_costs.c.valid_from, + resource_tracker_pricing_unit_costs.c.valid_to, + resource_tracker_pricing_unit_costs.c.created, + resource_tracker_pricing_unit_costs.c.comment, + resource_tracker_pricing_unit_costs.c.modified, + ).where( + resource_tracker_pricing_unit_costs.c.pricing_unit_cost_id + == pricing_unit_cost_id + ) + result = await conn.execute(query) + + row = result.first() + if row is None: + raise PricingUnitCostDoesNotExistsDBError( + pricing_unit_cost_id=pricing_unit_cost_id + ) + return PricingUnitCostsDB.model_validate(row) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/__init__.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/__init__.py deleted file mode 100644 index 93da4003de3..00000000000 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from ._base import BaseRepository - -__all__: tuple[str, ...] = ("BaseRepository",) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/_base.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/_base.py deleted file mode 100644 index 4a20b37c735..00000000000 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/_base.py +++ /dev/null @@ -1,12 +0,0 @@ -from dataclasses import dataclass - -from sqlalchemy.ext.asyncio import AsyncEngine - - -@dataclass -class BaseRepository: - """ - Repositories are pulled at every request - """ - - db_engine: AsyncEngine diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py deleted file mode 100644 index 46439f26e38..00000000000 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/repositories/resource_tracker.py +++ /dev/null @@ -1,1382 +0,0 @@ -import logging -from datetime import datetime -from decimal import Decimal -from typing import cast - -import sqlalchemy as sa -from models_library.api_schemas_resource_usage_tracker.credit_transactions import ( - WalletTotalCredits, -) -from models_library.api_schemas_storage import S3BucketName -from models_library.products import ProductName -from models_library.resource_tracker import ( - CreditClassification, - CreditTransactionId, - CreditTransactionStatus, - PricingPlanCreate, - PricingPlanId, - PricingPlanUpdate, - PricingUnitCostId, - PricingUnitId, - PricingUnitWithCostCreate, - PricingUnitWithCostUpdate, - ServiceRunId, - ServiceRunStatus, -) -from models_library.rest_ordering import OrderBy, OrderDirection -from models_library.services import ServiceKey, ServiceVersion -from models_library.users import UserID -from models_library.wallets import WalletID -from pydantic import PositiveInt -from simcore_postgres_database.models.projects_tags import projects_tags -from simcore_postgres_database.models.resource_tracker_credit_transactions import ( - resource_tracker_credit_transactions, -) -from simcore_postgres_database.models.resource_tracker_pricing_plan_to_service import ( - resource_tracker_pricing_plan_to_service, -) -from simcore_postgres_database.models.resource_tracker_pricing_plans import ( - resource_tracker_pricing_plans, -) -from simcore_postgres_database.models.resource_tracker_pricing_unit_costs import ( - resource_tracker_pricing_unit_costs, -) -from simcore_postgres_database.models.resource_tracker_pricing_units import ( - resource_tracker_pricing_units, -) -from simcore_postgres_database.models.resource_tracker_service_runs import ( - resource_tracker_service_runs, -) -from simcore_postgres_database.models.tags import tags -from sqlalchemy.dialects.postgresql import ARRAY, INTEGER - -from .....exceptions.errors import ( - CreditTransactionNotCreatedDBError, - PricingPlanAndPricingUnitCombinationDoesNotExistsDBError, - PricingPlanDoesNotExistsDBError, - PricingPlanNotCreatedDBError, - PricingPlanToServiceNotCreatedDBError, - PricingUnitCostDoesNotExistsDBError, - PricingUnitCostNotCreatedDBError, - PricingUnitNotCreatedDBError, - ServiceRunNotCreatedDBError, -) -from .....models.credit_transactions import ( - CreditTransactionCreate, - CreditTransactionCreditsAndStatusUpdate, - CreditTransactionCreditsUpdate, -) -from .....models.pricing_plans import ( - PricingPlansDB, - PricingPlansWithServiceDefaultPlanDB, - PricingPlanToServiceDB, -) -from .....models.pricing_unit_costs import PricingUnitCostsDB -from .....models.pricing_units import PricingUnitsDB -from .....models.service_runs import ( - OsparcCreditsAggregatedByServiceKeyDB, - ServiceRunCreate, - ServiceRunDB, - ServiceRunForCheckDB, - ServiceRunLastHeartbeatUpdate, - ServiceRunStoppedAtUpdate, - ServiceRunWithCreditsDB, -) -from ._base import BaseRepository - -_logger = logging.getLogger(__name__) - - -class ResourceTrackerRepository( - BaseRepository -): # pylint: disable=too-many-public-methods - ############### - # Service Run - ############### - - async def create_service_run(self, data: ServiceRunCreate) -> ServiceRunId: - async with self.db_engine.begin() as conn: - insert_stmt = ( - resource_tracker_service_runs.insert() - .values( - product_name=data.product_name, - service_run_id=data.service_run_id, - wallet_id=data.wallet_id, - wallet_name=data.wallet_name, - pricing_plan_id=data.pricing_plan_id, - pricing_unit_id=data.pricing_unit_id, - pricing_unit_cost_id=data.pricing_unit_cost_id, - pricing_unit_cost=data.pricing_unit_cost, - simcore_user_agent=data.simcore_user_agent, - user_id=data.user_id, - user_email=data.user_email, - project_id=f"{data.project_id}", - project_name=data.project_name, - node_id=f"{data.node_id}", - node_name=data.node_name, - parent_project_id=f"{data.parent_project_id}", - root_parent_project_id=f"{data.root_parent_project_id}", - root_parent_project_name=data.root_parent_project_name, - parent_node_id=f"{data.parent_node_id}", - root_parent_node_id=f"{data.root_parent_node_id}", - service_key=data.service_key, - service_version=data.service_version, - service_type=data.service_type, - service_resources=data.service_resources, - service_additional_metadata=data.service_additional_metadata, - started_at=data.started_at, - stopped_at=None, - service_run_status=ServiceRunStatus.RUNNING, - modified=sa.func.now(), - last_heartbeat_at=data.last_heartbeat_at, - ) - .returning(resource_tracker_service_runs.c.service_run_id) - ) - result = await conn.execute(insert_stmt) - row = result.first() - if row is None: - raise ServiceRunNotCreatedDBError(data=data) - return cast(ServiceRunId, row[0]) - - async def update_service_run_last_heartbeat( - self, data: ServiceRunLastHeartbeatUpdate - ) -> ServiceRunDB | None: - async with self.db_engine.begin() as conn: - update_stmt = ( - resource_tracker_service_runs.update() - .values( - modified=sa.func.now(), - last_heartbeat_at=data.last_heartbeat_at, - missed_heartbeat_counter=0, - ) - .where( - ( - resource_tracker_service_runs.c.service_run_id - == data.service_run_id - ) - & ( - resource_tracker_service_runs.c.service_run_status - == ServiceRunStatus.RUNNING - ) - & ( - resource_tracker_service_runs.c.last_heartbeat_at - <= data.last_heartbeat_at - ) - ) - .returning(sa.literal_column("*")) - ) - result = await conn.execute(update_stmt) - row = result.first() - if row is None: - return None - return ServiceRunDB.model_validate(row) - - async def update_service_run_stopped_at( - self, data: ServiceRunStoppedAtUpdate - ) -> ServiceRunDB | None: - async with self.db_engine.begin() as conn: - update_stmt = ( - resource_tracker_service_runs.update() - .values( - modified=sa.func.now(), - stopped_at=data.stopped_at, - service_run_status=data.service_run_status, - service_run_status_msg=data.service_run_status_msg, - ) - .where( - ( - resource_tracker_service_runs.c.service_run_id - == data.service_run_id - ) - & ( - resource_tracker_service_runs.c.service_run_status - == ServiceRunStatus.RUNNING - ) - ) - .returning(sa.literal_column("*")) - ) - result = await conn.execute(update_stmt) - row = result.first() - if row is None: - return None - return ServiceRunDB.model_validate(row) - - async def get_service_run_by_id( - self, service_run_id: ServiceRunId - ) -> ServiceRunDB | None: - async with self.db_engine.begin() as conn: - stmt = sa.select(resource_tracker_service_runs).where( - resource_tracker_service_runs.c.service_run_id == service_run_id - ) - result = await conn.execute(stmt) - row = result.first() - if row is None: - return None - return ServiceRunDB.model_validate(row) - - _project_tags_subquery = ( - sa.select( - projects_tags.c.project_uuid_for_rut, - sa.func.array_agg(tags.c.name).label("project_tags"), - ) - .select_from(projects_tags.join(tags, projects_tags.c.tag_id == tags.c.id)) - .group_by(projects_tags.c.project_uuid_for_rut) - ).subquery("project_tags_subquery") - - async def list_service_runs_by_product_and_user_and_wallet( - self, - product_name: ProductName, - *, - user_id: UserID | None, - wallet_id: WalletID | None, - offset: int, - limit: int, - service_run_status: ServiceRunStatus | None = None, - started_from: datetime | None = None, - started_until: datetime | None = None, - order_by: OrderBy | None = None, - ) -> list[ServiceRunWithCreditsDB]: - async with self.db_engine.begin() as conn: - query = ( - sa.select( - resource_tracker_service_runs.c.product_name, - resource_tracker_service_runs.c.service_run_id, - resource_tracker_service_runs.c.wallet_id, - resource_tracker_service_runs.c.wallet_name, - resource_tracker_service_runs.c.pricing_plan_id, - resource_tracker_service_runs.c.pricing_unit_id, - resource_tracker_service_runs.c.pricing_unit_cost_id, - resource_tracker_service_runs.c.pricing_unit_cost, - resource_tracker_service_runs.c.user_id, - resource_tracker_service_runs.c.user_email, - resource_tracker_service_runs.c.project_id, - resource_tracker_service_runs.c.project_name, - resource_tracker_service_runs.c.node_id, - resource_tracker_service_runs.c.node_name, - resource_tracker_service_runs.c.parent_project_id, - resource_tracker_service_runs.c.root_parent_project_id, - resource_tracker_service_runs.c.root_parent_project_name, - resource_tracker_service_runs.c.parent_node_id, - resource_tracker_service_runs.c.root_parent_node_id, - resource_tracker_service_runs.c.service_key, - resource_tracker_service_runs.c.service_version, - resource_tracker_service_runs.c.service_type, - resource_tracker_service_runs.c.service_resources, - resource_tracker_service_runs.c.started_at, - resource_tracker_service_runs.c.stopped_at, - resource_tracker_service_runs.c.service_run_status, - resource_tracker_service_runs.c.modified, - resource_tracker_service_runs.c.last_heartbeat_at, - resource_tracker_service_runs.c.service_run_status_msg, - resource_tracker_service_runs.c.missed_heartbeat_counter, - resource_tracker_credit_transactions.c.osparc_credits, - resource_tracker_credit_transactions.c.transaction_status, - sa.func.coalesce( - self._project_tags_subquery.c.project_tags, - sa.cast(sa.text("'{}'"), sa.ARRAY(sa.String)), - ).label("project_tags"), - ) - .select_from( - resource_tracker_service_runs.join( - resource_tracker_credit_transactions, - ( - resource_tracker_service_runs.c.product_name - == resource_tracker_credit_transactions.c.product_name - ) - & ( - resource_tracker_service_runs.c.service_run_id - == resource_tracker_credit_transactions.c.service_run_id - ), - isouter=True, - ).join( - self._project_tags_subquery, - resource_tracker_service_runs.c.project_id - == self._project_tags_subquery.c.project_uuid_for_rut, - isouter=True, - ) - ) - .where(resource_tracker_service_runs.c.product_name == product_name) - .offset(offset) - .limit(limit) - ) - - if user_id: - query = query.where(resource_tracker_service_runs.c.user_id == user_id) - if wallet_id: - query = query.where( - resource_tracker_service_runs.c.wallet_id == wallet_id - ) - if service_run_status: - query = query.where( - resource_tracker_service_runs.c.service_run_status - == service_run_status - ) - if started_from: - query = query.where( - sa.func.DATE(resource_tracker_service_runs.c.started_at) - >= started_from.date() - ) - if started_until: - query = query.where( - sa.func.DATE(resource_tracker_service_runs.c.started_at) - <= started_until.date() - ) - - if order_by: - if order_by.direction == OrderDirection.ASC: - query = query.order_by(sa.asc(order_by.field)) - else: - query = query.order_by(sa.desc(order_by.field)) - else: - # Default ordering - query = query.order_by( - resource_tracker_service_runs.c.started_at.desc() - ) - - result = await conn.execute(query) - - return [ - ServiceRunWithCreditsDB.model_validate(row) for row in result.fetchall() - ] - - async def get_osparc_credits_aggregated_by_service( - self, - product_name: ProductName, - *, - user_id: UserID | None, - wallet_id: WalletID, - offset: int, - limit: int, - started_from: datetime | None = None, - started_until: datetime | None = None, - ) -> tuple[int, list[OsparcCreditsAggregatedByServiceKeyDB]]: - async with self.db_engine.begin() as conn: - base_query = ( - sa.select( - resource_tracker_service_runs.c.service_key, - sa.func.SUM( - resource_tracker_credit_transactions.c.osparc_credits - ).label("osparc_credits"), - sa.func.SUM( - sa.func.round( - ( - sa.func.extract( - "epoch", - resource_tracker_service_runs.c.stopped_at, - ) - - sa.func.extract( - "epoch", - resource_tracker_service_runs.c.started_at, - ) - ) - / 3600, - 2, - ) - ).label("running_time_in_hours"), - ) - .select_from( - resource_tracker_service_runs.join( - resource_tracker_credit_transactions, - ( - resource_tracker_service_runs.c.product_name - == resource_tracker_credit_transactions.c.product_name - ) - & ( - resource_tracker_service_runs.c.service_run_id - == resource_tracker_credit_transactions.c.service_run_id - ), - isouter=True, - ) - ) - .where( - (resource_tracker_service_runs.c.product_name == product_name) - & ( - resource_tracker_credit_transactions.c.transaction_status - == CreditTransactionStatus.BILLED - ) - & ( - resource_tracker_credit_transactions.c.transaction_classification - == CreditClassification.DEDUCT_SERVICE_RUN - ) - & (resource_tracker_credit_transactions.c.wallet_id == wallet_id) - ) - .group_by(resource_tracker_service_runs.c.service_key) - ) - - if user_id: - base_query = base_query.where( - resource_tracker_service_runs.c.user_id == user_id - ) - if started_from: - base_query = base_query.where( - sa.func.DATE(resource_tracker_service_runs.c.started_at) - >= started_from.date() - ) - if started_until: - base_query = base_query.where( - sa.func.DATE(resource_tracker_service_runs.c.started_at) - <= started_until.date() - ) - - subquery = base_query.subquery() - count_query = sa.select(sa.func.count()).select_from(subquery) - count_result = await conn.execute(count_query) - - # Default ordering and pagination - list_query = ( - base_query.order_by(resource_tracker_service_runs.c.service_key.asc()) - .offset(offset) - .limit(limit) - ) - list_result = await conn.execute(list_query) - - return ( - cast(int, count_result.scalar()), - [ - OsparcCreditsAggregatedByServiceKeyDB.model_validate(row) - for row in list_result.fetchall() - ], - ) - - async def export_service_runs_table_to_s3( - self, - product_name: ProductName, - s3_bucket_name: S3BucketName, - s3_key: str, - s3_region: str, - *, - user_id: UserID | None, - wallet_id: WalletID | None, - started_from: datetime | None = None, - started_until: datetime | None = None, - order_by: OrderBy | None = None, - ): - async with self.db_engine.begin() as conn: - query = ( - sa.select( - resource_tracker_service_runs.c.product_name, - resource_tracker_service_runs.c.service_run_id, - resource_tracker_service_runs.c.wallet_name, - resource_tracker_service_runs.c.user_email, - resource_tracker_service_runs.c.root_parent_project_name.label( - "project_name" - ), - resource_tracker_service_runs.c.node_name, - resource_tracker_service_runs.c.service_key, - resource_tracker_service_runs.c.service_version, - resource_tracker_service_runs.c.service_type, - resource_tracker_service_runs.c.started_at, - resource_tracker_service_runs.c.stopped_at, - resource_tracker_credit_transactions.c.osparc_credits, - resource_tracker_credit_transactions.c.transaction_status, - sa.func.coalesce( - self._project_tags_subquery.c.project_tags, - sa.cast(sa.text("'{}'"), sa.ARRAY(sa.String)), - ).label("project_tags"), - ) - .select_from( - resource_tracker_service_runs.join( - resource_tracker_credit_transactions, - resource_tracker_service_runs.c.service_run_id - == resource_tracker_credit_transactions.c.service_run_id, - isouter=True, - ).join( - self._project_tags_subquery, - resource_tracker_service_runs.c.project_id - == self._project_tags_subquery.c.project_uuid_for_rut, - isouter=True, - ) - ) - .where(resource_tracker_service_runs.c.product_name == product_name) - ) - - if user_id: - query = query.where(resource_tracker_service_runs.c.user_id == user_id) - if wallet_id: - query = query.where( - resource_tracker_service_runs.c.wallet_id == wallet_id - ) - if started_from: - query = query.where( - sa.func.DATE(resource_tracker_service_runs.c.started_at) - >= started_from.date() - ) - if started_until: - query = query.where( - sa.func.DATE(resource_tracker_service_runs.c.started_at) - <= started_until.date() - ) - - if order_by: - if order_by.direction == OrderDirection.ASC: - query = query.order_by(sa.asc(order_by.field)) - else: - query = query.order_by(sa.desc(order_by.field)) - else: - # Default ordering - query = query.order_by( - resource_tracker_service_runs.c.started_at.desc() - ) - - compiled_query = ( - str(query.compile(compile_kwargs={"literal_binds": True})) - .replace("\n", "") - .replace("'", "''") - ) - - result = await conn.execute( - sa.DDL( - f""" - SELECT * from aws_s3.query_export_to_s3('{compiled_query}', - aws_commons.create_s3_uri('{s3_bucket_name}', '{s3_key}', '{s3_region}'), 'format csv, HEADER true'); - """ # noqa: S608 - ) - ) - row = result.first() - assert row - _logger.info( - "Rows uploaded %s, Files uploaded %s, Bytes uploaded %s", - row[0], - row[1], - row[2], - ) - - async def total_service_runs_by_product_and_user_and_wallet( - self, - product_name: ProductName, - *, - user_id: UserID | None, - wallet_id: WalletID | None, - service_run_status: ServiceRunStatus | None = None, - started_from: datetime | None = None, - started_until: datetime | None = None, - ) -> PositiveInt: - async with self.db_engine.begin() as conn: - query = ( - sa.select(sa.func.count()) - .select_from(resource_tracker_service_runs) - .where(resource_tracker_service_runs.c.product_name == product_name) - ) - - if user_id: - query = query.where(resource_tracker_service_runs.c.user_id == user_id) - if wallet_id: - query = query.where( - resource_tracker_service_runs.c.wallet_id == wallet_id - ) - if started_from: - query = query.where( - sa.func.DATE(resource_tracker_service_runs.c.started_at) - >= started_from.date() - ) - if started_until: - query = query.where( - sa.func.DATE(resource_tracker_service_runs.c.started_at) - <= started_until.date() - ) - if service_run_status: - query = query.where( - resource_tracker_service_runs.c.service_run_status - == service_run_status - ) - - result = await conn.execute(query) - row = result.first() - return cast(PositiveInt, row[0]) if row else 0 - - ### For Background check purpose: - - async def list_service_runs_with_running_status_across_all_products( - self, - *, - offset: int, - limit: int, - ) -> list[ServiceRunForCheckDB]: - async with self.db_engine.begin() as conn: - query = ( - sa.select( - resource_tracker_service_runs.c.service_run_id, - resource_tracker_service_runs.c.last_heartbeat_at, - resource_tracker_service_runs.c.missed_heartbeat_counter, - resource_tracker_service_runs.c.modified, - ) - .where( - resource_tracker_service_runs.c.service_run_status - == ServiceRunStatus.RUNNING - ) - .order_by(resource_tracker_service_runs.c.started_at.desc()) # NOTE: - .offset(offset) - .limit(limit) - ) - result = await conn.execute(query) - - return [ServiceRunForCheckDB.model_validate(row) for row in result.fetchall()] - - async def total_service_runs_with_running_status_across_all_products( - self, - ) -> PositiveInt: - async with self.db_engine.begin() as conn: - query = ( - sa.select(sa.func.count()) - .select_from(resource_tracker_service_runs) - .where( - resource_tracker_service_runs.c.service_run_status - == ServiceRunStatus.RUNNING - ) - ) - result = await conn.execute(query) - row = result.first() - return cast(PositiveInt, row[0]) if row else 0 - - async def update_service_missed_heartbeat_counter( - self, - service_run_id: ServiceRunId, - last_heartbeat_at: datetime, - missed_heartbeat_counter: int, - ) -> ServiceRunDB | None: - async with self.db_engine.begin() as conn: - update_stmt = ( - resource_tracker_service_runs.update() - .values( - modified=sa.func.now(), - missed_heartbeat_counter=missed_heartbeat_counter, - ) - .where( - (resource_tracker_service_runs.c.service_run_id == service_run_id) - & ( - resource_tracker_service_runs.c.service_run_status - == ServiceRunStatus.RUNNING - ) - & ( - resource_tracker_service_runs.c.last_heartbeat_at - == last_heartbeat_at - ) - ) - .returning(sa.literal_column("*")) - ) - - result = await conn.execute(update_stmt) - row = result.first() - if row is None: - return None - return ServiceRunDB.model_validate(row) - - ################################# - # Credit transactions - ################################# - - async def create_credit_transaction( - self, data: CreditTransactionCreate - ) -> CreditTransactionId: - async with self.db_engine.begin() as conn: - insert_stmt = ( - resource_tracker_credit_transactions.insert() - .values( - product_name=data.product_name, - wallet_id=data.wallet_id, - wallet_name=data.wallet_name, - pricing_plan_id=data.pricing_plan_id, - pricing_unit_id=data.pricing_unit_id, - pricing_unit_cost_id=data.pricing_unit_cost_id, - user_id=data.user_id, - user_email=data.user_email, - osparc_credits=data.osparc_credits, - transaction_status=data.transaction_status, - transaction_classification=data.transaction_classification, - service_run_id=data.service_run_id, - payment_transaction_id=data.payment_transaction_id, - created=data.created_at, - last_heartbeat_at=data.last_heartbeat_at, - modified=sa.func.now(), - ) - .returning(resource_tracker_credit_transactions.c.transaction_id) - ) - result = await conn.execute(insert_stmt) - row = result.first() - if row is None: - raise CreditTransactionNotCreatedDBError(data=data) - return cast(CreditTransactionId, row[0]) - - async def update_credit_transaction_credits( - self, data: CreditTransactionCreditsUpdate - ) -> CreditTransactionId | None: - async with self.db_engine.begin() as conn: - update_stmt = ( - resource_tracker_credit_transactions.update() - .values( - modified=sa.func.now(), - osparc_credits=data.osparc_credits, - last_heartbeat_at=data.last_heartbeat_at, - ) - .where( - ( - resource_tracker_credit_transactions.c.service_run_id - == data.service_run_id - ) - & ( - resource_tracker_credit_transactions.c.transaction_status - == CreditTransactionStatus.PENDING - ) - & ( - resource_tracker_credit_transactions.c.last_heartbeat_at - <= data.last_heartbeat_at - ) - ) - .returning(resource_tracker_credit_transactions.c.service_run_id) - ) - result = await conn.execute(update_stmt) - row = result.first() - if row is None: - return None - return cast(CreditTransactionId | None, row[0]) - - async def update_credit_transaction_credits_and_status( - self, data: CreditTransactionCreditsAndStatusUpdate - ) -> CreditTransactionId | None: - async with self.db_engine.begin() as conn: - update_stmt = ( - resource_tracker_credit_transactions.update() - .values( - modified=sa.func.now(), - osparc_credits=data.osparc_credits, - transaction_status=data.transaction_status, - ) - .where( - ( - resource_tracker_credit_transactions.c.service_run_id - == data.service_run_id - ) - & ( - resource_tracker_credit_transactions.c.transaction_status - == CreditTransactionStatus.PENDING - ) - ) - .returning(resource_tracker_credit_transactions.c.service_run_id) - ) - result = await conn.execute(update_stmt) - row = result.first() - if row is None: - return None - return cast(CreditTransactionId | None, row[0]) - - async def sum_credit_transactions_by_product_and_wallet( - self, product_name: ProductName, wallet_id: WalletID - ) -> WalletTotalCredits: - async with self.db_engine.begin() as conn: - sum_stmt = sa.select( - sa.func.sum(resource_tracker_credit_transactions.c.osparc_credits) - ).where( - (resource_tracker_credit_transactions.c.product_name == product_name) - & (resource_tracker_credit_transactions.c.wallet_id == wallet_id) - & ( - resource_tracker_credit_transactions.c.transaction_status.in_( - [ - CreditTransactionStatus.BILLED, - CreditTransactionStatus.PENDING, - ] - ) - ) - ) - result = await conn.execute(sum_stmt) - row = result.first() - if row is None or row[0] is None: - return WalletTotalCredits( - wallet_id=wallet_id, available_osparc_credits=Decimal(0) - ) - return WalletTotalCredits(wallet_id=wallet_id, available_osparc_credits=row[0]) - - ################################# - # Pricing plans - ################################# - - async def list_active_service_pricing_plans_by_product_and_service( - self, - product_name: ProductName, - service_key: ServiceKey, - service_version: ServiceVersion, - ) -> list[PricingPlansWithServiceDefaultPlanDB]: - # NOTE: consilidate with utils_services_environmnets.py - def _version(column_or_value): - # converts version value string to array[integer] that can be compared - return sa.func.string_to_array(column_or_value, ".").cast(ARRAY(INTEGER)) - - async with self.db_engine.begin() as conn: - # Firstly find the correct service version - query = ( - sa.select( - resource_tracker_pricing_plan_to_service.c.service_key, - resource_tracker_pricing_plan_to_service.c.service_version, - ) - .select_from( - resource_tracker_pricing_plan_to_service.join( - resource_tracker_pricing_plans, - ( - resource_tracker_pricing_plan_to_service.c.pricing_plan_id - == resource_tracker_pricing_plans.c.pricing_plan_id - ), - ) - ) - .where( - ( - _version( - resource_tracker_pricing_plan_to_service.c.service_version - ) - <= _version(service_version) - ) - & ( - resource_tracker_pricing_plan_to_service.c.service_key - == service_key - ) - & (resource_tracker_pricing_plans.c.product_name == product_name) - & (resource_tracker_pricing_plans.c.is_active.is_(True)) - ) - .order_by( - _version( - resource_tracker_pricing_plan_to_service.c.service_version - ).desc() - ) - .limit(1) - ) - result = await conn.execute(query) - row = result.first() - if row is None: - return [] - latest_service_key, latest_service_version = row - # Now choose all pricing plans connected to this service - query = ( - sa.select( - resource_tracker_pricing_plans.c.pricing_plan_id, - resource_tracker_pricing_plans.c.display_name, - resource_tracker_pricing_plans.c.description, - resource_tracker_pricing_plans.c.classification, - resource_tracker_pricing_plans.c.is_active, - resource_tracker_pricing_plans.c.created, - resource_tracker_pricing_plans.c.pricing_plan_key, - resource_tracker_pricing_plan_to_service.c.service_default_plan, - ) - .select_from( - resource_tracker_pricing_plan_to_service.join( - resource_tracker_pricing_plans, - ( - resource_tracker_pricing_plan_to_service.c.pricing_plan_id - == resource_tracker_pricing_plans.c.pricing_plan_id - ), - ) - ) - .where( - ( - _version( - resource_tracker_pricing_plan_to_service.c.service_version - ) - == _version(latest_service_version) - ) - & ( - resource_tracker_pricing_plan_to_service.c.service_key - == latest_service_key - ) - & (resource_tracker_pricing_plans.c.product_name == product_name) - & (resource_tracker_pricing_plans.c.is_active.is_(True)) - ) - .order_by( - resource_tracker_pricing_plan_to_service.c.pricing_plan_id.desc() - ) - ) - result = await conn.execute(query) - - return [ - PricingPlansWithServiceDefaultPlanDB.model_validate(row) - for row in result.fetchall() - ] - - async def get_pricing_plan( - self, product_name: ProductName, pricing_plan_id: PricingPlanId - ) -> PricingPlansDB: - async with self.db_engine.begin() as conn: - select_stmt = sa.select( - resource_tracker_pricing_plans.c.pricing_plan_id, - resource_tracker_pricing_plans.c.display_name, - resource_tracker_pricing_plans.c.description, - resource_tracker_pricing_plans.c.classification, - resource_tracker_pricing_plans.c.is_active, - resource_tracker_pricing_plans.c.created, - resource_tracker_pricing_plans.c.pricing_plan_key, - ).where( - (resource_tracker_pricing_plans.c.pricing_plan_id == pricing_plan_id) - & (resource_tracker_pricing_plans.c.product_name == product_name) - ) - result = await conn.execute(select_stmt) - row = result.first() - if row is None: - raise PricingPlanDoesNotExistsDBError(pricing_plan_id=pricing_plan_id) - return PricingPlansDB.model_validate(row) - - async def list_pricing_plans_by_product( - self, product_name: ProductName - ) -> list[PricingPlansDB]: - async with self.db_engine.begin() as conn: - select_stmt = sa.select( - resource_tracker_pricing_plans.c.pricing_plan_id, - resource_tracker_pricing_plans.c.display_name, - resource_tracker_pricing_plans.c.description, - resource_tracker_pricing_plans.c.classification, - resource_tracker_pricing_plans.c.is_active, - resource_tracker_pricing_plans.c.created, - resource_tracker_pricing_plans.c.pricing_plan_key, - ).where(resource_tracker_pricing_plans.c.product_name == product_name) - result = await conn.execute(select_stmt) - - return [PricingPlansDB.model_validate(row) for row in result.fetchall()] - - async def create_pricing_plan(self, data: PricingPlanCreate) -> PricingPlansDB: - async with self.db_engine.begin() as conn: - insert_stmt = ( - resource_tracker_pricing_plans.insert() - .values( - product_name=data.product_name, - display_name=data.display_name, - description=data.description, - classification=data.classification, - is_active=True, - created=sa.func.now(), - modified=sa.func.now(), - pricing_plan_key=data.pricing_plan_key, - ) - .returning( - *[ - resource_tracker_pricing_plans.c.pricing_plan_id, - resource_tracker_pricing_plans.c.display_name, - resource_tracker_pricing_plans.c.description, - resource_tracker_pricing_plans.c.classification, - resource_tracker_pricing_plans.c.is_active, - resource_tracker_pricing_plans.c.created, - resource_tracker_pricing_plans.c.pricing_plan_key, - ] - ) - ) - result = await conn.execute(insert_stmt) - row = result.first() - if row is None: - raise PricingPlanNotCreatedDBError(data=data) - return PricingPlansDB.model_validate(row) - - async def update_pricing_plan( - self, product_name: ProductName, data: PricingPlanUpdate - ) -> PricingPlansDB | None: - async with self.db_engine.begin() as conn: - update_stmt = ( - resource_tracker_pricing_plans.update() - .values( - display_name=data.display_name, - description=data.description, - is_active=data.is_active, - modified=sa.func.now(), - ) - .where( - ( - resource_tracker_pricing_plans.c.pricing_plan_id - == data.pricing_plan_id - ) - & (resource_tracker_pricing_plans.c.product_name == product_name) - ) - .returning( - *[ - resource_tracker_pricing_plans.c.pricing_plan_id, - resource_tracker_pricing_plans.c.display_name, - resource_tracker_pricing_plans.c.description, - resource_tracker_pricing_plans.c.classification, - resource_tracker_pricing_plans.c.is_active, - resource_tracker_pricing_plans.c.created, - resource_tracker_pricing_plans.c.pricing_plan_key, - ] - ) - ) - result = await conn.execute(update_stmt) - row = result.first() - if row is None: - return None - return PricingPlansDB.model_validate(row) - - ################################# - # Pricing plan to service - ################################# - - async def list_connected_services_to_pricing_plan_by_pricing_plan( - self, product_name: ProductName, pricing_plan_id: PricingPlanId - ) -> list[PricingPlanToServiceDB]: - async with self.db_engine.begin() as conn: - query = ( - sa.select( - resource_tracker_pricing_plan_to_service.c.pricing_plan_id, - resource_tracker_pricing_plan_to_service.c.service_key, - resource_tracker_pricing_plan_to_service.c.service_version, - resource_tracker_pricing_plan_to_service.c.created, - ) - .select_from( - resource_tracker_pricing_plan_to_service.join( - resource_tracker_pricing_plans, - ( - resource_tracker_pricing_plan_to_service.c.pricing_plan_id - == resource_tracker_pricing_plans.c.pricing_plan_id - ), - ) - ) - .where( - (resource_tracker_pricing_plans.c.product_name == product_name) - & ( - resource_tracker_pricing_plans.c.pricing_plan_id - == pricing_plan_id - ) - ) - .order_by( - resource_tracker_pricing_plan_to_service.c.pricing_plan_id.desc() - ) - ) - result = await conn.execute(query) - - return [ - PricingPlanToServiceDB.model_validate(row) for row in result.fetchall() - ] - - async def upsert_service_to_pricing_plan( - self, - product_name: ProductName, - pricing_plan_id: PricingPlanId, - service_key: ServiceKey, - service_version: ServiceVersion, - ) -> PricingPlanToServiceDB: - async with self.db_engine.begin() as conn: - query = ( - sa.select( - resource_tracker_pricing_plan_to_service.c.pricing_plan_id, - resource_tracker_pricing_plan_to_service.c.service_key, - resource_tracker_pricing_plan_to_service.c.service_version, - resource_tracker_pricing_plan_to_service.c.created, - ) - .select_from( - resource_tracker_pricing_plan_to_service.join( - resource_tracker_pricing_plans, - ( - resource_tracker_pricing_plan_to_service.c.pricing_plan_id - == resource_tracker_pricing_plans.c.pricing_plan_id - ), - ) - ) - .where( - (resource_tracker_pricing_plans.c.product_name == product_name) - & ( - resource_tracker_pricing_plans.c.pricing_plan_id - == pricing_plan_id - ) - & ( - resource_tracker_pricing_plan_to_service.c.service_key - == service_key - ) - & ( - resource_tracker_pricing_plan_to_service.c.service_version - == service_version - ) - ) - ) - result = await conn.execute(query) - row = result.first() - - if row is not None: - delete_stmt = resource_tracker_pricing_plan_to_service.delete().where( - ( - resource_tracker_pricing_plans.c.pricing_plan_id - == pricing_plan_id - ) - & ( - resource_tracker_pricing_plan_to_service.c.service_key - == service_key - ) - & ( - resource_tracker_pricing_plan_to_service.c.service_version - == service_version - ) - ) - await conn.execute(delete_stmt) - - insert_stmt = ( - resource_tracker_pricing_plan_to_service.insert() - .values( - pricing_plan_id=pricing_plan_id, - service_key=service_key, - service_version=service_version, - created=sa.func.now(), - modified=sa.func.now(), - service_default_plan=True, - ) - .returning( - *[ - resource_tracker_pricing_plan_to_service.c.pricing_plan_id, - resource_tracker_pricing_plan_to_service.c.service_key, - resource_tracker_pricing_plan_to_service.c.service_version, - resource_tracker_pricing_plan_to_service.c.created, - ] - ) - ) - result = await conn.execute(insert_stmt) - row = result.first() - if row is None: - raise PricingPlanToServiceNotCreatedDBError( - data=f"pricing_plan_id {pricing_plan_id}, service_key {service_key}, service_version {service_version}" - ) - return PricingPlanToServiceDB.model_validate(row) - - ################################# - # Pricing units - ################################# - - @staticmethod - def _pricing_units_select_stmt(): - return sa.select( - resource_tracker_pricing_units.c.pricing_unit_id, - resource_tracker_pricing_units.c.pricing_plan_id, - resource_tracker_pricing_units.c.unit_name, - resource_tracker_pricing_units.c.unit_extra_info, - resource_tracker_pricing_units.c.default, - resource_tracker_pricing_units.c.specific_info, - resource_tracker_pricing_units.c.created, - resource_tracker_pricing_units.c.modified, - resource_tracker_pricing_unit_costs.c.cost_per_unit.label( - "current_cost_per_unit" - ), - resource_tracker_pricing_unit_costs.c.pricing_unit_cost_id.label( - "current_cost_per_unit_id" - ), - ) - - async def list_pricing_units_by_pricing_plan( - self, - pricing_plan_id: PricingPlanId, - ) -> list[PricingUnitsDB]: - async with self.db_engine.begin() as conn: - query = ( - self._pricing_units_select_stmt() - .select_from( - resource_tracker_pricing_units.join( - resource_tracker_pricing_unit_costs, - ( - ( - resource_tracker_pricing_units.c.pricing_plan_id - == resource_tracker_pricing_unit_costs.c.pricing_plan_id - ) - & ( - resource_tracker_pricing_units.c.pricing_unit_id - == resource_tracker_pricing_unit_costs.c.pricing_unit_id - ) - ), - ) - ) - .where( - ( - resource_tracker_pricing_units.c.pricing_plan_id - == pricing_plan_id - ) - & (resource_tracker_pricing_unit_costs.c.valid_to.is_(None)) - ) - .order_by(resource_tracker_pricing_unit_costs.c.cost_per_unit.asc()) - ) - result = await conn.execute(query) - - return [PricingUnitsDB.model_validate(row) for row in result.fetchall()] - - async def get_valid_pricing_unit( - self, - product_name: ProductName, - pricing_plan_id: PricingPlanId, - pricing_unit_id: PricingUnitId, - ) -> PricingUnitsDB: - async with self.db_engine.begin() as conn: - query = ( - self._pricing_units_select_stmt() - .select_from( - resource_tracker_pricing_units.join( - resource_tracker_pricing_unit_costs, - ( - ( - resource_tracker_pricing_units.c.pricing_plan_id - == resource_tracker_pricing_unit_costs.c.pricing_plan_id - ) - & ( - resource_tracker_pricing_units.c.pricing_unit_id - == resource_tracker_pricing_unit_costs.c.pricing_unit_id - ) - ), - ).join( - resource_tracker_pricing_plans, - ( - resource_tracker_pricing_plans.c.pricing_plan_id - == resource_tracker_pricing_units.c.pricing_plan_id - ), - ) - ) - .where( - ( - resource_tracker_pricing_units.c.pricing_plan_id - == pricing_plan_id - ) - & ( - resource_tracker_pricing_units.c.pricing_unit_id - == pricing_unit_id - ) - & (resource_tracker_pricing_unit_costs.c.valid_to.is_(None)) - & (resource_tracker_pricing_plans.c.product_name == product_name) - ) - ) - result = await conn.execute(query) - - row = result.first() - if row is None: - raise PricingPlanAndPricingUnitCombinationDoesNotExistsDBError( - pricing_plan_id=pricing_plan_id, - pricing_unit_id=pricing_unit_id, - product_name=product_name, - ) - return PricingUnitsDB.model_validate(row) - - async def create_pricing_unit_with_cost( - self, data: PricingUnitWithCostCreate, pricing_plan_key: str - ) -> tuple[PricingUnitId, PricingUnitCostId]: - async with self.db_engine.begin() as conn: - # pricing units table - insert_stmt = ( - resource_tracker_pricing_units.insert() - .values( - pricing_plan_id=data.pricing_plan_id, - unit_name=data.unit_name, - unit_extra_info=data.unit_extra_info.model_dump(), - default=data.default, - specific_info=data.specific_info.model_dump(), - created=sa.func.now(), - modified=sa.func.now(), - ) - .returning(resource_tracker_pricing_units.c.pricing_unit_id) - ) - result = await conn.execute(insert_stmt) - row = result.first() - if row is None: - raise PricingUnitNotCreatedDBError(data=data) - _pricing_unit_id = row[0] - - # pricing unit cost table - insert_stmt = ( - resource_tracker_pricing_unit_costs.insert() - .values( - pricing_plan_id=data.pricing_plan_id, - pricing_plan_key=pricing_plan_key, - pricing_unit_id=_pricing_unit_id, - pricing_unit_name=data.unit_name, - cost_per_unit=data.cost_per_unit, - valid_from=sa.func.now(), - valid_to=None, - created=sa.func.now(), - comment=data.comment, - modified=sa.func.now(), - ) - .returning(resource_tracker_pricing_unit_costs.c.pricing_unit_cost_id) - ) - result = await conn.execute(insert_stmt) - row = result.first() - if row is None: - raise PricingUnitCostNotCreatedDBError(data=data) - _pricing_unit_cost_id = row[0] - - return (_pricing_unit_id, _pricing_unit_cost_id) - - async def update_pricing_unit_with_cost( - self, data: PricingUnitWithCostUpdate, pricing_plan_key: str - ) -> None: - async with self.db_engine.begin() as conn: - # pricing units table - update_stmt = ( - resource_tracker_pricing_units.update() - .values( - unit_name=data.unit_name, - unit_extra_info=data.unit_extra_info.model_dump(), - default=data.default, - specific_info=data.specific_info.model_dump(), - modified=sa.func.now(), - ) - .where( - resource_tracker_pricing_units.c.pricing_unit_id - == data.pricing_unit_id - ) - .returning(resource_tracker_pricing_units.c.pricing_unit_id) - ) - await conn.execute(update_stmt) - - # If price change, then we update pricing unit cost table - if data.pricing_unit_cost_update: - # Firstly we close previous price - update_stmt = ( - resource_tracker_pricing_unit_costs.update() - .values( - valid_to=sa.func.now(), # <-- Closing previous price - modified=sa.func.now(), - ) - .where( - resource_tracker_pricing_unit_costs.c.pricing_unit_id - == data.pricing_unit_id - ) - .returning(resource_tracker_pricing_unit_costs.c.pricing_unit_id) - ) - result = await conn.execute(update_stmt) - - # Then we create a new price - insert_stmt = ( - resource_tracker_pricing_unit_costs.insert() - .values( - pricing_plan_id=data.pricing_plan_id, - pricing_plan_key=pricing_plan_key, - pricing_unit_id=data.pricing_unit_id, - pricing_unit_name=data.unit_name, - cost_per_unit=data.pricing_unit_cost_update.cost_per_unit, - valid_from=sa.func.now(), - valid_to=None, # <-- New price is valid - created=sa.func.now(), - comment=data.pricing_unit_cost_update.comment, - modified=sa.func.now(), - ) - .returning( - resource_tracker_pricing_unit_costs.c.pricing_unit_cost_id - ) - ) - result = await conn.execute(insert_stmt) - row = result.first() - if row is None: - raise PricingUnitCostNotCreatedDBError(data=data) - - ################################# - # Pricing unit-costs - ################################# - - async def get_pricing_unit_cost_by_id( - self, pricing_unit_cost_id: PricingUnitCostId - ) -> PricingUnitCostsDB: - async with self.db_engine.begin() as conn: - query = sa.select( - resource_tracker_pricing_unit_costs.c.pricing_unit_cost_id, - resource_tracker_pricing_unit_costs.c.pricing_plan_id, - resource_tracker_pricing_unit_costs.c.pricing_plan_key, - resource_tracker_pricing_unit_costs.c.pricing_unit_id, - resource_tracker_pricing_unit_costs.c.pricing_unit_name, - resource_tracker_pricing_unit_costs.c.cost_per_unit, - resource_tracker_pricing_unit_costs.c.valid_from, - resource_tracker_pricing_unit_costs.c.valid_to, - resource_tracker_pricing_unit_costs.c.created, - resource_tracker_pricing_unit_costs.c.comment, - resource_tracker_pricing_unit_costs.c.modified, - ).where( - resource_tracker_pricing_unit_costs.c.pricing_unit_cost_id - == pricing_unit_cost_id - ) - result = await conn.execute(query) - - row = result.first() - if row is None: - raise PricingUnitCostDoesNotExistsDBError( - pricing_unit_cost_id=pricing_unit_cost_id - ) - return PricingUnitCostsDB.model_validate(row) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/service_runs_db.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/service_runs_db.py new file mode 100644 index 00000000000..b7452604870 --- /dev/null +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/service_runs_db.py @@ -0,0 +1,621 @@ +import logging +from datetime import datetime +from typing import cast + +import sqlalchemy as sa +from models_library.api_schemas_storage import S3BucketName +from models_library.products import ProductName +from models_library.resource_tracker import ( + CreditClassification, + CreditTransactionStatus, + ServiceRunId, + ServiceRunStatus, +) +from models_library.rest_ordering import OrderBy, OrderDirection +from models_library.users import UserID +from models_library.wallets import WalletID +from pydantic import PositiveInt +from simcore_postgres_database.models.projects_tags import projects_tags +from simcore_postgres_database.models.resource_tracker_credit_transactions import ( + resource_tracker_credit_transactions, +) +from simcore_postgres_database.models.resource_tracker_service_runs import ( + resource_tracker_service_runs, +) +from simcore_postgres_database.models.tags import tags +from simcore_postgres_database.utils_repos import transaction_context +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine + +from ....exceptions.errors import ServiceRunNotCreatedDBError +from ....models.service_runs import ( + OsparcCreditsAggregatedByServiceKeyDB, + ServiceRunCreate, + ServiceRunDB, + ServiceRunForCheckDB, + ServiceRunLastHeartbeatUpdate, + ServiceRunStoppedAtUpdate, + ServiceRunWithCreditsDB, +) + +_logger = logging.getLogger(__name__) + + +async def create_service_run( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + data: ServiceRunCreate, +) -> ServiceRunId: + async with transaction_context(engine, connection) as conn: + insert_stmt = ( + resource_tracker_service_runs.insert() + .values( + product_name=data.product_name, + service_run_id=data.service_run_id, + wallet_id=data.wallet_id, + wallet_name=data.wallet_name, + pricing_plan_id=data.pricing_plan_id, + pricing_unit_id=data.pricing_unit_id, + pricing_unit_cost_id=data.pricing_unit_cost_id, + pricing_unit_cost=data.pricing_unit_cost, + simcore_user_agent=data.simcore_user_agent, + user_id=data.user_id, + user_email=data.user_email, + project_id=f"{data.project_id}", + project_name=data.project_name, + node_id=f"{data.node_id}", + node_name=data.node_name, + parent_project_id=f"{data.parent_project_id}", + root_parent_project_id=f"{data.root_parent_project_id}", + root_parent_project_name=data.root_parent_project_name, + parent_node_id=f"{data.parent_node_id}", + root_parent_node_id=f"{data.root_parent_node_id}", + service_key=data.service_key, + service_version=data.service_version, + service_type=data.service_type, + service_resources=data.service_resources, + service_additional_metadata=data.service_additional_metadata, + started_at=data.started_at, + stopped_at=None, + service_run_status=ServiceRunStatus.RUNNING, + modified=sa.func.now(), + last_heartbeat_at=data.last_heartbeat_at, + ) + .returning(resource_tracker_service_runs.c.service_run_id) + ) + result = await conn.execute(insert_stmt) + row = result.first() + if row is None: + raise ServiceRunNotCreatedDBError(data=data) + return cast(ServiceRunId, row[0]) + + +async def update_service_run_last_heartbeat( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + data: ServiceRunLastHeartbeatUpdate, +) -> ServiceRunDB | None: + async with transaction_context(engine, connection) as conn: + update_stmt = ( + resource_tracker_service_runs.update() + .values( + modified=sa.func.now(), + last_heartbeat_at=data.last_heartbeat_at, + missed_heartbeat_counter=0, + ) + .where( + (resource_tracker_service_runs.c.service_run_id == data.service_run_id) + & ( + resource_tracker_service_runs.c.service_run_status + == ServiceRunStatus.RUNNING + ) + & ( + resource_tracker_service_runs.c.last_heartbeat_at + <= data.last_heartbeat_at + ) + ) + .returning(sa.literal_column("*")) + ) + result = await conn.execute(update_stmt) + row = result.first() + if row is None: + return None + return ServiceRunDB.model_validate(row) + + +async def update_service_run_stopped_at( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + data: ServiceRunStoppedAtUpdate, +) -> ServiceRunDB | None: + async with transaction_context(engine, connection) as conn: + update_stmt = ( + resource_tracker_service_runs.update() + .values( + modified=sa.func.now(), + stopped_at=data.stopped_at, + service_run_status=data.service_run_status, + service_run_status_msg=data.service_run_status_msg, + ) + .where( + (resource_tracker_service_runs.c.service_run_id == data.service_run_id) + & ( + resource_tracker_service_runs.c.service_run_status + == ServiceRunStatus.RUNNING + ) + ) + .returning(sa.literal_column("*")) + ) + result = await conn.execute(update_stmt) + row = result.first() + if row is None: + return None + return ServiceRunDB.model_validate(row) + + +async def get_service_run_by_id( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + service_run_id: ServiceRunId, +) -> ServiceRunDB | None: + async with transaction_context(engine, connection) as conn: + stmt = sa.select(resource_tracker_service_runs).where( + resource_tracker_service_runs.c.service_run_id == service_run_id + ) + result = await conn.execute(stmt) + row = result.first() + if row is None: + return None + return ServiceRunDB.model_validate(row) + + +_project_tags_subquery = ( + sa.select( + projects_tags.c.project_uuid_for_rut, + sa.func.array_agg(tags.c.name).label("project_tags"), + ) + .select_from(projects_tags.join(tags, projects_tags.c.tag_id == tags.c.id)) + .group_by(projects_tags.c.project_uuid_for_rut) +).subquery("project_tags_subquery") + + +async def list_service_runs_by_product_and_user_and_wallet( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + user_id: UserID | None, + wallet_id: WalletID | None, + offset: int, + limit: int, + service_run_status: ServiceRunStatus | None = None, + started_from: datetime | None = None, + started_until: datetime | None = None, + order_by: OrderBy | None = None, +) -> list[ServiceRunWithCreditsDB]: + async with transaction_context(engine, connection) as conn: + query = ( + sa.select( + resource_tracker_service_runs.c.product_name, + resource_tracker_service_runs.c.service_run_id, + resource_tracker_service_runs.c.wallet_id, + resource_tracker_service_runs.c.wallet_name, + resource_tracker_service_runs.c.pricing_plan_id, + resource_tracker_service_runs.c.pricing_unit_id, + resource_tracker_service_runs.c.pricing_unit_cost_id, + resource_tracker_service_runs.c.pricing_unit_cost, + resource_tracker_service_runs.c.user_id, + resource_tracker_service_runs.c.user_email, + resource_tracker_service_runs.c.project_id, + resource_tracker_service_runs.c.project_name, + resource_tracker_service_runs.c.node_id, + resource_tracker_service_runs.c.node_name, + resource_tracker_service_runs.c.parent_project_id, + resource_tracker_service_runs.c.root_parent_project_id, + resource_tracker_service_runs.c.root_parent_project_name, + resource_tracker_service_runs.c.parent_node_id, + resource_tracker_service_runs.c.root_parent_node_id, + resource_tracker_service_runs.c.service_key, + resource_tracker_service_runs.c.service_version, + resource_tracker_service_runs.c.service_type, + resource_tracker_service_runs.c.service_resources, + resource_tracker_service_runs.c.started_at, + resource_tracker_service_runs.c.stopped_at, + resource_tracker_service_runs.c.service_run_status, + resource_tracker_service_runs.c.modified, + resource_tracker_service_runs.c.last_heartbeat_at, + resource_tracker_service_runs.c.service_run_status_msg, + resource_tracker_service_runs.c.missed_heartbeat_counter, + resource_tracker_credit_transactions.c.osparc_credits, + resource_tracker_credit_transactions.c.transaction_status, + sa.func.coalesce( + _project_tags_subquery.c.project_tags, + sa.cast(sa.text("'{}'"), sa.ARRAY(sa.String)), + ).label("project_tags"), + ) + .select_from( + resource_tracker_service_runs.join( + resource_tracker_credit_transactions, + ( + resource_tracker_service_runs.c.product_name + == resource_tracker_credit_transactions.c.product_name + ) + & ( + resource_tracker_service_runs.c.service_run_id + == resource_tracker_credit_transactions.c.service_run_id + ), + isouter=True, + ).join( + _project_tags_subquery, + resource_tracker_service_runs.c.project_id + == _project_tags_subquery.c.project_uuid_for_rut, + isouter=True, + ) + ) + .where(resource_tracker_service_runs.c.product_name == product_name) + .offset(offset) + .limit(limit) + ) + + if user_id: + query = query.where(resource_tracker_service_runs.c.user_id == user_id) + if wallet_id: + query = query.where(resource_tracker_service_runs.c.wallet_id == wallet_id) + if service_run_status: + query = query.where( + resource_tracker_service_runs.c.service_run_status == service_run_status + ) + if started_from: + query = query.where( + sa.func.DATE(resource_tracker_service_runs.c.started_at) + >= started_from.date() + ) + if started_until: + query = query.where( + sa.func.DATE(resource_tracker_service_runs.c.started_at) + <= started_until.date() + ) + + if order_by: + if order_by.direction == OrderDirection.ASC: + query = query.order_by(sa.asc(order_by.field)) + else: + query = query.order_by(sa.desc(order_by.field)) + else: + # Default ordering + query = query.order_by(resource_tracker_service_runs.c.started_at.desc()) + + result = await conn.execute(query) + + return [ServiceRunWithCreditsDB.model_validate(row) for row in result.fetchall()] + + +async def get_osparc_credits_aggregated_by_service( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + user_id: UserID | None, + wallet_id: WalletID, + offset: int, + limit: int, + started_from: datetime | None = None, + started_until: datetime | None = None, +) -> tuple[int, list[OsparcCreditsAggregatedByServiceKeyDB]]: + async with transaction_context(engine, connection) as conn: + base_query = ( + sa.select( + resource_tracker_service_runs.c.service_key, + sa.func.SUM( + resource_tracker_credit_transactions.c.osparc_credits + ).label("osparc_credits"), + sa.func.SUM( + sa.func.round( + ( + sa.func.extract( + "epoch", + resource_tracker_service_runs.c.stopped_at, + ) + - sa.func.extract( + "epoch", + resource_tracker_service_runs.c.started_at, + ) + ) + / 3600, + 2, + ) + ).label("running_time_in_hours"), + ) + .select_from( + resource_tracker_service_runs.join( + resource_tracker_credit_transactions, + ( + resource_tracker_service_runs.c.product_name + == resource_tracker_credit_transactions.c.product_name + ) + & ( + resource_tracker_service_runs.c.service_run_id + == resource_tracker_credit_transactions.c.service_run_id + ), + isouter=True, + ) + ) + .where( + (resource_tracker_service_runs.c.product_name == product_name) + & ( + resource_tracker_credit_transactions.c.transaction_status + == CreditTransactionStatus.BILLED + ) + & ( + resource_tracker_credit_transactions.c.transaction_classification + == CreditClassification.DEDUCT_SERVICE_RUN + ) + & (resource_tracker_credit_transactions.c.wallet_id == wallet_id) + ) + .group_by(resource_tracker_service_runs.c.service_key) + ) + + if user_id: + base_query = base_query.where( + resource_tracker_service_runs.c.user_id == user_id + ) + if started_from: + base_query = base_query.where( + sa.func.DATE(resource_tracker_service_runs.c.started_at) + >= started_from.date() + ) + if started_until: + base_query = base_query.where( + sa.func.DATE(resource_tracker_service_runs.c.started_at) + <= started_until.date() + ) + + subquery = base_query.subquery() + count_query = sa.select(sa.func.count()).select_from(subquery) + count_result = await conn.execute(count_query) + + # Default ordering and pagination + list_query = ( + base_query.order_by(resource_tracker_service_runs.c.service_key.asc()) + .offset(offset) + .limit(limit) + ) + list_result = await conn.execute(list_query) + + return ( + cast(int, count_result.scalar()), + [ + OsparcCreditsAggregatedByServiceKeyDB.model_validate(row) + for row in list_result.fetchall() + ], + ) + + +async def export_service_runs_table_to_s3( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + s3_bucket_name: S3BucketName, + s3_key: str, + s3_region: str, + user_id: UserID | None, + wallet_id: WalletID | None, + started_from: datetime | None = None, + started_until: datetime | None = None, + order_by: OrderBy | None = None, +): + async with transaction_context(engine, connection) as conn: + query = ( + sa.select( + resource_tracker_service_runs.c.product_name, + resource_tracker_service_runs.c.service_run_id, + resource_tracker_service_runs.c.wallet_name, + resource_tracker_service_runs.c.user_email, + resource_tracker_service_runs.c.root_parent_project_name.label( + "project_name" + ), + resource_tracker_service_runs.c.node_name, + resource_tracker_service_runs.c.service_key, + resource_tracker_service_runs.c.service_version, + resource_tracker_service_runs.c.service_type, + resource_tracker_service_runs.c.started_at, + resource_tracker_service_runs.c.stopped_at, + resource_tracker_credit_transactions.c.osparc_credits, + resource_tracker_credit_transactions.c.transaction_status, + sa.func.coalesce( + _project_tags_subquery.c.project_tags, + sa.cast(sa.text("'{}'"), sa.ARRAY(sa.String)), + ).label("project_tags"), + ) + .select_from( + resource_tracker_service_runs.join( + resource_tracker_credit_transactions, + resource_tracker_service_runs.c.service_run_id + == resource_tracker_credit_transactions.c.service_run_id, + isouter=True, + ).join( + _project_tags_subquery, + resource_tracker_service_runs.c.project_id + == _project_tags_subquery.c.project_uuid_for_rut, + isouter=True, + ) + ) + .where(resource_tracker_service_runs.c.product_name == product_name) + ) + + if user_id: + query = query.where(resource_tracker_service_runs.c.user_id == user_id) + if wallet_id: + query = query.where(resource_tracker_service_runs.c.wallet_id == wallet_id) + if started_from: + query = query.where( + sa.func.DATE(resource_tracker_service_runs.c.started_at) + >= started_from.date() + ) + if started_until: + query = query.where( + sa.func.DATE(resource_tracker_service_runs.c.started_at) + <= started_until.date() + ) + + if order_by: + if order_by.direction == OrderDirection.ASC: + query = query.order_by(sa.asc(order_by.field)) + else: + query = query.order_by(sa.desc(order_by.field)) + else: + # Default ordering + query = query.order_by(resource_tracker_service_runs.c.started_at.desc()) + + compiled_query = ( + str(query.compile(compile_kwargs={"literal_binds": True})) + .replace("\n", "") + .replace("'", "''") + ) + + result = await conn.execute( + sa.DDL( + f""" + SELECT * from aws_s3.query_export_to_s3('{compiled_query}', + aws_commons.create_s3_uri('{s3_bucket_name}', '{s3_key}', '{s3_region}'), 'format csv, HEADER true'); + """ # noqa: S608 + ) + ) + row = result.first() + assert row + _logger.info( + "Rows uploaded %s, Files uploaded %s, Bytes uploaded %s", + row[0], + row[1], + row[2], + ) + + +async def total_service_runs_by_product_and_user_and_wallet( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + user_id: UserID | None, + wallet_id: WalletID | None, + service_run_status: ServiceRunStatus | None = None, + started_from: datetime | None = None, + started_until: datetime | None = None, +) -> PositiveInt: + async with transaction_context(engine, connection) as conn: + query = ( + sa.select(sa.func.count()) + .select_from(resource_tracker_service_runs) + .where(resource_tracker_service_runs.c.product_name == product_name) + ) + + if user_id: + query = query.where(resource_tracker_service_runs.c.user_id == user_id) + if wallet_id: + query = query.where(resource_tracker_service_runs.c.wallet_id == wallet_id) + if started_from: + query = query.where( + sa.func.DATE(resource_tracker_service_runs.c.started_at) + >= started_from.date() + ) + if started_until: + query = query.where( + sa.func.DATE(resource_tracker_service_runs.c.started_at) + <= started_until.date() + ) + if service_run_status: + query = query.where( + resource_tracker_service_runs.c.service_run_status == service_run_status + ) + + result = await conn.execute(query) + row = result.first() + return cast(PositiveInt, row[0]) if row else 0 + + +### For Background check purpose: + + +async def list_service_runs_with_running_status_across_all_products( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + offset: int, + limit: int, +) -> list[ServiceRunForCheckDB]: + async with transaction_context(engine, connection) as conn: + query = ( + sa.select( + resource_tracker_service_runs.c.service_run_id, + resource_tracker_service_runs.c.last_heartbeat_at, + resource_tracker_service_runs.c.missed_heartbeat_counter, + resource_tracker_service_runs.c.modified, + ) + .where( + resource_tracker_service_runs.c.service_run_status + == ServiceRunStatus.RUNNING + ) + .order_by(resource_tracker_service_runs.c.started_at.desc()) # NOTE: + .offset(offset) + .limit(limit) + ) + result = await conn.execute(query) + + return [ServiceRunForCheckDB.model_validate(row) for row in result.fetchall()] + + +async def total_service_runs_with_running_status_across_all_products( + engine: AsyncEngine, connection: AsyncConnection | None = None +) -> PositiveInt: + async with transaction_context(engine, connection) as conn: + query = ( + sa.select(sa.func.count()) + .select_from(resource_tracker_service_runs) + .where( + resource_tracker_service_runs.c.service_run_status + == ServiceRunStatus.RUNNING + ) + ) + result = await conn.execute(query) + row = result.first() + return cast(PositiveInt, row[0]) if row else 0 + + +async def update_service_missed_heartbeat_counter( + engine: AsyncEngine, + connection: AsyncConnection | None = None, + *, + service_run_id: ServiceRunId, + last_heartbeat_at: datetime, + missed_heartbeat_counter: int, +) -> ServiceRunDB | None: + async with transaction_context(engine, connection) as conn: + update_stmt = ( + resource_tracker_service_runs.update() + .values( + modified=sa.func.now(), + missed_heartbeat_counter=missed_heartbeat_counter, + ) + .where( + (resource_tracker_service_runs.c.service_run_id == service_run_id) + & ( + resource_tracker_service_runs.c.service_run_status + == ServiceRunStatus.RUNNING + ) + & ( + resource_tracker_service_runs.c.last_heartbeat_at + == last_heartbeat_at + ) + ) + .returning(sa.literal_column("*")) + ) + + result = await conn.execute(update_stmt) + row = result.first() + if row is None: + return None + return ServiceRunDB.model_validate(row) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_plans.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_plans.py index 9c3dc38bef3..ed34c334187 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_plans.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_plans.py @@ -14,12 +14,13 @@ ) from models_library.services import ServiceKey, ServiceVersion from pydantic import TypeAdapter +from sqlalchemy.ext.asyncio import AsyncEngine -from ..api.rest.dependencies import get_repository +from ..api.rest.dependencies import get_resource_tracker_db_engine from ..exceptions.errors import PricingPlanNotFoundForServiceError from ..models.pricing_plans import PricingPlansDB, PricingPlanToServiceDB from ..models.pricing_units import PricingUnitsDB -from .modules.db.repositories.resource_tracker import ResourceTrackerRepository +from .modules.db import pricing_plans_db async def _create_pricing_plan_get( @@ -52,12 +53,15 @@ async def get_service_default_pricing_plan( product_name: ProductName, service_key: ServiceKey, service_version: ServiceVersion, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], ) -> PricingPlanGet: - active_service_pricing_plans = await resource_tracker_repo.list_active_service_pricing_plans_by_product_and_service( - product_name, service_key, service_version + active_service_pricing_plans = ( + await pricing_plans_db.list_active_service_pricing_plans_by_product_and_service( + db_engine, + product_name=product_name, + service_key=service_key, + service_version=service_version, + ) ) default_pricing_plan = None @@ -71,10 +75,8 @@ async def get_service_default_pricing_plan( service_key=service_key, service_version=service_version ) - pricing_plan_unit_db = ( - await resource_tracker_repo.list_pricing_units_by_pricing_plan( - pricing_plan_id=default_pricing_plan.pricing_plan_id - ) + pricing_plan_unit_db = await pricing_plans_db.list_pricing_units_by_pricing_plan( + db_engine, pricing_plan_id=default_pricing_plan.pricing_plan_id ) return await _create_pricing_plan_get(default_pricing_plan, pricing_plan_unit_db) @@ -83,14 +85,12 @@ async def get_service_default_pricing_plan( async def list_connected_services_to_pricing_plan_by_pricing_plan( product_name: ProductName, pricing_plan_id: PricingPlanId, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], ): output_list: list[ PricingPlanToServiceDB - ] = await resource_tracker_repo.list_connected_services_to_pricing_plan_by_pricing_plan( - product_name=product_name, pricing_plan_id=pricing_plan_id + ] = await pricing_plans_db.list_connected_services_to_pricing_plan_by_pricing_plan( + db_engine, product_name=product_name, pricing_plan_id=pricing_plan_id ) return [ TypeAdapter(PricingPlanToServiceGet).validate_python(item.model_dump()) @@ -103,12 +103,11 @@ async def connect_service_to_pricing_plan( pricing_plan_id: PricingPlanId, service_key: ServiceKey, service_version: ServiceVersion, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], ) -> PricingPlanToServiceGet: output: PricingPlanToServiceDB = ( - await resource_tracker_repo.upsert_service_to_pricing_plan( + await pricing_plans_db.upsert_service_to_pricing_plan( + db_engine, product_name=product_name, pricing_plan_id=pricing_plan_id, service_key=service_key, @@ -120,14 +119,12 @@ async def connect_service_to_pricing_plan( async def list_pricing_plans_by_product( product_name: ProductName, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], ) -> list[PricingPlanGet]: pricing_plans_list_db: list[ PricingPlansDB - ] = await resource_tracker_repo.list_pricing_plans_by_product( - product_name=product_name + ] = await pricing_plans_db.list_pricing_plans_by_product( + db_engine, product_name=product_name ) return [ PricingPlanGet( @@ -147,32 +144,24 @@ async def list_pricing_plans_by_product( async def get_pricing_plan( product_name: ProductName, pricing_plan_id: PricingPlanId, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], ) -> PricingPlanGet: - pricing_plan_db = await resource_tracker_repo.get_pricing_plan( - product_name=product_name, pricing_plan_id=pricing_plan_id + pricing_plan_db = await pricing_plans_db.get_pricing_plan( + db_engine, product_name=product_name, pricing_plan_id=pricing_plan_id ) - pricing_plan_unit_db = ( - await resource_tracker_repo.list_pricing_units_by_pricing_plan( - pricing_plan_id=pricing_plan_db.pricing_plan_id - ) + pricing_plan_unit_db = await pricing_plans_db.list_pricing_units_by_pricing_plan( + db_engine, pricing_plan_id=pricing_plan_db.pricing_plan_id ) return await _create_pricing_plan_get(pricing_plan_db, pricing_plan_unit_db) async def create_pricing_plan( data: PricingPlanCreate, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], ) -> PricingPlanGet: - pricing_plan_db = await resource_tracker_repo.create_pricing_plan(data=data) - pricing_plan_unit_db = ( - await resource_tracker_repo.list_pricing_units_by_pricing_plan( - pricing_plan_id=pricing_plan_db.pricing_plan_id - ) + pricing_plan_db = await pricing_plans_db.create_pricing_plan(db_engine, data=data) + pricing_plan_unit_db = await pricing_plans_db.list_pricing_units_by_pricing_plan( + db_engine, pricing_plan_id=pricing_plan_db.pricing_plan_id ) return await _create_pricing_plan_get(pricing_plan_db, pricing_plan_unit_db) @@ -180,24 +169,20 @@ async def create_pricing_plan( async def update_pricing_plan( product_name: ProductName, data: PricingPlanUpdate, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], ) -> PricingPlanGet: # Check whether pricing plan exists - pricing_plan_db = await resource_tracker_repo.get_pricing_plan( - product_name=product_name, pricing_plan_id=data.pricing_plan_id + pricing_plan_db = await pricing_plans_db.get_pricing_plan( + db_engine, product_name=product_name, pricing_plan_id=data.pricing_plan_id ) # Update pricing plan - pricing_plan_updated_db = await resource_tracker_repo.update_pricing_plan( - product_name=product_name, data=data + pricing_plan_updated_db = await pricing_plans_db.update_pricing_plan( + db_engine, product_name=product_name, data=data ) if pricing_plan_updated_db: pricing_plan_db = pricing_plan_updated_db - pricing_plan_unit_db = ( - await resource_tracker_repo.list_pricing_units_by_pricing_plan( - pricing_plan_id=pricing_plan_db.pricing_plan_id - ) + pricing_plan_unit_db = await pricing_plans_db.list_pricing_units_by_pricing_plan( + db_engine, pricing_plan_id=pricing_plan_db.pricing_plan_id ) return await _create_pricing_plan_get(pricing_plan_db, pricing_plan_unit_db) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_units.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_units.py index f2aee53dd80..0a1e72cad65 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_units.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/pricing_units.py @@ -11,21 +11,23 @@ PricingUnitWithCostCreate, PricingUnitWithCostUpdate, ) +from sqlalchemy.ext.asyncio import AsyncEngine -from ..api.rest.dependencies import get_repository -from .modules.db.repositories.resource_tracker import ResourceTrackerRepository +from ..api.rest.dependencies import get_resource_tracker_db_engine +from .modules.db import pricing_plans_db async def get_pricing_unit( product_name: ProductName, pricing_plan_id: PricingPlanId, pricing_unit_id: PricingUnitId, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], ) -> PricingUnitGet: - pricing_unit = await resource_tracker_repo.get_valid_pricing_unit( - product_name, pricing_plan_id, pricing_unit_id + pricing_unit = await pricing_plans_db.get_valid_pricing_unit( + db_engine, + product_name=product_name, + pricing_plan_id=pricing_plan_id, + pricing_unit_id=pricing_unit_id, ) return PricingUnitGet( @@ -42,21 +44,22 @@ async def get_pricing_unit( async def create_pricing_unit( product_name: ProductName, data: PricingUnitWithCostCreate, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], ) -> PricingUnitGet: # Check whether pricing plan exists - pricing_plan_db = await resource_tracker_repo.get_pricing_plan( - product_name=product_name, pricing_plan_id=data.pricing_plan_id + pricing_plan_db = await pricing_plans_db.get_pricing_plan( + db_engine, product_name=product_name, pricing_plan_id=data.pricing_plan_id ) # Create new pricing unit - pricing_unit_id, _ = await resource_tracker_repo.create_pricing_unit_with_cost( - data=data, pricing_plan_key=pricing_plan_db.pricing_plan_key + pricing_unit_id, _ = await pricing_plans_db.create_pricing_unit_with_cost( + db_engine, data=data, pricing_plan_key=pricing_plan_db.pricing_plan_key ) - pricing_unit = await resource_tracker_repo.get_valid_pricing_unit( - product_name, data.pricing_plan_id, pricing_unit_id + pricing_unit = await pricing_plans_db.get_valid_pricing_unit( + db_engine, + product_name=product_name, + pricing_plan_id=data.pricing_plan_id, + pricing_unit_id=pricing_unit_id, ) return PricingUnitGet( pricing_unit_id=pricing_unit.pricing_unit_id, @@ -72,26 +75,30 @@ async def create_pricing_unit( async def update_pricing_unit( product_name: ProductName, data: PricingUnitWithCostUpdate, - resource_tracker_repo: Annotated[ - ResourceTrackerRepository, Depends(get_repository(ResourceTrackerRepository)) - ], + db_engine: Annotated[AsyncEngine, Depends(get_resource_tracker_db_engine)], ) -> PricingUnitGet: # Check whether pricing unit exists - await resource_tracker_repo.get_valid_pricing_unit( - product_name, data.pricing_plan_id, data.pricing_unit_id + await pricing_plans_db.get_valid_pricing_unit( + db_engine, + product_name=product_name, + pricing_plan_id=data.pricing_plan_id, + pricing_unit_id=data.pricing_unit_id, ) # Get pricing plan - pricing_plan_db = await resource_tracker_repo.get_pricing_plan( - product_name, data.pricing_plan_id + pricing_plan_db = await pricing_plans_db.get_pricing_plan( + db_engine, product_name=product_name, pricing_plan_id=data.pricing_plan_id ) # Update pricing unit and cost - await resource_tracker_repo.update_pricing_unit_with_cost( - data=data, pricing_plan_key=pricing_plan_db.pricing_plan_key + await pricing_plans_db.update_pricing_unit_with_cost( + db_engine, data=data, pricing_plan_key=pricing_plan_db.pricing_plan_key ) - pricing_unit = await resource_tracker_repo.get_valid_pricing_unit( - product_name, data.pricing_plan_id, data.pricing_unit_id + pricing_unit = await pricing_plans_db.get_valid_pricing_unit( + db_engine, + product_name=product_name, + pricing_plan_id=data.pricing_plan_id, + pricing_unit_id=data.pricing_unit_id, ) return PricingUnitGet( pricing_unit_id=pricing_unit.pricing_unit_id, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/process_message_running_service.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/process_message_running_service.py index 4907c84ecb1..8300ede8283 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/process_message_running_service.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/process_message_running_service.py @@ -21,6 +21,7 @@ ) from models_library.services import ServiceType from pydantic import TypeAdapter +from sqlalchemy.ext.asyncio import AsyncEngine from ..models.credit_transactions import ( CreditTransactionCreate, @@ -32,7 +33,7 @@ ServiceRunLastHeartbeatUpdate, ServiceRunStoppedAtUpdate, ) -from .modules.db.repositories.resource_tracker import ResourceTrackerRepository +from .modules.db import credit_transactions_db, pricing_plans_db, service_runs_db from .modules.rabbitmq import RabbitMQClient, get_rabbitmq_client from .utils import ( compute_service_run_credit_costs, @@ -53,24 +54,22 @@ async def process_message(app: FastAPI, data: bytes) -> bool: rabbit_message.message_type, rabbit_message.service_run_id, ) - resource_tracker_repo: ResourceTrackerRepository = ResourceTrackerRepository( - db_engine=app.state.engine - ) + _db_engine = app.state.engine rabbitmq_client = get_rabbitmq_client(app) await RABBIT_MSG_TYPE_TO_PROCESS_HANDLER[rabbit_message.message_type]( - resource_tracker_repo, rabbit_message, rabbitmq_client + _db_engine, rabbit_message, rabbitmq_client ) return True async def _process_start_event( - resource_tracker_repo: ResourceTrackerRepository, + db_engine: AsyncEngine, msg: RabbitResourceTrackingStartedMessage, rabbitmq_client: RabbitMQClient, ): - service_run_db = await resource_tracker_repo.get_service_run_by_id( - service_run_id=msg.service_run_id + service_run_db = await service_runs_db.get_service_run_by_id( + db_engine, service_run_id=msg.service_run_id ) if service_run_db: # NOTE: After we find out why sometimes RUT recieves multiple start events and fix it, we can change it to log level `error` @@ -90,8 +89,8 @@ async def _process_start_event( ) pricing_unit_cost = None if msg.pricing_unit_cost_id: - pricing_unit_cost_db = await resource_tracker_repo.get_pricing_unit_cost_by_id( - pricing_unit_cost_id=msg.pricing_unit_cost_id + pricing_unit_cost_db = await pricing_plans_db.get_pricing_unit_cost_by_id( + db_engine, pricing_unit_cost_id=msg.pricing_unit_cost_id ) pricing_unit_cost = pricing_unit_cost_db.cost_per_unit @@ -125,7 +124,9 @@ async def _process_start_event( service_run_status=ServiceRunStatus.RUNNING, last_heartbeat_at=msg.created_at, ) - service_run_id = await resource_tracker_repo.create_service_run(create_service_run) + service_run_id = await service_runs_db.create_service_run( + db_engine, data=create_service_run + ) if msg.wallet_id and msg.wallet_name: transaction_create = CreditTransactionCreate( @@ -145,21 +146,23 @@ async def _process_start_event( created_at=msg.created_at, last_heartbeat_at=msg.created_at, ) - await resource_tracker_repo.create_credit_transaction(transaction_create) + await credit_transactions_db.create_credit_transaction( + db_engine, data=transaction_create + ) # Publish wallet total credits to RabbitMQ await sum_credit_transactions_and_publish_to_rabbitmq( - resource_tracker_repo, rabbitmq_client, msg.product_name, msg.wallet_id + db_engine, rabbitmq_client, msg.product_name, msg.wallet_id ) async def _process_heartbeat_event( - resource_tracker_repo: ResourceTrackerRepository, + db_engine: AsyncEngine, msg: RabbitResourceTrackingHeartbeatMessage, rabbitmq_client: RabbitMQClient, ): - service_run_db = await resource_tracker_repo.get_service_run_by_id( - service_run_id=msg.service_run_id + service_run_db = await service_runs_db.get_service_run_by_id( + db_engine, service_run_id=msg.service_run_id ) if not service_run_db: _logger.error( @@ -181,8 +184,8 @@ async def _process_heartbeat_event( update_service_run_last_heartbeat = ServiceRunLastHeartbeatUpdate( service_run_id=msg.service_run_id, last_heartbeat_at=msg.created_at ) - running_service = await resource_tracker_repo.update_service_run_last_heartbeat( - update_service_run_last_heartbeat + running_service = await service_runs_db.update_service_run_last_heartbeat( + db_engine, data=update_service_run_last_heartbeat ) if running_service is None: _logger.info("Nothing to update: %s", msg) @@ -201,19 +204,19 @@ async def _process_heartbeat_event( osparc_credits=make_negative(computed_credits), last_heartbeat_at=msg.created_at, ) - await resource_tracker_repo.update_credit_transaction_credits( - update_credit_transaction + await credit_transactions_db.update_credit_transaction_credits( + db_engine, data=update_credit_transaction ) # Publish wallet total credits to RabbitMQ wallet_total_credits = await sum_credit_transactions_and_publish_to_rabbitmq( - resource_tracker_repo, + db_engine, rabbitmq_client, running_service.product_name, running_service.wallet_id, ) if wallet_total_credits.available_osparc_credits < CreditsLimit.OUT_OF_CREDITS: await publish_to_rabbitmq_wallet_credits_limit_reached( - resource_tracker_repo, + db_engine, rabbitmq_client, product_name=running_service.product_name, wallet_id=running_service.wallet_id, @@ -223,12 +226,12 @@ async def _process_heartbeat_event( async def _process_stop_event( - resource_tracker_repo: ResourceTrackerRepository, + db_engine: AsyncEngine, msg: RabbitResourceTrackingStoppedMessage, rabbitmq_client: RabbitMQClient, ): - service_run_db = await resource_tracker_repo.get_service_run_by_id( - service_run_id=msg.service_run_id + service_run_db = await service_runs_db.get_service_run_by_id( + db_engine, service_run_id=msg.service_run_id ) if not service_run_db: # NOTE: ANE/MD discussed. When the RUT receives a stop event and has not received before any start or heartbeat event, it probably means that @@ -262,8 +265,8 @@ async def _process_stop_event( service_run_status_msg=_run_status_msg, ) - running_service = await resource_tracker_repo.update_service_run_stopped_at( - update_service_run_stopped_at + running_service = await service_runs_db.update_service_run_stopped_at( + db_engine, data=update_service_run_stopped_at ) if running_service is None: @@ -287,12 +290,12 @@ async def _process_stop_event( else CreditTransactionStatus.NOT_BILLED ), ) - await resource_tracker_repo.update_credit_transaction_credits_and_status( - update_credit_transaction + await credit_transactions_db.update_credit_transaction_credits_and_status( + db_engine, data=update_credit_transaction ) # Publish wallet total credits to RabbitMQ await sum_credit_transactions_and_publish_to_rabbitmq( - resource_tracker_repo, + db_engine, rabbitmq_client, running_service.product_name, running_service.wallet_id, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py index fff896c8ec0..b4d9127733e 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/service_runs.py @@ -19,9 +19,10 @@ from models_library.users import UserID from models_library.wallets import WalletID from pydantic import AnyUrl, PositiveInt, TypeAdapter +from sqlalchemy.ext.asyncio import AsyncEngine from ..models.service_runs import ServiceRunWithCreditsDB -from .modules.db.repositories.resource_tracker import ResourceTrackerRepository +from .modules.db import service_runs_db _PRESIGNED_LINK_EXPIRATION_SEC = 7200 @@ -29,7 +30,7 @@ async def list_service_runs( user_id: UserID, product_name: ProductName, - resource_tracker_repo: ResourceTrackerRepository, + db_engine: AsyncEngine, limit: int = 20, offset: int = 0, wallet_id: WalletID | None = None, @@ -45,17 +46,21 @@ async def list_service_runs( # Situation when we want to see all usage of a specific user (ex. for Non billable product) if wallet_id is None and access_all_wallet_usage is False: - total_service_runs: PositiveInt = await resource_tracker_repo.total_service_runs_by_product_and_user_and_wallet( - product_name, - user_id=user_id, - wallet_id=None, - started_from=started_from, - started_until=started_until, + total_service_runs: PositiveInt = ( + await service_runs_db.total_service_runs_by_product_and_user_and_wallet( + db_engine, + product_name=product_name, + user_id=user_id, + wallet_id=None, + started_from=started_from, + started_until=started_until, + ) ) service_runs_db_model: list[ ServiceRunWithCreditsDB - ] = await resource_tracker_repo.list_service_runs_by_product_and_user_and_wallet( - product_name, + ] = await service_runs_db.list_service_runs_by_product_and_user_and_wallet( + db_engine, + product_name=product_name, user_id=user_id, wallet_id=None, offset=offset, @@ -66,8 +71,9 @@ async def list_service_runs( ) # Situation when accountant user can see all users usage of the wallet elif wallet_id and access_all_wallet_usage is True: - total_service_runs: PositiveInt = await resource_tracker_repo.total_service_runs_by_product_and_user_and_wallet( # type: ignore[no-redef] - product_name, + total_service_runs: PositiveInt = await service_runs_db.total_service_runs_by_product_and_user_and_wallet( # type: ignore[no-redef] + db_engine, + product_name=product_name, user_id=None, wallet_id=wallet_id, started_from=started_from, @@ -75,8 +81,9 @@ async def list_service_runs( ) service_runs_db_model: list[ # type: ignore[no-redef] ServiceRunWithCreditsDB - ] = await resource_tracker_repo.list_service_runs_by_product_and_user_and_wallet( - product_name, + ] = await service_runs_db.list_service_runs_by_product_and_user_and_wallet( + db_engine, + product_name=product_name, user_id=None, wallet_id=wallet_id, offset=offset, @@ -87,8 +94,9 @@ async def list_service_runs( ) # Situation when regular user can see only his usage of the wallet elif wallet_id and access_all_wallet_usage is False: - total_service_runs: PositiveInt = await resource_tracker_repo.total_service_runs_by_product_and_user_and_wallet( # type: ignore[no-redef] - product_name, + total_service_runs: PositiveInt = await service_runs_db.total_service_runs_by_product_and_user_and_wallet( # type: ignore[no-redef] + db_engine, + product_name=product_name, user_id=user_id, wallet_id=wallet_id, started_from=started_from, @@ -96,8 +104,9 @@ async def list_service_runs( ) service_runs_db_model: list[ # type: ignore[no-redef] ServiceRunWithCreditsDB - ] = await resource_tracker_repo.list_service_runs_by_product_and_user_and_wallet( - product_name, + ] = await service_runs_db.list_service_runs_by_product_and_user_and_wallet( + db_engine, + product_name=product_name, user_id=user_id, wallet_id=wallet_id, offset=offset, @@ -147,7 +156,7 @@ async def export_service_runs( s3_region: str, user_id: UserID, product_name: ProductName, - resource_tracker_repo: ResourceTrackerRepository, + db_engine: AsyncEngine, wallet_id: WalletID | None = None, access_all_wallet_usage: bool = False, order_by: OrderBy | None = None, @@ -165,7 +174,8 @@ async def export_service_runs( ) # Export CSV to S3 - await resource_tracker_repo.export_service_runs_table_to_s3( + await service_runs_db.export_service_runs_table_to_s3( + db_engine, product_name=product_name, s3_bucket_name=s3_bucket_name, s3_key=s3_object_key, @@ -188,7 +198,7 @@ async def export_service_runs( async def get_osparc_credits_aggregated_usages_page( user_id: UserID, product_name: ProductName, - resource_tracker_repo: ResourceTrackerRepository, + db_engine: AsyncEngine, aggregated_by: ServicesAggregatedUsagesType, time_period: ServicesAggregatedUsagesTimePeriod, wallet_id: WalletID, @@ -204,7 +214,8 @@ async def get_osparc_credits_aggregated_usages_page( ( count_output_list_db, output_list_db, - ) = await resource_tracker_repo.get_osparc_credits_aggregated_by_service( + ) = await service_runs_db.get_osparc_credits_aggregated_by_service( + db_engine, product_name=product_name, user_id=user_id if access_all_wallet_usage is False else None, wallet_id=wallet_id, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/utils.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/utils.py index 73aa7416244..6047ac2e904 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/utils.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/utils.py @@ -19,8 +19,9 @@ from models_library.wallets import WalletID from pydantic import PositiveInt from servicelib.rabbitmq import RabbitMQClient +from sqlalchemy.ext.asyncio import AsyncEngine -from .modules.db.repositories.resource_tracker import ResourceTrackerRepository +from .modules.db import credit_transactions_db, service_runs_db _logger = logging.getLogger(__name__) @@ -30,15 +31,16 @@ def make_negative(n): async def sum_credit_transactions_and_publish_to_rabbitmq( - resource_tracker_repo: ResourceTrackerRepository, + db_engine: AsyncEngine, rabbitmq_client: RabbitMQClient, product_name: ProductName, wallet_id: WalletID, ) -> WalletTotalCredits: wallet_total_credits = ( - await resource_tracker_repo.sum_credit_transactions_by_product_and_wallet( - product_name, - wallet_id, + await credit_transactions_db.sum_credit_transactions_by_product_and_wallet( + db_engine, + product_name=product_name, + wallet_id=wallet_id, ) ) publish_message = WalletCreditsMessage.model_construct( @@ -77,7 +79,7 @@ async def _publish_to_rabbitmq_wallet_credits_limit_reached( async def publish_to_rabbitmq_wallet_credits_limit_reached( - resource_tracker_repo: ResourceTrackerRepository, + db_engine: AsyncEngine, rabbitmq_client: RabbitMQClient, product_name: ProductName, wallet_id: WalletID, @@ -86,8 +88,9 @@ async def publish_to_rabbitmq_wallet_credits_limit_reached( ): # Get all current running services for that wallet total_count: PositiveInt = ( - await resource_tracker_repo.total_service_runs_by_product_and_user_and_wallet( - product_name, + await service_runs_db.total_service_runs_by_product_and_user_and_wallet( + db_engine, + product_name=product_name, user_id=None, wallet_id=wallet_id, service_run_status=ServiceRunStatus.RUNNING, @@ -95,13 +98,16 @@ async def publish_to_rabbitmq_wallet_credits_limit_reached( ) for offset in range(0, total_count, _BATCH_SIZE): - batch_services = await resource_tracker_repo.list_service_runs_by_product_and_user_and_wallet( - product_name, - user_id=None, - wallet_id=wallet_id, - offset=offset, - limit=_BATCH_SIZE, - service_run_status=ServiceRunStatus.RUNNING, + batch_services = ( + await service_runs_db.list_service_runs_by_product_and_user_and_wallet( + db_engine, + product_name=product_name, + user_id=None, + wallet_id=wallet_id, + offset=offset, + limit=_BATCH_SIZE, + service_run_status=ServiceRunStatus.RUNNING, + ) ) await asyncio.gather( diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py index 56c9c102df6..44a6ce56016 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py @@ -31,7 +31,7 @@ @pytest.fixture async def mocked_export(mocker: MockerFixture) -> AsyncMock: return mocker.patch( - "simcore_service_resource_usage_tracker.services.service_runs.ResourceTrackerRepository.export_service_runs_table_to_s3", + "simcore_service_resource_usage_tracker.services.service_runs.service_runs_db.export_service_runs_table_to_s3", autospec=True, ) diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task_periodic_heartbeat_check.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task_periodic_heartbeat_check.py index 35114a3cdf6..8ebe34bbd2d 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task_periodic_heartbeat_check.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task_periodic_heartbeat_check.py @@ -23,9 +23,6 @@ from simcore_service_resource_usage_tracker.services.background_task_periodic_heartbeat_check import ( periodic_check_of_running_services_task, ) -from simcore_service_resource_usage_tracker.services.modules.db.repositories.resource_tracker import ( - ResourceTrackerRepository, -) pytest_simcore_core_services_selection = ["postgres", "rabbit"] pytest_simcore_ops_services_selection = [ @@ -132,9 +129,6 @@ async def test_process_event_functions( ): engine = initialized_app.state.engine app_settings: ApplicationSettings = initialized_app.state.settings - resource_tracker_repo: ResourceTrackerRepository = ResourceTrackerRepository( - db_engine=engine - ) for _ in range(app_settings.RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL): await periodic_check_of_running_services_task(initialized_app) diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py index da321f593f3..57eb9735e68 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message.py @@ -8,9 +8,6 @@ SimcorePlatformStatus, ) from servicelib.rabbitmq import RabbitMQClient -from simcore_service_resource_usage_tracker.services.modules.db.repositories.resource_tracker import ( - ResourceTrackerRepository, -) from simcore_service_resource_usage_tracker.services.process_message_running_service import ( _process_heartbeat_event, _process_start_event, @@ -43,10 +40,7 @@ async def test_process_event_functions( pricing_unit_id=None, pricing_unit_cost_id=None, ) - resource_tracker_repo: ResourceTrackerRepository = ResourceTrackerRepository( - db_engine=engine - ) - await _process_start_event(resource_tracker_repo, msg, publisher) + await _process_start_event(engine, msg, publisher) output = await assert_service_runs_db_row(postgres_db, msg.service_run_id) assert output.stopped_at is None assert output.service_run_status == "RUNNING" @@ -55,7 +49,7 @@ async def test_process_event_functions( heartbeat_msg = RabbitResourceTrackingHeartbeatMessage( service_run_id=msg.service_run_id, created_at=datetime.now(tz=timezone.utc) ) - await _process_heartbeat_event(resource_tracker_repo, heartbeat_msg, publisher) + await _process_heartbeat_event(engine, heartbeat_msg, publisher) output = await assert_service_runs_db_row(postgres_db, msg.service_run_id) assert output.stopped_at is None assert output.service_run_status == "RUNNING" @@ -66,7 +60,7 @@ async def test_process_event_functions( created_at=datetime.now(tz=timezone.utc), simcore_platform_status=SimcorePlatformStatus.OK, ) - await _process_stop_event(resource_tracker_repo, stopped_msg, publisher) + await _process_stop_event(engine, stopped_msg, publisher) output = await assert_service_runs_db_row(postgres_db, msg.service_run_id) assert output.stopped_at is not None assert output.service_run_status == "SUCCESS" diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py index 637a2219f94..b29863f0b57 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py @@ -31,9 +31,6 @@ resource_tracker_pricing_units, ) from simcore_postgres_database.models.services import services_meta_data -from simcore_service_resource_usage_tracker.services.modules.db.repositories.resource_tracker import ( - ResourceTrackerRepository, -) from simcore_service_resource_usage_tracker.services.process_message_running_service import ( _process_heartbeat_event, _process_start_event, @@ -207,10 +204,8 @@ async def test_process_event_functions( pricing_unit_id=1, pricing_unit_cost_id=1, ) - resource_tracker_repo: ResourceTrackerRepository = ResourceTrackerRepository( - db_engine=engine - ) - await _process_start_event(resource_tracker_repo, msg, publisher) + + await _process_start_event(engine, msg, publisher) output = await assert_credit_transactions_db_row(postgres_db, msg.service_run_id) assert output.osparc_credits == 0.0 assert output.transaction_status == "PENDING" @@ -222,7 +217,7 @@ async def test_process_event_functions( heartbeat_msg = RabbitResourceTrackingHeartbeatMessage( service_run_id=msg.service_run_id, created_at=datetime.now(tz=timezone.utc) ) - await _process_heartbeat_event(resource_tracker_repo, heartbeat_msg, publisher) + await _process_heartbeat_event(engine, heartbeat_msg, publisher) output = await assert_credit_transactions_db_row( postgres_db, msg.service_run_id, modified_at ) @@ -240,7 +235,7 @@ async def test_process_event_functions( created_at=datetime.now(tz=timezone.utc), simcore_platform_status=SimcorePlatformStatus.OK, ) - await _process_stop_event(resource_tracker_repo, stopped_msg, publisher) + await _process_stop_event(engine, stopped_msg, publisher) output = await assert_credit_transactions_db_row( postgres_db, msg.service_run_id, modified_at ) diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py index 5b903cf759d..ccffbc9f42e 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py @@ -31,9 +31,6 @@ resource_tracker_pricing_units, ) from simcore_postgres_database.models.services import services_meta_data -from simcore_service_resource_usage_tracker.services.modules.db.repositories.resource_tracker import ( - ResourceTrackerRepository, -) from simcore_service_resource_usage_tracker.services.process_message_running_service import ( _process_heartbeat_event, _process_start_event, @@ -149,10 +146,8 @@ async def test_process_event_functions( pricing_unit_id=1, pricing_unit_cost_id=1, ) - resource_tracker_repo: ResourceTrackerRepository = ResourceTrackerRepository( - db_engine=engine - ) - await _process_start_event(resource_tracker_repo, msg, publisher) + + await _process_start_event(engine, msg, publisher) output = await assert_credit_transactions_db_row(postgres_db, msg.service_run_id) assert output.osparc_credits == 0.0 assert output.transaction_status == "PENDING" @@ -164,7 +159,7 @@ async def test_process_event_functions( heartbeat_msg = RabbitResourceTrackingHeartbeatMessage( service_run_id=msg.service_run_id, created_at=datetime.now(tz=timezone.utc) ) - await _process_heartbeat_event(resource_tracker_repo, heartbeat_msg, publisher) + await _process_heartbeat_event(engine, heartbeat_msg, publisher) output = await assert_credit_transactions_db_row( postgres_db, msg.service_run_id, modified_at ) @@ -177,7 +172,7 @@ async def test_process_event_functions( created_at=datetime.now(tz=timezone.utc), simcore_platform_status=SimcorePlatformStatus.OK, ) - await _process_stop_event(resource_tracker_repo, stopped_msg, publisher) + await _process_stop_event(engine, stopped_msg, publisher) output = await assert_credit_transactions_db_row( postgres_db, msg.service_run_id, modified_at ) From fd4a43fc8d08c9390c625b119ea5846217d635f3 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Sun, 1 Dec 2024 15:41:38 +0100 Subject: [PATCH 02/26] fix --- .../services/modules/db/service_runs_db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/service_runs_db.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/service_runs_db.py index b7452604870..a4ea563803d 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/service_runs_db.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/modules/db/service_runs_db.py @@ -1,3 +1,4 @@ +# pylint: disable=too-many-arguments import logging from datetime import datetime from typing import cast From 561c284fa5b63496f550d4c86129f279d0a7e222 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 3 Dec 2024 15:34:05 +0100 Subject: [PATCH 03/26] adding shortuuid --- .../postgres-database/requirements/_base.in | 1 + .../postgres-database/requirements/_base.txt | 47 +++++++++++-------- .../requirements/_migration.txt | 2 +- .../postgres-database/requirements/_test.txt | 17 +++---- .../postgres-database/requirements/_tools.txt | 36 +++++++------- 5 files changed, 56 insertions(+), 47 deletions(-) diff --git a/packages/postgres-database/requirements/_base.in b/packages/postgres-database/requirements/_base.in index 645b7aae0fb..0b809a54e17 100644 --- a/packages/postgres-database/requirements/_base.in +++ b/packages/postgres-database/requirements/_base.in @@ -10,3 +10,4 @@ pydantic sqlalchemy[postgresql_psycopg2binary,postgresql_asyncpg] # SEE extras in https://github.com/sqlalchemy/sqlalchemy/blob/main/setup.cfg#L43 opentelemetry-instrumentation-asyncpg yarl +shortuuid diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt index 4eddd14e0e4..1f9b551837b 100644 --- a/packages/postgres-database/requirements/_base.txt +++ b/packages/postgres-database/requirements/_base.txt @@ -1,5 +1,11 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile _base.in +# alembic==1.13.3 - # via -r requirements/_base.in + # via -r _base.in annotated-types==0.7.0 # via pydantic async-timeout==4.0.3 @@ -18,8 +24,8 @@ importlib-metadata==8.4.0 # via opentelemetry-api mako==1.3.5 # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt + # -c ../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt # alembic markupsafe==2.1.5 # via mako @@ -33,34 +39,34 @@ opentelemetry-api==1.27.0 opentelemetry-instrumentation==0.48b0 # via opentelemetry-instrumentation-asyncpg opentelemetry-instrumentation-asyncpg==0.48b0 - # via -r requirements/_base.in + # via -r _base.in opentelemetry-semantic-conventions==0.48b0 # via opentelemetry-instrumentation-asyncpg orjson==3.10.11 # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/common-library/requirements/_base.in + # -c ../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt + # -r ../../../packages/common-library/requirements/_base.in psycopg2-binary==2.9.9 # via sqlalchemy pydantic==2.9.2 # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/_base.in + # -c ../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt + # -r ../../../packages/common-library/requirements/_base.in + # -r _base.in # pydantic-extra-types pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.10.0 - # via -r requirements/../../../packages/common-library/requirements/_base.in -setuptools==75.2.0 - # via opentelemetry-instrumentation -sqlalchemy==1.4.54 + # via -r ../../../packages/common-library/requirements/_base.in +shortuuid==1.0.13 + # via -r _base.in +sqlalchemy[postgresql-asyncpg,postgresql-psycopg2binary,postgresql_asyncpg,postgresql_psycopg2binary]==1.4.54 # via - # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/_base.in + # -c ../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c ../../../requirements/constraints.txt + # -r _base.in # alembic typing-extensions==4.12.2 # via @@ -73,6 +79,9 @@ wrapt==1.16.0 # deprecated # opentelemetry-instrumentation yarl==1.12.1 - # via -r requirements/_base.in + # via -r _base.in zipp==3.20.2 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/packages/postgres-database/requirements/_migration.txt b/packages/postgres-database/requirements/_migration.txt index a0dd4d6577f..903b84d3681 100644 --- a/packages/postgres-database/requirements/_migration.txt +++ b/packages/postgres-database/requirements/_migration.txt @@ -6,7 +6,7 @@ certifi==2024.8.30 # via # -c requirements/../../../requirements/constraints.txt # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via -r requirements/_migration.in diff --git a/packages/postgres-database/requirements/_test.txt b/packages/postgres-database/requirements/_test.txt index d6059bacd37..49636a365c3 100644 --- a/packages/postgres-database/requirements/_test.txt +++ b/packages/postgres-database/requirements/_test.txt @@ -6,11 +6,11 @@ async-timeout==4.0.3 # aiopg attrs==24.2.0 # via pytest-docker -coverage==7.6.1 +coverage==7.6.8 # via # -r requirements/_test.in # pytest-cov -faker==29.0.0 +faker==33.1.0 # via -r requirements/_test.in greenlet==3.1.1 # via @@ -19,11 +19,11 @@ greenlet==3.1.1 # sqlalchemy iniconfig==2.0.0 # via pytest -mypy==1.12.0 +mypy==1.13.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy -packaging==24.1 +packaging==24.2 # via pytest pluggy==1.5.0 # via pytest @@ -32,7 +32,7 @@ psycopg2-binary==2.9.9 # -c requirements/_base.txt # aiopg # sqlalchemy -pytest==8.3.3 +pytest==8.3.4 # via # -r requirements/_test.in # pytest-asyncio @@ -43,7 +43,7 @@ pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via -r requirements/_test.in pytest-docker==3.1.1 # via -r requirements/_test.in @@ -70,14 +70,15 @@ sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy types-docker==7.1.0.20240827 # via -r requirements/_test.in -types-psycopg2==2.9.21.20240819 +types-psycopg2==2.9.21.20241019 # via -r requirements/_test.in -types-requests==2.32.0.20240914 +types-requests==2.32.0.20241016 # via types-docker typing-extensions==4.12.2 # via # -c requirements/_base.txt # -c requirements/_migration.txt + # faker # mypy # sqlalchemy2-stubs urllib3==2.2.3 diff --git a/packages/postgres-database/requirements/_tools.txt b/packages/postgres-database/requirements/_tools.txt index 61c9a3ec7e1..d86067d0d10 100644 --- a/packages/postgres-database/requirements/_tools.txt +++ b/packages/postgres-database/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -12,13 +12,13 @@ click==8.1.7 # via # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv -identify==2.6.1 +identify==2.6.3 # via pre-commit isort==5.13.2 # via @@ -26,7 +26,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.12.0 +mypy==1.13.0 # via # -c requirements/_test.txt # -r requirements/../../../requirements/devenv.txt @@ -37,14 +37,14 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.1 +packaging==24.2 # via # -c requirements/_test.txt # black # build pathspec==0.12.1 # via black -pip==24.2 +pip==24.3.1 # via pip-tools pip-tools==7.4.1 # via -r requirements/../../../requirements/devenv.txt @@ -53,11 +53,11 @@ platformdirs==4.3.6 # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.0 +pylint==3.3.2 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools @@ -66,12 +66,10 @@ pyyaml==6.0.2 # -c requirements/../../../requirements/constraints.txt # -c requirements/_test.txt # pre-commit -ruff==0.6.7 +ruff==0.8.1 # via -r requirements/../../../requirements/devenv.txt -setuptools==75.2.0 - # via - # -c requirements/_base.txt - # pip-tools +setuptools==75.6.0 + # via pip-tools tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 @@ -79,7 +77,7 @@ typing-extensions==4.12.2 # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.28.0 # via pre-commit -wheel==0.44.0 +wheel==0.45.1 # via pip-tools From 9f060de548cf3787529f6baa9888fa68391b1835 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 3 Dec 2024 15:55:44 +0100 Subject: [PATCH 04/26] add license db tables --- .../dd0d2a5a993b_add_license_db_tables.py | 137 ++++++++++++++++++ .../models/license_packages.py | 67 +++++++++ .../resource_tracker_license_checkouts.py | 86 +++++++++++ .../resource_tracker_license_purchases.py | 66 +++++++++ 4 files changed, 356 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/dd0d2a5a993b_add_license_db_tables.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/license_packages.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_checkouts.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/dd0d2a5a993b_add_license_db_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/dd0d2a5a993b_add_license_db_tables.py new file mode 100644 index 00000000000..3fd4e33d53e --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/dd0d2a5a993b_add_license_db_tables.py @@ -0,0 +1,137 @@ +"""add license db tables + +Revision ID: dd0d2a5a993b +Revises: e05bdc5b3c7b +Create Date: 2024-12-03 14:55:22.308786+00:00 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "dd0d2a5a993b" +down_revision = "e05bdc5b3c7b" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "resource_tracker_license_purchases", + sa.Column("license_purchase_id", sa.String(), nullable=False), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column("license_package_id", sa.BigInteger(), nullable=False), + sa.Column("wallet_id", sa.BigInteger(), nullable=False), + sa.Column( + "start_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "expire_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("purchased_by_user", sa.BigInteger(), nullable=False), + sa.Column( + "purchased_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("license_purchase_id"), + ) + op.create_table( + "resource_tracker_license_checkouts", + sa.Column("license_checkout_id", sa.BigInteger(), nullable=False), + sa.Column("license_package_id", sa.BigInteger(), nullable=True), + sa.Column("wallet_id", sa.BigInteger(), nullable=False), + sa.Column("user_id", sa.BigInteger(), nullable=False), + sa.Column("user_email", sa.String(), nullable=True), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column("service_run_id", sa.String(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("stopped_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("num_of_seats", sa.SmallInteger(), nullable=False), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["service_run_id"], + ["resource_tracker_service_runs.service_run_id"], + name="fk_resource_tracker_license_checkouts_service_run_id", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("license_checkout_id"), + ) + op.create_index( + op.f("ix_resource_tracker_license_checkouts_wallet_id"), + "resource_tracker_license_checkouts", + ["wallet_id"], + unique=False, + ) + op.create_table( + "license_packages", + sa.Column("license_package_id", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column( + "license_resource_type", + sa.Enum("VIP_MODEL", name="licenseresourcetype"), + nullable=False, + ), + sa.Column("pricing_plan_id", sa.BigInteger(), nullable=False), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["pricing_plan_id"], + ["resource_tracker_pricing_plans.pricing_plan_id"], + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["product_name"], + ["products.name"], + name="fk_resource_tracker_license_packages_product_name", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("license_package_id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("license_packages") + op.drop_index( + op.f("ix_resource_tracker_license_checkouts_wallet_id"), + table_name="resource_tracker_license_checkouts", + ) + op.drop_table("resource_tracker_license_checkouts") + op.drop_table("resource_tracker_license_purchases") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/license_packages.py b/packages/postgres-database/src/simcore_postgres_database/models/license_packages.py new file mode 100644 index 00000000000..9c587804109 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/license_packages.py @@ -0,0 +1,67 @@ +""" resource_tracker_service_runs table +""" + +import enum + +import shortuuid +import sqlalchemy as sa + +from ._common import RefActions, column_created_datetime, column_modified_datetime +from .base import metadata + + +def _custom_id_generator(): + return f"lpa_{shortuuid.uuid()}" + + +class LicenseResourceType(str, enum.Enum): + VIP_MODEL = "VIP_MODEL" + + +license_packages = sa.Table( + "license_packages", + metadata, + sa.Column( + "license_package_id", + sa.String, + nullable=False, + primary_key=True, + default=_custom_id_generator, + ), + sa.Column( + "name", + sa.String, + nullable=False, + ), + sa.Column( + "license_resource_type", + sa.Enum(LicenseResourceType), + nullable=False, + doc="Item type, ex. VIP_MODEL", + ), + sa.Column( + "pricing_plan_id", + sa.BigInteger, + sa.ForeignKey( + "resource_tracker_pricing_plans.pricing_plan_id", + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.RESTRICT, + ), + nullable=False, + ), + sa.Column( + "product_name", + sa.String, + sa.ForeignKey( + "products.name", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_resource_tracker_license_packages_product_name", + ), + nullable=False, + doc="Product name", + ), + column_created_datetime(timezone=True), + column_modified_datetime(timezone=True), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_checkouts.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_checkouts.py new file mode 100644 index 00000000000..3cac6555f51 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_checkouts.py @@ -0,0 +1,86 @@ +""" resource_tracker_service_runs table +""" + +import shortuuid +import sqlalchemy as sa + +from ._common import RefActions, column_modified_datetime +from .base import metadata + + +def _custom_id_generator(): + return f"rlc_{shortuuid.uuid()}" + + +resource_tracker_license_checkouts = sa.Table( + "resource_tracker_license_checkouts", + metadata, + sa.Column( + "license_checkout_id", + sa.BigInteger, + nullable=False, + primary_key=True, + default=_custom_id_generator, + ), + sa.Column( + "license_package_id", + sa.BigInteger, + nullable=True, + ), + sa.Column( + "wallet_id", + sa.BigInteger, + nullable=False, + index=True, + ), + sa.Column( + "user_id", + sa.BigInteger, + nullable=False, + ), + sa.Column( + "user_email", + sa.String, + nullable=True, + ), + sa.Column("product_name", sa.String, nullable=False, doc="Product name"), + sa.Column( + "service_run_id", + sa.String, + sa.ForeignKey( + "resource_tracker_service_runs.service_run_id", + name="fk_resource_tracker_license_checkouts_service_run_id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.RESTRICT, + ), + nullable=True, + ), + sa.Column( + "started_at", + sa.DateTime(timezone=True), + nullable=False, + doc="Timestamp when the service was started", + ), + sa.Column( + "stopped_at", + sa.DateTime(timezone=True), + nullable=True, + doc="Timestamp when the service was stopped", + ), + sa.Column( + "num_of_seats", + sa.SmallInteger, + nullable=False, + ), + column_modified_datetime(timezone=True), +) + +# We define the partial index +# sa.Index( +# "ix_resource_tracker_credit_transactions_status_running", +# resource_tracker_service_runs.c.service_run_status, +# postgresql_where=( +# resource_tracker_service_runs.c.service_run_status +# == ResourceTrackerServiceRunStatus.RUNNING +# ), +# ) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py new file mode 100644 index 00000000000..48abfe4df43 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py @@ -0,0 +1,66 @@ +""" resource_tracker_service_runs table +""" + + +import shortuuid +import sqlalchemy as sa + +from ._common import column_modified_datetime +from .base import metadata + + +def _custom_id_generator(): + return f"rlp_{shortuuid.uuid()}" + + +resource_tracker_license_purchases = sa.Table( + "resource_tracker_license_purchases", + metadata, + sa.Column( + "license_purchase_id", + sa.String, + nullable=False, + primary_key=True, + default=_custom_id_generator, + ), + sa.Column( + "product_name", + sa.String, + nullable=False, + doc="Product name", + ), + sa.Column( + "license_package_id", + sa.BigInteger, + nullable=False, + ), + sa.Column( + "wallet_id", + sa.BigInteger, + nullable=False, + ), + sa.Column( + "start_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.sql.func.now(), + ), + sa.Column( + "expire_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.sql.func.now(), + ), + sa.Column( + "purchased_by_user", + sa.BigInteger, + nullable=False, + ), + sa.Column( + "purchased_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.sql.func.now(), + ), + column_modified_datetime(timezone=True), +) From 6f9c844f1c292f216c339855ac3c99e65ac1d04b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 3 Dec 2024 16:14:56 +0100 Subject: [PATCH 05/26] upgrade postgres package - shortuuid --- services/web/server/requirements/_base.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index 11bc97fb4bb..5d607aa94fe 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -702,6 +702,10 @@ setuptools==69.1.1 # opentelemetry-instrumentation shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via + # -r requirements/../../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via # jsonschema @@ -854,7 +858,9 @@ yarl==1.9.4 # via # -c requirements/./constraints.txt # -r requirements/../../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../../packages/service-library/requirements/_base.in # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq From c296b0da2ebaad1edbe7405c4d32d6db5cc12a67 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 3 Dec 2024 16:32:11 +0100 Subject: [PATCH 06/26] upgrade postgres package - shortuuid --- ... => 707fe8c2e7f5_add_license_db_tables.py} | 19 ++++++++++------- .../resource_tracker_license_checkouts.py | 21 ++++++++++++------- 2 files changed, 24 insertions(+), 16 deletions(-) rename packages/postgres-database/src/simcore_postgres_database/migration/versions/{dd0d2a5a993b_add_license_db_tables.py => 707fe8c2e7f5_add_license_db_tables.py} (89%) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/dd0d2a5a993b_add_license_db_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/707fe8c2e7f5_add_license_db_tables.py similarity index 89% rename from packages/postgres-database/src/simcore_postgres_database/migration/versions/dd0d2a5a993b_add_license_db_tables.py rename to packages/postgres-database/src/simcore_postgres_database/migration/versions/707fe8c2e7f5_add_license_db_tables.py index 3fd4e33d53e..d2df7d12ab9 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/dd0d2a5a993b_add_license_db_tables.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/707fe8c2e7f5_add_license_db_tables.py @@ -1,15 +1,15 @@ """add license db tables -Revision ID: dd0d2a5a993b +Revision ID: 707fe8c2e7f5 Revises: e05bdc5b3c7b -Create Date: 2024-12-03 14:55:22.308786+00:00 +Create Date: 2024-12-03 15:32:02.797511+00:00 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. -revision = "dd0d2a5a993b" +revision = "707fe8c2e7f5" down_revision = "e05bdc5b3c7b" branch_labels = None depends_on = None @@ -52,8 +52,8 @@ def upgrade(): ) op.create_table( "resource_tracker_license_checkouts", - sa.Column("license_checkout_id", sa.BigInteger(), nullable=False), - sa.Column("license_package_id", sa.BigInteger(), nullable=True), + sa.Column("license_checkout_id", sa.String(), nullable=False), + sa.Column("license_package_id", sa.String(), nullable=True), sa.Column("wallet_id", sa.BigInteger(), nullable=False), sa.Column("user_id", sa.BigInteger(), nullable=False), sa.Column("user_email", sa.String(), nullable=True), @@ -69,9 +69,12 @@ def upgrade(): nullable=False, ), sa.ForeignKeyConstraint( - ["service_run_id"], - ["resource_tracker_service_runs.service_run_id"], - name="fk_resource_tracker_license_checkouts_service_run_id", + ["product_name", "service_run_id"], + [ + "resource_tracker_service_runs.product_name", + "resource_tracker_service_runs.service_run_id", + ], + name="resource_tracker_license_checkouts_service_run_id_fkey", onupdate="CASCADE", ondelete="RESTRICT", ), diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_checkouts.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_checkouts.py index 3cac6555f51..a97e31494b1 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_checkouts.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_checkouts.py @@ -17,14 +17,14 @@ def _custom_id_generator(): metadata, sa.Column( "license_checkout_id", - sa.BigInteger, + sa.String, nullable=False, primary_key=True, default=_custom_id_generator, ), sa.Column( "license_package_id", - sa.BigInteger, + sa.String, nullable=True, ), sa.Column( @@ -47,12 +47,6 @@ def _custom_id_generator(): sa.Column( "service_run_id", sa.String, - sa.ForeignKey( - "resource_tracker_service_runs.service_run_id", - name="fk_resource_tracker_license_checkouts_service_run_id", - onupdate=RefActions.CASCADE, - ondelete=RefActions.RESTRICT, - ), nullable=True, ), sa.Column( @@ -73,6 +67,17 @@ def _custom_id_generator(): nullable=False, ), column_modified_datetime(timezone=True), + # --------------------------- + sa.ForeignKeyConstraint( + ["product_name", "service_run_id"], + [ + "resource_tracker_service_runs.product_name", + "resource_tracker_service_runs.service_run_id", + ], + name="resource_tracker_license_checkouts_service_run_id_fkey", + onupdate=RefActions.CASCADE, + ondelete=RefActions.RESTRICT, + ), ) # We define the partial index From bc83eb0ba7d644badc7d5b11e726c120e100f5fa Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 10:44:27 +0100 Subject: [PATCH 07/26] license goods DB layer --- .../api_schemas_webserver/license_goods.py | 22 +++ .../src/models_library/license_goods.py | 43 +++++ .../ba974648bbdf_add_license_db_tables.py | 140 ++++++++++++++ .../{license_packages.py => license_goods.py} | 8 +- .../resource_tracker_license_purchases.py | 2 +- .../simcore_postgres_database/utils_repos.py | 6 + .../licenses/__init__.py | 0 .../licenses/_exceptions_handlers.py | 48 +++++ .../licenses/_license_goods_api.py | 63 ++++++ .../licenses/_license_goods_db.py | 179 ++++++++++++++++++ .../licenses/_license_goods_handlers.py | 110 +++++++++++ .../licenses/_models.py | 54 ++++++ .../simcore_service_webserver/licenses/api.py | 10 + .../licenses/errors.py | 9 + .../licenses/plugin.py | 26 +++ .../security/_authz_access_roles.py | 1 + 16 files changed, 716 insertions(+), 5 deletions(-) create mode 100644 packages/models-library/src/models_library/api_schemas_webserver/license_goods.py create mode 100644 packages/models-library/src/models_library/license_goods.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/ba974648bbdf_add_license_db_tables.py rename packages/postgres-database/src/simcore_postgres_database/models/{license_packages.py => license_goods.py} (92%) create mode 100644 services/web/server/src/simcore_service_webserver/licenses/__init__.py create mode 100644 services/web/server/src/simcore_service_webserver/licenses/_exceptions_handlers.py create mode 100644 services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py create mode 100644 services/web/server/src/simcore_service_webserver/licenses/_license_goods_db.py create mode 100644 services/web/server/src/simcore_service_webserver/licenses/_license_goods_handlers.py create mode 100644 services/web/server/src/simcore_service_webserver/licenses/_models.py create mode 100644 services/web/server/src/simcore_service_webserver/licenses/api.py create mode 100644 services/web/server/src/simcore_service_webserver/licenses/errors.py create mode 100644 services/web/server/src/simcore_service_webserver/licenses/plugin.py diff --git a/packages/models-library/src/models_library/api_schemas_webserver/license_goods.py b/packages/models-library/src/models_library/api_schemas_webserver/license_goods.py new file mode 100644 index 00000000000..fb074fce9e2 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_webserver/license_goods.py @@ -0,0 +1,22 @@ +from datetime import datetime +from typing import NamedTuple + +from models_library.license_goods import LicenseGoodID, LicenseResourceType +from models_library.resource_tracker import PricingPlanId +from pydantic import PositiveInt + +from ._base import OutputSchema + + +class LicenseGoodGet(OutputSchema): + license_good_id: LicenseGoodID + name: str + license_resource_type: LicenseResourceType + pricing_plan_id: PricingPlanId + created_at: datetime + modified_at: datetime + + +class LicenseGoodGetPage(NamedTuple): + items: list[LicenseGoodGet] + total: PositiveInt diff --git a/packages/models-library/src/models_library/license_goods.py b/packages/models-library/src/models_library/license_goods.py new file mode 100644 index 00000000000..bbdf868f1db --- /dev/null +++ b/packages/models-library/src/models_library/license_goods.py @@ -0,0 +1,43 @@ +from datetime import datetime +from enum import auto +from typing import TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, PositiveInt + +from .products import ProductName +from .resource_tracker import PricingPlanId +from .utils.enums import StrAutoEnum + +LicenseGoodID: TypeAlias = PositiveInt + + +class LicenseResourceType(StrAutoEnum): + VIP_MODEL = auto() + + +# +# DB +# + + +class LicenseGoodDB(BaseModel): + license_good_id: LicenseGoodID + name: str + license_resource_type: LicenseResourceType + pricing_plan_id: PricingPlanId + product_name: ProductName + created: datetime = Field( + ..., + description="Timestamp on creation", + ) + modified: datetime = Field( + ..., + description="Timestamp of last modification", + ) + # ---- + model_config = ConfigDict(from_attributes=True) + + +class LicenseGoodUpdateDB(BaseModel): + name: str | None = None + pricing_plan_id: PricingPlanId diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/ba974648bbdf_add_license_db_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ba974648bbdf_add_license_db_tables.py new file mode 100644 index 00000000000..ea4cdd73e86 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ba974648bbdf_add_license_db_tables.py @@ -0,0 +1,140 @@ +"""add license db tables + +Revision ID: ba974648bbdf +Revises: e05bdc5b3c7b +Create Date: 2024-12-04 08:37:55.511823+00:00 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "ba974648bbdf" +down_revision = "e05bdc5b3c7b" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "resource_tracker_license_purchases", + sa.Column("license_purchase_id", sa.String(), nullable=False), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column("license_good_id", sa.BigInteger(), nullable=False), + sa.Column("wallet_id", sa.BigInteger(), nullable=False), + sa.Column( + "start_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "expire_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("purchased_by_user", sa.BigInteger(), nullable=False), + sa.Column( + "purchased_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("license_purchase_id"), + ) + op.create_table( + "resource_tracker_license_checkouts", + sa.Column("license_checkout_id", sa.String(), nullable=False), + sa.Column("license_package_id", sa.String(), nullable=True), + sa.Column("wallet_id", sa.BigInteger(), nullable=False), + sa.Column("user_id", sa.BigInteger(), nullable=False), + sa.Column("user_email", sa.String(), nullable=True), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column("service_run_id", sa.String(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("stopped_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("num_of_seats", sa.SmallInteger(), nullable=False), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["product_name", "service_run_id"], + [ + "resource_tracker_service_runs.product_name", + "resource_tracker_service_runs.service_run_id", + ], + name="resource_tracker_license_checkouts_service_run_id_fkey", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("license_checkout_id"), + ) + op.create_index( + op.f("ix_resource_tracker_license_checkouts_wallet_id"), + "resource_tracker_license_checkouts", + ["wallet_id"], + unique=False, + ) + op.create_table( + "license_goods", + sa.Column("license_good_id", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column( + "license_resource_type", + sa.Enum("VIP_MODEL", name="licenseresourcetype"), + nullable=False, + ), + sa.Column("pricing_plan_id", sa.BigInteger(), nullable=False), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["pricing_plan_id"], + ["resource_tracker_pricing_plans.pricing_plan_id"], + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["product_name"], + ["products.name"], + name="fk_resource_tracker_license_packages_product_name", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("license_good_id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("license_goods") + op.drop_index( + op.f("ix_resource_tracker_license_checkouts_wallet_id"), + table_name="resource_tracker_license_checkouts", + ) + op.drop_table("resource_tracker_license_checkouts") + op.drop_table("resource_tracker_license_purchases") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/license_packages.py b/packages/postgres-database/src/simcore_postgres_database/models/license_goods.py similarity index 92% rename from packages/postgres-database/src/simcore_postgres_database/models/license_packages.py rename to packages/postgres-database/src/simcore_postgres_database/models/license_goods.py index 9c587804109..9ae1b735430 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/license_packages.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/license_goods.py @@ -11,18 +11,18 @@ def _custom_id_generator(): - return f"lpa_{shortuuid.uuid()}" + return f"lgo_{shortuuid.uuid()}" class LicenseResourceType(str, enum.Enum): VIP_MODEL = "VIP_MODEL" -license_packages = sa.Table( - "license_packages", +license_goods = sa.Table( + "license_goods", metadata, sa.Column( - "license_package_id", + "license_good_id", sa.String, nullable=False, primary_key=True, diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py index 48abfe4df43..a9f699e8962 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py @@ -30,7 +30,7 @@ def _custom_id_generator(): doc="Product name", ), sa.Column( - "license_package_id", + "license_good_id", sa.BigInteger, nullable=False, ), diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_repos.py b/packages/postgres-database/src/simcore_postgres_database/utils_repos.py index f2c96313ea9..b304b3c0053 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_repos.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_repos.py @@ -11,6 +11,9 @@ async def pass_or_acquire_connection( engine: AsyncEngine, connection: AsyncConnection | None = None ) -> AsyncIterator[AsyncConnection]: + """ + When to use: For READ operations only! + """ # NOTE: When connection is passed, the engine is actually not needed # NOTE: Creator is responsible of closing connection is_connection_created = connection is None @@ -30,6 +33,9 @@ async def pass_or_acquire_connection( async def transaction_context( engine: AsyncEngine, connection: AsyncConnection | None = None ): + """ + When to use: For WRITE operations only! + """ async with pass_or_acquire_connection(engine, connection) as conn: if conn.in_transaction(): async with conn.begin_nested(): # inner transaction (savepoint) diff --git a/services/web/server/src/simcore_service_webserver/licenses/__init__.py b/services/web/server/src/simcore_service_webserver/licenses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/web/server/src/simcore_service_webserver/licenses/_exceptions_handlers.py b/services/web/server/src/simcore_service_webserver/licenses/_exceptions_handlers.py new file mode 100644 index 00000000000..1bb16355b80 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_exceptions_handlers.py @@ -0,0 +1,48 @@ +import logging + +from servicelib.aiohttp import status + +from ..exception_handling import ( + ExceptionToHttpErrorMap, + HttpErrorInfo, + exception_handling_decorator, + to_exceptions_handlers_map, +) +from ..projects.exceptions import ProjectRunningConflictError, ProjectStoppingError +from .errors import ( + WorkspaceAccessForbiddenError, + WorkspaceGroupNotFoundError, + WorkspaceNotFoundError, +) + +_logger = logging.getLogger(__name__) + + +_TO_HTTP_ERROR_MAP: ExceptionToHttpErrorMap = { + WorkspaceGroupNotFoundError: HttpErrorInfo( + status.HTTP_404_NOT_FOUND, + "Workspace {workspace_id} group {group_id} not found.", + ), + WorkspaceAccessForbiddenError: HttpErrorInfo( + status.HTTP_403_FORBIDDEN, + "Does not have access to this workspace", + ), + WorkspaceNotFoundError: HttpErrorInfo( + status.HTTP_404_NOT_FOUND, + "Workspace not found. {reason}", + ), + # Trashing + ProjectRunningConflictError: HttpErrorInfo( + status.HTTP_409_CONFLICT, + "One or more studies in this workspace are in use and cannot be trashed. Please stop all services first and try again", + ), + ProjectStoppingError: HttpErrorInfo( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Something went wrong while stopping running services in studies within this workspace before trashing. Aborting trash.", + ), +} + + +handle_plugin_requests_exceptions = exception_handling_decorator( + to_exceptions_handlers_map(_TO_HTTP_ERROR_MAP) +) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py new file mode 100644 index 00000000000..d05e2f689af --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py @@ -0,0 +1,63 @@ +# pylint: disable=unused-argument + +import logging + +import _license_goods_db +from aiohttp import web +from models_library.api_schemas_webserver.license_goods import ( + LicenseGoodGet, + LicenseGoodGetPage, +) +from models_library.license_goods import LicenseGoodID +from models_library.products import ProductName +from models_library.rest_ordering import OrderBy +from models_library.users import UserID +from pydantic import NonNegativeInt + +from ._models import LicenseGoodsBodyParams + +_logger = logging.getLogger(__name__) + + +async def get_license_good( + app: web.Application, + *, + license_good_id: LicenseGoodID, + product_name: ProductName, +) -> LicenseGoodGet: + + license_good_db = await _license_goods_db.get( + app, license_good_id=license_good_id, product_name=product_name + ) + return LicenseGoodGet.model_construct(**license_good_db.model_dump()) + + +async def list_license_goods( + app: web.Application, + *, + product_name: ProductName, + offset: NonNegativeInt, + limit: int, + order_by: OrderBy, +) -> LicenseGoodGetPage: + total_count, license_good_db_list = await _license_goods_db.list_( + app, product_name=product_name, offset=offset, limit=limit, order_by=order_by + ) + return LicenseGoodGetPage( + items=[ + LicenseGoodGet.model_construct(**license_good_db.model_dump()) + for license_good_db in license_good_db_list + ], + total=total_count, + ) + + +async def purchase_license_good( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + license_good_id: LicenseGoodID, + body_params: LicenseGoodsBodyParams, +) -> None: + raise NotImplementedError diff --git a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_db.py b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_db.py new file mode 100644 index 00000000000..6668060830e --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_db.py @@ -0,0 +1,179 @@ +""" Database API + + - Adds a layer to the postgres API with a focus on the projects comments + +""" + +import logging +from typing import cast + +from aiohttp import web +from models_library.license_goods import ( + LicenseGoodDB, + LicenseGoodID, + LicenseGoodUpdateDB, + LicenseResourceType, +) +from models_library.products import ProductName +from models_library.resource_tracker import PricingPlanId +from models_library.rest_ordering import OrderBy, OrderDirection +from pydantic import NonNegativeInt +from simcore_postgres_database.models.license_goods import license_goods +from simcore_postgres_database.utils_repos import ( + pass_or_acquire_connection, + transaction_context, +) +from sqlalchemy import asc, desc, func +from sqlalchemy.ext.asyncio import AsyncConnection +from sqlalchemy.sql import select + +from ..db.plugin import get_asyncpg_engine +from .errors import LicenseGoodNotFoundError + +_logger = logging.getLogger(__name__) + + +_SELECTION_ARGS = ( + license_goods.c.license_good_id, + license_goods.c.name, + license_goods.c.license_resource_type, + license_goods.c.pricing_plan_id, + license_goods.c.product_name, + license_goods.c.created, + license_goods.c.modified, +) + +assert set(LicenseGoodDB.model_fields) == {c.name for c in _SELECTION_ARGS} # nosec + + +async def create( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + name: str, + license_resource_type: LicenseResourceType, + pricing_plan_id: PricingPlanId, +) -> LicenseGoodDB: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + license_goods.insert() + .values( + name=name, + license_resource_type=license_resource_type, + pricing_plan_id=pricing_plan_id, + product_name=product_name, + created=func.now(), + modified=func.now(), + ) + .returning(*_SELECTION_ARGS) + ) + row = await result.first() + return LicenseGoodDB.model_validate(row) + + +async def list_( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + offset: NonNegativeInt, + limit: NonNegativeInt, + order_by: OrderBy, +) -> tuple[int, list[LicenseGoodDB]]: + base_query = ( + select(*_SELECTION_ARGS) + .select_from(license_goods) + .where(license_goods.c.product_name == product_name) + ) + + # Select total count from base_query + subquery = base_query.subquery() + count_query = select(func.count()).select_from(subquery) + + # Ordering and pagination + if order_by.direction == OrderDirection.ASC: + list_query = base_query.order_by(asc(getattr(license_goods.c, order_by.field))) + else: + list_query = base_query.order_by(desc(getattr(license_goods.c, order_by.field))) + list_query = list_query.offset(offset).limit(limit) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + total_count = await conn.scalar(count_query) + + result = await conn.stream(list_query) + items: list[LicenseGoodDB] = [ + LicenseGoodDB.model_validate(row) async for row in result + ] + + return cast(int, total_count), items + + +async def get( + app: web.Application, + connection: AsyncConnection | None = None, + *, + license_good_id: LicenseGoodID, + product_name: ProductName, +) -> LicenseGoodDB: + base_query = ( + select(*_SELECTION_ARGS) + .select_from(license_goods) + .where( + (license_goods.c.license_good_id == license_good_id) + & (license_goods.c.product_name == product_name) + ) + ) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(base_query) + row = await result.first() + if row is None: + raise LicenseGoodNotFoundError(license_good_id=license_good_id) + return LicenseGoodDB.model_validate(row) + + +async def update( + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + license_good_id: LicenseGoodID, + updates: LicenseGoodUpdateDB, +) -> LicenseGoodDB: + # NOTE: at least 'touch' if updated_values is empty + _updates = { + **updates.dict(exclude_unset=True), + "modified": func.now(), + } + + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream( + license_goods.update() + .values(**_updates) + .where( + (license_goods.c.license_good_id == license_good_id) + & (license_goods.c.product_name == product_name) + ) + .returning(*_SELECTION_ARGS) + ) + row = await result.first() + if row is None: + raise LicenseGoodNotFoundError(license_good_id=license_good_id) + return LicenseGoodDB.model_validate(row) + + +async def delete_workspace( + app: web.Application, + connection: AsyncConnection | None = None, + *, + license_good_id: LicenseGoodID, + product_name: ProductName, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.execute( + license_goods.delete().where( + (license_goods.c.license_good_id == license_good_id) + & (license_goods.c.product_name == product_name) + ) + ) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_handlers.py b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_handlers.py new file mode 100644 index 00000000000..cf44888520e --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_handlers.py @@ -0,0 +1,110 @@ +import logging + +from aiohttp import web +from models_library.api_schemas_webserver.license_goods import ( + LicenseGoodGet, + LicenseGoodGetPage, +) +from models_library.rest_ordering import OrderBy +from models_library.rest_pagination import Page +from models_library.rest_pagination_utils import paginate_data +from servicelib.aiohttp import status +from servicelib.aiohttp.requests_validation import ( + parse_request_body_as, + parse_request_path_parameters_as, + parse_request_query_parameters_as, +) +from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON +from servicelib.rest_constants import RESPONSE_MODEL_POLICY + +from .._meta import API_VTAG as VTAG +from ..login.decorators import login_required +from ..security.decorators import permission_required +from ..utils_aiohttp import envelope_json_response +from . import _license_goods_api +from ._exceptions_handlers import handle_plugin_requests_exceptions +from ._models import ( + LicenseGoodsBodyParams, + LicenseGoodsListQueryParams, + LicenseGoodsPathParams, + LicenseGoodsRequestContext, +) + +_logger = logging.getLogger(__name__) + + +routes = web.RouteTableDef() + + +@routes.get(f"/{VTAG}/license-goods", name="list_license_goods") +@login_required +@permission_required("license-goods.*") +@handle_plugin_requests_exceptions +async def list_workspaces(request: web.Request): + req_ctx = LicenseGoodsRequestContext.model_validate(request) + query_params: LicenseGoodsListQueryParams = parse_request_query_parameters_as( + LicenseGoodsListQueryParams, request + ) + + license_good_get_page: LicenseGoodGetPage = ( + await _license_goods_api.list_license_goods( + app=request.app, + product_name=req_ctx.product_name, + offset=query_params.offset, + limit=query_params.limit, + order_by=OrderBy.model_construct(**query_params.order_by.model_dump()), + ) + ) + + page = Page[LicenseGoodGet].model_validate( + paginate_data( + chunk=license_good_get_page.items, + request_url=request.url, + total=license_good_get_page.total, + limit=query_params.limit, + offset=query_params.offset, + ) + ) + return web.Response( + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), + content_type=MIMETYPE_APPLICATION_JSON, + ) + + +@routes.get(f"/{VTAG}/license-goods/{{license_good_id}}", name="get_license_good") +@login_required +@permission_required("license-goods.*") +@handle_plugin_requests_exceptions +async def get_workspace(request: web.Request): + req_ctx = LicenseGoodsRequestContext.model_validate(request) + path_params = parse_request_path_parameters_as(LicenseGoodsPathParams, request) + + license_good_get: LicenseGoodGet = await _license_goods_api.get_license_good( + app=request.app, + license_good_id=path_params.license_good_id, + product_name=req_ctx.product_name, + ) + + return envelope_json_response(license_good_get) + + +@routes.post( + f"/{VTAG}/license-goods/{{license_good_id}}:purchase", + name="purchase_license_good", +) +@login_required +@permission_required("license-goods.*") +@handle_plugin_requests_exceptions +async def purchase_license_good(request: web.Request): + req_ctx = LicenseGoodsRequestContext.model_validate(request) + path_params = parse_request_path_parameters_as(LicenseGoodsPathParams, request) + body_params = await parse_request_body_as(LicenseGoodsBodyParams, request) + + await _license_goods_api.purchase_license_good( + app=request.app, + user_id=req_ctx.user_id, + license_good_id=path_params.license_good_id, + product_name=req_ctx.product_name, + body_params=body_params, + ) + return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_models.py b/services/web/server/src/simcore_service_webserver/licenses/_models.py new file mode 100644 index 00000000000..fb5579dc34f --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/_models.py @@ -0,0 +1,54 @@ +import logging + +from models_library.basic_types import IDStr +from models_library.license_goods import LicenseGoodID +from models_library.rest_base import RequestParameters, StrictRequestParameters +from models_library.rest_ordering import ( + OrderBy, + OrderDirection, + create_ordering_query_model_class, +) +from models_library.rest_pagination import PageQueryParameters +from models_library.users import UserID +from models_library.wallets import WalletID +from pydantic import BaseModel, ConfigDict, Field +from servicelib.request_keys import RQT_USERID_KEY + +from .._constants import RQ_PRODUCT_KEY + +_logger = logging.getLogger(__name__) + + +class LicenseGoodsRequestContext(RequestParameters): + user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[literal-required] + product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[literal-required] + + +class LicenseGoodsPathParams(StrictRequestParameters): + license_good_id: LicenseGoodID + + +_LicenseGoodsListOrderQueryParams: type[ + RequestParameters +] = create_ordering_query_model_class( + ordering_fields={ + "modified_at", + "name", + }, + default=OrderBy(field=IDStr("modified_at"), direction=OrderDirection.DESC), + ordering_fields_api_to_column_map={"modified_at": "modified"}, +) + + +class LicenseGoodsListQueryParams( + PageQueryParameters, + _LicenseGoodsListOrderQueryParams, # type: ignore[misc, valid-type] +): + ... + + +class LicenseGoodsBodyParams(BaseModel): + wallet_id: WalletID + num_of_seeds: int + + model_config = ConfigDict(extra="forbid") diff --git a/services/web/server/src/simcore_service_webserver/licenses/api.py b/services/web/server/src/simcore_service_webserver/licenses/api.py new file mode 100644 index 00000000000..0194d539dcd --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/api.py @@ -0,0 +1,10 @@ +# mypy: disable-error-code=truthy-function +from ._license_goods_api import check_user_workspace_access, get_workspace + +assert get_workspace # nosec +assert check_user_workspace_access # nosec + +__all__: tuple[str, ...] = ( + "get_workspace", + "check_user_workspace_access", +) diff --git a/services/web/server/src/simcore_service_webserver/licenses/errors.py b/services/web/server/src/simcore_service_webserver/licenses/errors.py new file mode 100644 index 00000000000..3e1550fdbf1 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/errors.py @@ -0,0 +1,9 @@ +from ..errors import WebServerBaseError + + +class LicensesValueError(WebServerBaseError, ValueError): + ... + + +class LicenseGoodNotFoundError(LicensesValueError): + msg_template = "License good {license_good_id} not found" diff --git a/services/web/server/src/simcore_service_webserver/licenses/plugin.py b/services/web/server/src/simcore_service_webserver/licenses/plugin.py new file mode 100644 index 00000000000..956d1d5fe84 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/licenses/plugin.py @@ -0,0 +1,26 @@ +""" tags management subsystem + +""" +import logging + +from aiohttp import web +from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY +from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup + +from . import _license_goods_handlers + +_logger = logging.getLogger(__name__) + + +@app_module_setup( + __name__, + ModuleCategory.ADDON, + settings_name="WEBSERVER_LICENSE", + depends=["simcore_service_webserver.rest"], + logger=_logger, +) +def setup_workspaces(app: web.Application): + assert app[APP_SETTINGS_KEY].WEBSERVER_LICENSE # nosec + + # routes + app.router.add_routes(_license_goods_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py index 919486962d3..b0cba4579a7 100644 --- a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py +++ b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py @@ -59,6 +59,7 @@ class PermissionDict(TypedDict, total=False): "folder.delete", "folder.access_rights.update", "groups.*", + "license-goods.*", "product.price.read", "project.folders.*", "project.access_rights.update", From f2113ec891e2c5e357395acf056f209767d733a9 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 10:47:58 +0100 Subject: [PATCH 08/26] exeption handling --- .../licenses/_exceptions_handlers.py | 30 +++---------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_exceptions_handlers.py b/services/web/server/src/simcore_service_webserver/licenses/_exceptions_handlers.py index 1bb16355b80..d4f0e45d1d2 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_exceptions_handlers.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_exceptions_handlers.py @@ -8,38 +8,16 @@ exception_handling_decorator, to_exceptions_handlers_map, ) -from ..projects.exceptions import ProjectRunningConflictError, ProjectStoppingError -from .errors import ( - WorkspaceAccessForbiddenError, - WorkspaceGroupNotFoundError, - WorkspaceNotFoundError, -) +from .errors import LicenseGoodNotFoundError _logger = logging.getLogger(__name__) _TO_HTTP_ERROR_MAP: ExceptionToHttpErrorMap = { - WorkspaceGroupNotFoundError: HttpErrorInfo( - status.HTTP_404_NOT_FOUND, - "Workspace {workspace_id} group {group_id} not found.", - ), - WorkspaceAccessForbiddenError: HttpErrorInfo( - status.HTTP_403_FORBIDDEN, - "Does not have access to this workspace", - ), - WorkspaceNotFoundError: HttpErrorInfo( + LicenseGoodNotFoundError: HttpErrorInfo( status.HTTP_404_NOT_FOUND, - "Workspace not found. {reason}", - ), - # Trashing - ProjectRunningConflictError: HttpErrorInfo( - status.HTTP_409_CONFLICT, - "One or more studies in this workspace are in use and cannot be trashed. Please stop all services first and try again", - ), - ProjectStoppingError: HttpErrorInfo( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Something went wrong while stopping running services in studies within this workspace before trashing. Aborting trash.", - ), + "Market item {license_good_id} not found.", + ) } From e570bebd2cb09a94b3ecf64ad2902795245f95c7 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 11:04:19 +0100 Subject: [PATCH 09/26] open api specs --- api/specs/web-server/_license_goods.py | 60 ++++++ api/specs/web-server/openapi.py | 1 + .../api/v0/openapi.yaml | 185 ++++++++++++++++++ 3 files changed, 246 insertions(+) create mode 100644 api/specs/web-server/_license_goods.py diff --git a/api/specs/web-server/_license_goods.py b/api/specs/web-server/_license_goods.py new file mode 100644 index 00000000000..25f64f7eded --- /dev/null +++ b/api/specs/web-server/_license_goods.py @@ -0,0 +1,60 @@ +""" Helper script to generate OAS automatically +""" + +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + +from typing import Annotated + +from _common import as_query +from fastapi import APIRouter, Depends, status +from models_library.api_schemas_webserver.license_goods import LicenseGoodGet +from models_library.generics import Envelope +from models_library.rest_error import EnvelopedError +from simcore_service_webserver._meta import API_VTAG +from simcore_service_webserver.licenses._exceptions_handlers import _TO_HTTP_ERROR_MAP +from simcore_service_webserver.licenses._models import ( + LicenseGoodsBodyParams, + LicenseGoodsListQueryParams, + LicenseGoodsPathParams, +) + +router = APIRouter( + prefix=f"/{API_VTAG}", + tags=[ + "licenses", + ], + responses={ + i.status_code: {"model": EnvelopedError} for i in _TO_HTTP_ERROR_MAP.values() + }, +) + + +@router.get( + "/license-goods", + response_model=Envelope[list[LicenseGoodGet]], +) +async def list_workspaces( + _query: Annotated[as_query(LicenseGoodsListQueryParams), Depends()], +): + ... + + +@router.get( + "/license-goods/{license_good_id}", + response_model=Envelope[LicenseGoodGet], +) +async def get_workspace( + _path: Annotated[LicenseGoodsPathParams, Depends()], +): + ... + + +@router.post("/license-goods/{license_good_id}", status_code=status.HTTP_204_NO_CONTENT) +async def create_workspace_group( + _path: Annotated[LicenseGoodsPathParams, Depends()], + _body: LicenseGoodsBodyParams, +): + ... diff --git a/api/specs/web-server/openapi.py b/api/specs/web-server/openapi.py index 8e6b562c96d..e5b9e12d485 100644 --- a/api/specs/web-server/openapi.py +++ b/api/specs/web-server/openapi.py @@ -35,6 +35,7 @@ "_exporter", "_folders", "_long_running_tasks", + "_license_goods", "_metamodeling", "_nih_sparc", "_nih_sparc_redirections", diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 49278e0f128..897a7b63fb5 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2915,6 +2915,106 @@ paths: content: application/json: schema: {} + /v0/license-goods: + get: + tags: + - licenses + summary: List Workspaces + operationId: list_workspaces + parameters: + - name: order_by + in: query + required: false + schema: + type: string + contentMediaType: application/json + contentSchema: {} + default: '{"field":"modified","direction":"desc"}' + title: Order By + - name: limit + in: query + required: false + schema: + type: integer + default: 20 + title: Limit + - name: offset + in: query + required: false + schema: + type: integer + default: 0 + title: Offset + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_list_LicenseGoodGet__' + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + /v0/license-goods/{license_good_id}: + get: + tags: + - licenses + summary: Get Workspace + operationId: get_workspace + parameters: + - name: license_good_id + in: path + required: true + schema: + type: integer + exclusiveMinimum: true + title: License Good Id + minimum: 0 + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_LicenseGoodGet_' + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + post: + tags: + - licenses + summary: Create Workspace Group + operationId: create_workspace_group + parameters: + - name: license_good_id + in: path + required: true + schema: + type: integer + exclusiveMinimum: true + title: License Good Id + minimum: 0 + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/LicenseGoodsBodyParams' + responses: + '204': + description: Successful Response + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found /v0/projects/{project_uuid}/checkpoint/{ref_id}/iterations: get: tags: @@ -7710,6 +7810,19 @@ components: title: Error type: object title: Envelope[InvitationInfo] + Envelope_LicenseGoodGet_: + properties: + data: + anyOf: + - $ref: '#/components/schemas/LicenseGoodGet' + - type: 'null' + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[LicenseGoodGet] Envelope_Log_: properties: data: @@ -8483,6 +8596,22 @@ components: title: Error type: object title: Envelope[list[GroupUserGet]] + Envelope_list_LicenseGoodGet__: + properties: + data: + anyOf: + - items: + $ref: '#/components/schemas/LicenseGoodGet' + type: array + - type: 'null' + title: Data + error: + anyOf: + - {} + - type: 'null' + title: Error + type: object + title: Envelope[list[LicenseGoodGet]] Envelope_list_OsparcCreditsAggregatedByServiceGet__: properties: data: @@ -9878,6 +10007,62 @@ components: additionalProperties: false type: object title: InvitationInfo + LicenseGoodGet: + properties: + licenseGoodId: + type: integer + exclusiveMinimum: true + title: Licensegoodid + minimum: 0 + name: + type: string + title: Name + licenseResourceType: + $ref: '#/components/schemas/LicenseResourceType' + pricingPlanId: + type: integer + exclusiveMinimum: true + title: Pricingplanid + minimum: 0 + createdAt: + type: string + format: date-time + title: Createdat + modifiedAt: + type: string + format: date-time + title: Modifiedat + type: object + required: + - licenseGoodId + - name + - licenseResourceType + - pricingPlanId + - createdAt + - modifiedAt + title: LicenseGoodGet + LicenseGoodsBodyParams: + properties: + wallet_id: + type: integer + exclusiveMinimum: true + title: Wallet Id + minimum: 0 + num_of_seeds: + type: integer + title: Num Of Seeds + additionalProperties: false + type: object + required: + - wallet_id + - num_of_seeds + title: LicenseGoodsBodyParams + LicenseResourceType: + type: string + enum: + - VIP_MODEL + const: VIP_MODEL + title: LicenseResourceType Limits: properties: cpus: From 60b3132d0f3748da2e6d6e92fc904f056c446d2a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 11:52:21 +0100 Subject: [PATCH 10/26] adding db test --- .../src/models_library/license_goods.py | 2 +- .../licenses/_license_goods_db.py | 2 +- .../unit/with_dbs/04/licenses/conftest.py | 17 ++++ .../04/licenses/test_license_goods_db.py | 96 +++++++++++++++++++ 4 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 services/web/server/tests/unit/with_dbs/04/licenses/conftest.py create mode 100644 services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py diff --git a/packages/models-library/src/models_library/license_goods.py b/packages/models-library/src/models_library/license_goods.py index bbdf868f1db..65f1b1db9e1 100644 --- a/packages/models-library/src/models_library/license_goods.py +++ b/packages/models-library/src/models_library/license_goods.py @@ -40,4 +40,4 @@ class LicenseGoodDB(BaseModel): class LicenseGoodUpdateDB(BaseModel): name: str | None = None - pricing_plan_id: PricingPlanId + pricing_plan_id: PricingPlanId | None = None diff --git a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_db.py b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_db.py index 6668060830e..9e6828ed055 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_db.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_db.py @@ -163,7 +163,7 @@ async def update( return LicenseGoodDB.model_validate(row) -async def delete_workspace( +async def delete( app: web.Application, connection: AsyncConnection | None = None, *, diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py new file mode 100644 index 00000000000..fa008269aaf --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py @@ -0,0 +1,17 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name +from collections.abc import Iterator + +import pytest +import sqlalchemy as sa +from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.workspaces import workspaces + + +@pytest.fixture +def workspaces_clean_db(postgres_db: sa.engine.Engine) -> Iterator[None]: + with postgres_db.connect() as con: + yield + con.execute(workspaces.delete()) + con.execute(projects.delete()) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py new file mode 100644 index 00000000000..736e1e31362 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py @@ -0,0 +1,96 @@ +from collections.abc import AsyncIterator + +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements +from http import HTTPStatus + +import pytest +from aiohttp.test_utils import TestClient +from models_library.license_goods import ( + LicenseGoodDB, + LicenseGoodUpdateDB, + LicenseResourceType, +) +from models_library.rest_ordering import OrderBy +from pytest_simcore.helpers.webserver_login import UserInfoDict +from servicelib.aiohttp import status +from simcore_service_webserver.db.models import UserRole +from simcore_service_webserver.licenses import _license_goods_db +from simcore_service_webserver.licenses.errors import LicenseGoodNotFoundError +from simcore_service_webserver.projects.models import ProjectDict + + +@pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) +async def test_license_goods_db_crud( + client: TestClient, + logged_user: UserInfoDict, + user_project: ProjectDict, + osparc_product_name: str, + expected: HTTPStatus, + workspaces_clean_db: AsyncIterator[None], +): + assert client.app + + output: tuple[int, list[LicenseGoodDB]] = await _license_goods_db.list_( + client.app, + product_name=osparc_product_name, + offset=0, + limit=10, + order_by=OrderBy(field="modified"), + ) + assert output[0] == 0 + + license_good_db = await _license_goods_db.create( + client.app, + product_name=osparc_product_name, + name="Model A", + license_resource_type=LicenseResourceType.VIP_MODEL, + pricing_plan_id=1, + ) + _license_good_id = license_good_db.license_good_id + + output: tuple[int, list[LicenseGoodDB]] = await _license_goods_db.list_( + client.app, + product_name=osparc_product_name, + offset=0, + limit=10, + order_by=OrderBy(field="modified"), + ) + assert output[0] == 1 + + license_good_db = await _license_goods_db.get( + client.app, + license_good_id=_license_good_id, + product_name=osparc_product_name, + ) + assert license_good_db.name == "Model A" + + await _license_goods_db.update( + client.app, + license_good_id=_license_good_id, + product_name=osparc_product_name, + updates=LicenseGoodUpdateDB(name="Model B"), + ) + + license_good_db = await _license_goods_db.get( + client.app, + license_good_id=_license_good_id, + product_name=osparc_product_name, + ) + assert license_good_db.name == "Model B" + + license_good_db = await _license_goods_db.delete( + client.app, + license_good_id=_license_good_id, + product_name=osparc_product_name, + ) + + with pytest.raises(LicenseGoodNotFoundError): + await _license_goods_db.get( + client.app, + license_good_id=_license_good_id, + product_name=osparc_product_name, + ) From 10988aaac614283ce3aa037e7c713648f3974372 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 11:53:52 +0100 Subject: [PATCH 11/26] remove db migration: --- .../707fe8c2e7f5_add_license_db_tables.py | 140 ------------------ .../ba974648bbdf_add_license_db_tables.py | 140 ------------------ 2 files changed, 280 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/707fe8c2e7f5_add_license_db_tables.py delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/ba974648bbdf_add_license_db_tables.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/707fe8c2e7f5_add_license_db_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/707fe8c2e7f5_add_license_db_tables.py deleted file mode 100644 index d2df7d12ab9..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/707fe8c2e7f5_add_license_db_tables.py +++ /dev/null @@ -1,140 +0,0 @@ -"""add license db tables - -Revision ID: 707fe8c2e7f5 -Revises: e05bdc5b3c7b -Create Date: 2024-12-03 15:32:02.797511+00:00 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = "707fe8c2e7f5" -down_revision = "e05bdc5b3c7b" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "resource_tracker_license_purchases", - sa.Column("license_purchase_id", sa.String(), nullable=False), - sa.Column("product_name", sa.String(), nullable=False), - sa.Column("license_package_id", sa.BigInteger(), nullable=False), - sa.Column("wallet_id", sa.BigInteger(), nullable=False), - sa.Column( - "start_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "expire_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column("purchased_by_user", sa.BigInteger(), nullable=False), - sa.Column( - "purchased_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "modified", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.PrimaryKeyConstraint("license_purchase_id"), - ) - op.create_table( - "resource_tracker_license_checkouts", - sa.Column("license_checkout_id", sa.String(), nullable=False), - sa.Column("license_package_id", sa.String(), nullable=True), - sa.Column("wallet_id", sa.BigInteger(), nullable=False), - sa.Column("user_id", sa.BigInteger(), nullable=False), - sa.Column("user_email", sa.String(), nullable=True), - sa.Column("product_name", sa.String(), nullable=False), - sa.Column("service_run_id", sa.String(), nullable=True), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("stopped_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("num_of_seats", sa.SmallInteger(), nullable=False), - sa.Column( - "modified", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.ForeignKeyConstraint( - ["product_name", "service_run_id"], - [ - "resource_tracker_service_runs.product_name", - "resource_tracker_service_runs.service_run_id", - ], - name="resource_tracker_license_checkouts_service_run_id_fkey", - onupdate="CASCADE", - ondelete="RESTRICT", - ), - sa.PrimaryKeyConstraint("license_checkout_id"), - ) - op.create_index( - op.f("ix_resource_tracker_license_checkouts_wallet_id"), - "resource_tracker_license_checkouts", - ["wallet_id"], - unique=False, - ) - op.create_table( - "license_packages", - sa.Column("license_package_id", sa.String(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column( - "license_resource_type", - sa.Enum("VIP_MODEL", name="licenseresourcetype"), - nullable=False, - ), - sa.Column("pricing_plan_id", sa.BigInteger(), nullable=False), - sa.Column("product_name", sa.String(), nullable=False), - sa.Column( - "created", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "modified", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.ForeignKeyConstraint( - ["pricing_plan_id"], - ["resource_tracker_pricing_plans.pricing_plan_id"], - name="fk_resource_tracker_license_packages_pricing_plan_id", - onupdate="CASCADE", - ondelete="RESTRICT", - ), - sa.ForeignKeyConstraint( - ["product_name"], - ["products.name"], - name="fk_resource_tracker_license_packages_product_name", - onupdate="CASCADE", - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("license_package_id"), - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("license_packages") - op.drop_index( - op.f("ix_resource_tracker_license_checkouts_wallet_id"), - table_name="resource_tracker_license_checkouts", - ) - op.drop_table("resource_tracker_license_checkouts") - op.drop_table("resource_tracker_license_purchases") - # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/ba974648bbdf_add_license_db_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ba974648bbdf_add_license_db_tables.py deleted file mode 100644 index ea4cdd73e86..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/ba974648bbdf_add_license_db_tables.py +++ /dev/null @@ -1,140 +0,0 @@ -"""add license db tables - -Revision ID: ba974648bbdf -Revises: e05bdc5b3c7b -Create Date: 2024-12-04 08:37:55.511823+00:00 - -""" -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = "ba974648bbdf" -down_revision = "e05bdc5b3c7b" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "resource_tracker_license_purchases", - sa.Column("license_purchase_id", sa.String(), nullable=False), - sa.Column("product_name", sa.String(), nullable=False), - sa.Column("license_good_id", sa.BigInteger(), nullable=False), - sa.Column("wallet_id", sa.BigInteger(), nullable=False), - sa.Column( - "start_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "expire_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column("purchased_by_user", sa.BigInteger(), nullable=False), - sa.Column( - "purchased_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "modified", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.PrimaryKeyConstraint("license_purchase_id"), - ) - op.create_table( - "resource_tracker_license_checkouts", - sa.Column("license_checkout_id", sa.String(), nullable=False), - sa.Column("license_package_id", sa.String(), nullable=True), - sa.Column("wallet_id", sa.BigInteger(), nullable=False), - sa.Column("user_id", sa.BigInteger(), nullable=False), - sa.Column("user_email", sa.String(), nullable=True), - sa.Column("product_name", sa.String(), nullable=False), - sa.Column("service_run_id", sa.String(), nullable=True), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("stopped_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("num_of_seats", sa.SmallInteger(), nullable=False), - sa.Column( - "modified", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.ForeignKeyConstraint( - ["product_name", "service_run_id"], - [ - "resource_tracker_service_runs.product_name", - "resource_tracker_service_runs.service_run_id", - ], - name="resource_tracker_license_checkouts_service_run_id_fkey", - onupdate="CASCADE", - ondelete="RESTRICT", - ), - sa.PrimaryKeyConstraint("license_checkout_id"), - ) - op.create_index( - op.f("ix_resource_tracker_license_checkouts_wallet_id"), - "resource_tracker_license_checkouts", - ["wallet_id"], - unique=False, - ) - op.create_table( - "license_goods", - sa.Column("license_good_id", sa.String(), nullable=False), - sa.Column("name", sa.String(), nullable=False), - sa.Column( - "license_resource_type", - sa.Enum("VIP_MODEL", name="licenseresourcetype"), - nullable=False, - ), - sa.Column("pricing_plan_id", sa.BigInteger(), nullable=False), - sa.Column("product_name", sa.String(), nullable=False), - sa.Column( - "created", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "modified", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.ForeignKeyConstraint( - ["pricing_plan_id"], - ["resource_tracker_pricing_plans.pricing_plan_id"], - name="fk_resource_tracker_license_packages_pricing_plan_id", - onupdate="CASCADE", - ondelete="RESTRICT", - ), - sa.ForeignKeyConstraint( - ["product_name"], - ["products.name"], - name="fk_resource_tracker_license_packages_product_name", - onupdate="CASCADE", - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("license_good_id"), - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("license_goods") - op.drop_index( - op.f("ix_resource_tracker_license_checkouts_wallet_id"), - table_name="resource_tracker_license_checkouts", - ) - op.drop_table("resource_tracker_license_checkouts") - op.drop_table("resource_tracker_license_purchases") - # ### end Alembic commands ### From 9b24815ee58bec6a3a2d13f9564b9a50349a2fc4 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 11:54:27 +0100 Subject: [PATCH 12/26] add db migration: --- .../4901050f94f4_add_license_db_tables.py | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/4901050f94f4_add_license_db_tables.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/4901050f94f4_add_license_db_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/4901050f94f4_add_license_db_tables.py new file mode 100644 index 00000000000..c5dd3fa5bc5 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/4901050f94f4_add_license_db_tables.py @@ -0,0 +1,140 @@ +"""add license db tables + +Revision ID: 4901050f94f4 +Revises: e05bdc5b3c7b +Create Date: 2024-12-04 10:54:13.440309+00:00 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "4901050f94f4" +down_revision = "e05bdc5b3c7b" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "resource_tracker_license_purchases", + sa.Column("license_purchase_id", sa.String(), nullable=False), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column("license_good_id", sa.BigInteger(), nullable=False), + sa.Column("wallet_id", sa.BigInteger(), nullable=False), + sa.Column( + "start_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "expire_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("purchased_by_user", sa.BigInteger(), nullable=False), + sa.Column( + "purchased_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("license_purchase_id"), + ) + op.create_table( + "resource_tracker_license_checkouts", + sa.Column("license_checkout_id", sa.String(), nullable=False), + sa.Column("license_package_id", sa.String(), nullable=True), + sa.Column("wallet_id", sa.BigInteger(), nullable=False), + sa.Column("user_id", sa.BigInteger(), nullable=False), + sa.Column("user_email", sa.String(), nullable=True), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column("service_run_id", sa.String(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("stopped_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("num_of_seats", sa.SmallInteger(), nullable=False), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["product_name", "service_run_id"], + [ + "resource_tracker_service_runs.product_name", + "resource_tracker_service_runs.service_run_id", + ], + name="resource_tracker_license_checkouts_service_run_id_fkey", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("license_checkout_id"), + ) + op.create_index( + op.f("ix_resource_tracker_license_checkouts_wallet_id"), + "resource_tracker_license_checkouts", + ["wallet_id"], + unique=False, + ) + op.create_table( + "license_goods", + sa.Column("license_good_id", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column( + "license_resource_type", + sa.Enum("VIP_MODEL", name="licenseresourcetype"), + nullable=False, + ), + sa.Column("pricing_plan_id", sa.BigInteger(), nullable=False), + sa.Column("product_name", sa.String(), nullable=False), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["pricing_plan_id"], + ["resource_tracker_pricing_plans.pricing_plan_id"], + name="fk_resource_tracker_license_packages_pricing_plan_id", + onupdate="CASCADE", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["product_name"], + ["products.name"], + name="fk_resource_tracker_license_packages_product_name", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("license_good_id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("license_goods") + op.drop_index( + op.f("ix_resource_tracker_license_checkouts_wallet_id"), + table_name="resource_tracker_license_checkouts", + ) + op.drop_table("resource_tracker_license_checkouts") + op.drop_table("resource_tracker_license_purchases") + # ### end Alembic commands ### From 28c86ff0c1b038f796619ebae51cfc3b09f4028a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 15:00:04 +0100 Subject: [PATCH 13/26] open api specs --- .../src/models_library/license_goods.py | 4 +- .../api/v0/openapi.yaml | 14 ++----- .../04/licenses/test_license_goods_db.py | 39 ++++++++++++++++++- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/packages/models-library/src/models_library/license_goods.py b/packages/models-library/src/models_library/license_goods.py index 65f1b1db9e1..d5d710ce61d 100644 --- a/packages/models-library/src/models_library/license_goods.py +++ b/packages/models-library/src/models_library/license_goods.py @@ -2,13 +2,13 @@ from enum import auto from typing import TypeAlias -from pydantic import BaseModel, ConfigDict, Field, PositiveInt +from pydantic import BaseModel, ConfigDict, Field from .products import ProductName from .resource_tracker import PricingPlanId from .utils.enums import StrAutoEnum -LicenseGoodID: TypeAlias = PositiveInt +LicenseGoodID: TypeAlias = str class LicenseResourceType(StrAutoEnum): diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 897a7b63fb5..96e787bca1b 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2969,10 +2969,8 @@ paths: in: path required: true schema: - type: integer - exclusiveMinimum: true + type: string title: License Good Id - minimum: 0 responses: '200': description: Successful Response @@ -2996,10 +2994,8 @@ paths: in: path required: true schema: - type: integer - exclusiveMinimum: true + type: string title: License Good Id - minimum: 0 requestBody: required: true content: @@ -4544,7 +4540,7 @@ paths: '403': description: ProjectInvalidRightsError '404': - description: UserDefaultWalletNotFoundError, ProjectNotFoundError + description: ProjectNotFoundError, UserDefaultWalletNotFoundError '409': description: ProjectTooManyProjectOpenedError '422': @@ -10010,10 +10006,8 @@ components: LicenseGoodGet: properties: licenseGoodId: - type: integer - exclusiveMinimum: true + type: string title: Licensegoodid - minimum: 0 name: type: string title: Name diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py index 736e1e31362..b99dc0b9fff 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py @@ -17,12 +17,47 @@ from models_library.rest_ordering import OrderBy from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status +from simcore_postgres_database.models.resource_tracker_pricing_plans import ( + resource_tracker_pricing_plans, +) +from simcore_postgres_database.utils_repos import transaction_context from simcore_service_webserver.db.models import UserRole +from simcore_service_webserver.db.plugin import get_asyncpg_engine from simcore_service_webserver.licenses import _license_goods_db from simcore_service_webserver.licenses.errors import LicenseGoodNotFoundError from simcore_service_webserver.projects.models import ProjectDict +@pytest.fixture +async def pricing_plan_id( + client: TestClient, + osparc_product_name: str, +) -> AsyncIterator[int]: + assert client.app + + async with transaction_context(get_asyncpg_engine(client.app)) as conn: + result = await conn.execute( + resource_tracker_pricing_plans.insert() + .values( + product_name=osparc_product_name, + display_name="ISolve Thermal", + description="", + classification="TIER", + is_active=True, + pricing_plan_key="isolve-thermal", + ) + .returning(resource_tracker_pricing_plans.c.pricing_plan_id) + ) + row = result.first() + + assert row + + yield int(row[0]) + + async with transaction_context(get_asyncpg_engine(client.app)) as conn: + result = await conn.execute(resource_tracker_pricing_plans.delete()) + + @pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) async def test_license_goods_db_crud( client: TestClient, @@ -30,7 +65,7 @@ async def test_license_goods_db_crud( user_project: ProjectDict, osparc_product_name: str, expected: HTTPStatus, - workspaces_clean_db: AsyncIterator[None], + pricing_plan_id: int, ): assert client.app @@ -48,7 +83,7 @@ async def test_license_goods_db_crud( product_name=osparc_product_name, name="Model A", license_resource_type=LicenseResourceType.VIP_MODEL, - pricing_plan_id=1, + pricing_plan_id=pricing_plan_id, ) _license_good_id = license_good_db.license_good_id From 03c5f084a812c75ade8e585ecd073b7b67708247 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 15:52:59 +0100 Subject: [PATCH 14/26] adding db test --- .../simcore_service_webserver/application.py | 4 ++ .../application_settings.py | 1 + .../licenses/_license_goods_api.py | 20 +++++- .../simcore_service_webserver/licenses/api.py | 9 +-- .../licenses/plugin.py | 6 +- .../unit/with_dbs/04/licenses/conftest.py | 45 ++++++++++--- .../04/licenses/test_license_goods_db.py | 37 ----------- .../licenses/test_license_goods_handlers.py | 66 +++++++++++++++++++ 8 files changed, 127 insertions(+), 61 deletions(-) create mode 100644 services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_handlers.py diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index 79477051ddb..359f7189403 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -26,6 +26,7 @@ from .garbage_collector.plugin import setup_garbage_collector from .groups.plugin import setup_groups from .invitations.plugin import setup_invitations +from .licenses.plugin import setup_licenses from .login.plugin import setup_login from .long_running_tasks import setup_long_running_tasks from .meta_modeling.plugin import setup_meta_modeling @@ -143,6 +144,9 @@ def create_application() -> web.Application: setup_scicrunch(app) setup_tags(app) + # licenses + setup_licenses(app) + setup_announcements(app) setup_publications(app) setup_studies_dispatcher(app) diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index ed4e519141b..b15cd73e1c8 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -271,6 +271,7 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): WEBSERVER_DB_LISTENER: bool = True WEBSERVER_FOLDERS: bool = True WEBSERVER_GROUPS: bool = True + WEBSERVER_LICENSES: bool = True WEBSERVER_META_MODELING: bool = True WEBSERVER_NOTIFICATIONS: bool = Field(default=True) WEBSERVER_PRODUCTS: bool = True diff --git a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py index d05e2f689af..3dad109fbaf 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py +++ b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py @@ -2,7 +2,6 @@ import logging -import _license_goods_db from aiohttp import web from models_library.api_schemas_webserver.license_goods import ( LicenseGoodGet, @@ -14,6 +13,7 @@ from models_library.users import UserID from pydantic import NonNegativeInt +from . import _license_goods_db from ._models import LicenseGoodsBodyParams _logger = logging.getLogger(__name__) @@ -29,7 +29,14 @@ async def get_license_good( license_good_db = await _license_goods_db.get( app, license_good_id=license_good_id, product_name=product_name ) - return LicenseGoodGet.model_construct(**license_good_db.model_dump()) + return LicenseGoodGet( + license_good_id=license_good_db.license_good_id, + name=license_good_db.name, + license_resource_type=license_good_db.license_resource_type, + pricing_plan_id=license_good_db.pricing_plan_id, + created_at=license_good_db.created, + modified_at=license_good_db.modified, + ) async def list_license_goods( @@ -45,7 +52,14 @@ async def list_license_goods( ) return LicenseGoodGetPage( items=[ - LicenseGoodGet.model_construct(**license_good_db.model_dump()) + LicenseGoodGet( + license_good_id=license_good_db.license_good_id, + name=license_good_db.name, + license_resource_type=license_good_db.license_resource_type, + pricing_plan_id=license_good_db.pricing_plan_id, + created_at=license_good_db.created, + modified_at=license_good_db.modified, + ) for license_good_db in license_good_db_list ], total=total_count, diff --git a/services/web/server/src/simcore_service_webserver/licenses/api.py b/services/web/server/src/simcore_service_webserver/licenses/api.py index 0194d539dcd..07b8034ea85 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/api.py +++ b/services/web/server/src/simcore_service_webserver/licenses/api.py @@ -1,10 +1,3 @@ # mypy: disable-error-code=truthy-function -from ._license_goods_api import check_user_workspace_access, get_workspace -assert get_workspace # nosec -assert check_user_workspace_access # nosec - -__all__: tuple[str, ...] = ( - "get_workspace", - "check_user_workspace_access", -) +__all__: tuple[str, ...] = () diff --git a/services/web/server/src/simcore_service_webserver/licenses/plugin.py b/services/web/server/src/simcore_service_webserver/licenses/plugin.py index 956d1d5fe84..dbd76fdc52e 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/plugin.py +++ b/services/web/server/src/simcore_service_webserver/licenses/plugin.py @@ -15,12 +15,12 @@ @app_module_setup( __name__, ModuleCategory.ADDON, - settings_name="WEBSERVER_LICENSE", + settings_name="WEBSERVER_LICENSES", depends=["simcore_service_webserver.rest"], logger=_logger, ) -def setup_workspaces(app: web.Application): - assert app[APP_SETTINGS_KEY].WEBSERVER_LICENSE # nosec +def setup_licenses(app: web.Application): + assert app[APP_SETTINGS_KEY].WEBSERVER_LICENSES # nosec # routes app.router.add_routes(_license_goods_handlers.routes) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py index fa008269aaf..9352c782d88 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py @@ -1,17 +1,42 @@ +from collections.abc import AsyncIterator + # pylint:disable=unused-variable # pylint:disable=unused-argument # pylint:disable=redefined-outer-name -from collections.abc import Iterator - import pytest -import sqlalchemy as sa -from simcore_postgres_database.models.projects import projects -from simcore_postgres_database.models.workspaces import workspaces +from aiohttp.test_utils import TestClient +from simcore_postgres_database.models.resource_tracker_pricing_plans import ( + resource_tracker_pricing_plans, +) +from simcore_postgres_database.utils_repos import transaction_context +from simcore_service_webserver.db.plugin import get_asyncpg_engine @pytest.fixture -def workspaces_clean_db(postgres_db: sa.engine.Engine) -> Iterator[None]: - with postgres_db.connect() as con: - yield - con.execute(workspaces.delete()) - con.execute(projects.delete()) +async def pricing_plan_id( + client: TestClient, + osparc_product_name: str, +) -> AsyncIterator[int]: + assert client.app + + async with transaction_context(get_asyncpg_engine(client.app)) as conn: + result = await conn.execute( + resource_tracker_pricing_plans.insert() + .values( + product_name=osparc_product_name, + display_name="ISolve Thermal", + description="", + classification="TIER", + is_active=True, + pricing_plan_key="isolve-thermal", + ) + .returning(resource_tracker_pricing_plans.c.pricing_plan_id) + ) + row = result.first() + + assert row + + yield int(row[0]) + + async with transaction_context(get_asyncpg_engine(client.app)) as conn: + result = await conn.execute(resource_tracker_pricing_plans.delete()) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py index b99dc0b9fff..083ec23c146 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py @@ -1,5 +1,3 @@ -from collections.abc import AsyncIterator - # pylint: disable=redefined-outer-name # pylint: disable=unused-argument # pylint: disable=unused-variable @@ -17,47 +15,12 @@ from models_library.rest_ordering import OrderBy from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status -from simcore_postgres_database.models.resource_tracker_pricing_plans import ( - resource_tracker_pricing_plans, -) -from simcore_postgres_database.utils_repos import transaction_context from simcore_service_webserver.db.models import UserRole -from simcore_service_webserver.db.plugin import get_asyncpg_engine from simcore_service_webserver.licenses import _license_goods_db from simcore_service_webserver.licenses.errors import LicenseGoodNotFoundError from simcore_service_webserver.projects.models import ProjectDict -@pytest.fixture -async def pricing_plan_id( - client: TestClient, - osparc_product_name: str, -) -> AsyncIterator[int]: - assert client.app - - async with transaction_context(get_asyncpg_engine(client.app)) as conn: - result = await conn.execute( - resource_tracker_pricing_plans.insert() - .values( - product_name=osparc_product_name, - display_name="ISolve Thermal", - description="", - classification="TIER", - is_active=True, - pricing_plan_key="isolve-thermal", - ) - .returning(resource_tracker_pricing_plans.c.pricing_plan_id) - ) - row = result.first() - - assert row - - yield int(row[0]) - - async with transaction_context(get_asyncpg_engine(client.app)) as conn: - result = await conn.execute(resource_tracker_pricing_plans.delete()) - - @pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) async def test_license_goods_db_crud( client: TestClient, diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_handlers.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_handlers.py new file mode 100644 index 00000000000..b4fea027a7f --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_handlers.py @@ -0,0 +1,66 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements +from http import HTTPStatus + +import pytest +from aiohttp.test_utils import TestClient +from models_library.api_schemas_webserver.license_goods import LicenseGoodGet +from models_library.license_goods import LicenseResourceType +from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.webserver_login import UserInfoDict +from servicelib.aiohttp import status +from simcore_service_webserver.db.models import UserRole +from simcore_service_webserver.licenses import _license_goods_db +from simcore_service_webserver.projects.models import ProjectDict + + +@pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) +async def test_license_goods_db_crud( + client: TestClient, + logged_user: UserInfoDict, + user_project: ProjectDict, + osparc_product_name: str, + expected: HTTPStatus, + pricing_plan_id: int, +): + assert client.app + + # list + url = client.app.router["list_license_goods"].url_for() + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert data == [] + + license_good_db = await _license_goods_db.create( + client.app, + product_name=osparc_product_name, + name="Model A", + license_resource_type=LicenseResourceType.VIP_MODEL, + pricing_plan_id=pricing_plan_id, + ) + _license_good_id = license_good_db.license_good_id + + # list + url = client.app.router["list_license_goods"].url_for() + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + assert LicenseGoodGet(**data[0]) + + # get + url = client.app.router["get_license_good"].url_for( + license_good_id=_license_good_id + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert LicenseGoodGet(**data) + + # purchase + url = client.app.router["purchase_license_good"].url_for( + license_good_id=_license_good_id + ) + resp = await client.post(f"{url}", json={"wallet_id": 1, "num_of_seeds": 5}) + # NOTE: Not yet implemented From d0107ada1040f5bce8d00217be82e352861a5be7 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 16:08:43 +0100 Subject: [PATCH 15/26] fix naming --- api/specs/web-server/_license_goods.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/specs/web-server/_license_goods.py b/api/specs/web-server/_license_goods.py index 25f64f7eded..3327a85132c 100644 --- a/api/specs/web-server/_license_goods.py +++ b/api/specs/web-server/_license_goods.py @@ -36,7 +36,7 @@ "/license-goods", response_model=Envelope[list[LicenseGoodGet]], ) -async def list_workspaces( +async def list_license_goods( _query: Annotated[as_query(LicenseGoodsListQueryParams), Depends()], ): ... @@ -46,14 +46,14 @@ async def list_workspaces( "/license-goods/{license_good_id}", response_model=Envelope[LicenseGoodGet], ) -async def get_workspace( +async def get_license_good( _path: Annotated[LicenseGoodsPathParams, Depends()], ): ... @router.post("/license-goods/{license_good_id}", status_code=status.HTTP_204_NO_CONTENT) -async def create_workspace_group( +async def purchase_license_good( _path: Annotated[LicenseGoodsPathParams, Depends()], _body: LicenseGoodsBodyParams, ): From 44fbf75316840577d004438c5c3f2df742e6c1f2 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 16:09:12 +0100 Subject: [PATCH 16/26] open api specs --- .../api/v0/openapi.yaml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index e9c728d392d..72d11c20aa8 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2919,8 +2919,8 @@ paths: get: tags: - licenses - summary: List Workspaces - operationId: list_workspaces + summary: List License Goods + operationId: list_license_goods parameters: - name: order_by in: query @@ -2962,8 +2962,8 @@ paths: get: tags: - licenses - summary: Get Workspace - operationId: get_workspace + summary: Get License Good + operationId: get_license_good parameters: - name: license_good_id in: path @@ -2987,8 +2987,8 @@ paths: post: tags: - licenses - summary: Create Workspace Group - operationId: create_workspace_group + summary: Purchase License Good + operationId: purchase_license_good parameters: - name: license_good_id in: path @@ -4540,7 +4540,7 @@ paths: '403': description: ProjectInvalidRightsError '404': - description: ProjectNotFoundError, UserDefaultWalletNotFoundError + description: UserDefaultWalletNotFoundError, ProjectNotFoundError '409': description: ProjectTooManyProjectOpenedError '422': @@ -14213,9 +14213,7 @@ components: required: - walletId - name - - description - owner - - thumbnail - status - created - modified @@ -14264,9 +14262,7 @@ components: required: - walletId - name - - description - owner - - thumbnail - status - created - modified From deeb0f130832c4bf59baa1cc47fd03d442d5f2c0 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 16:13:29 +0100 Subject: [PATCH 17/26] fix test --- .../web/server/tests/unit/with_dbs/04/licenses/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py index 9352c782d88..b16538764d4 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py @@ -5,6 +5,7 @@ # pylint:disable=redefined-outer-name import pytest from aiohttp.test_utils import TestClient +from simcore_postgres_database.models.license_goods import license_goods from simcore_postgres_database.models.resource_tracker_pricing_plans import ( resource_tracker_pricing_plans, ) @@ -39,4 +40,5 @@ async def pricing_plan_id( yield int(row[0]) async with transaction_context(get_asyncpg_engine(client.app)) as conn: - result = await conn.execute(resource_tracker_pricing_plans.delete()) + await conn.execute(license_goods.delete()) + await conn.execute(resource_tracker_pricing_plans.delete()) From fb069460b5ecc9b2c2512b09acc52476d32ef5dc Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 4 Dec 2024 16:34:40 +0100 Subject: [PATCH 18/26] propagate downstream dependencies everywhere --- .../requirements/_base.txt | 2 + .../postgres-database/requirements/_base.in | 4 +- .../postgres-database/requirements/_base.txt | 47 ++++++++----------- .../postgres-database/requirements/_tools.txt | 4 +- packages/simcore-sdk/requirements/_base.txt | 4 +- services/director-v2/requirements/_base.txt | 4 ++ services/director/requirements/_base.txt | 2 +- .../dynamic-scheduler/requirements/_base.txt | 2 + .../dynamic-sidecar/requirements/_base.txt | 4 ++ services/efs-guardian/requirements/_base.txt | 6 ++- services/payments/requirements/_base.txt | 2 + .../requirements/_base.txt | 8 ++-- services/storage/requirements/_base.txt | 4 ++ tests/swarm-deploy/requirements/_test.txt | 8 ++-- 14 files changed, 61 insertions(+), 40 deletions(-) diff --git a/packages/notifications-library/requirements/_base.txt b/packages/notifications-library/requirements/_base.txt index 560e3e1e3b6..bdf2b72c07b 100644 --- a/packages/notifications-library/requirements/_base.txt +++ b/packages/notifications-library/requirements/_base.txt @@ -154,6 +154,8 @@ rpds-py==0.20.0 # referencing shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sqlalchemy==1.4.54 diff --git a/packages/postgres-database/requirements/_base.in b/packages/postgres-database/requirements/_base.in index 0b809a54e17..042a33cbd73 100644 --- a/packages/postgres-database/requirements/_base.in +++ b/packages/postgres-database/requirements/_base.in @@ -6,8 +6,8 @@ --requirement ../../../packages/common-library/requirements/_base.in alembic +opentelemetry-instrumentation-asyncpg pydantic +shortuuid sqlalchemy[postgresql_psycopg2binary,postgresql_asyncpg] # SEE extras in https://github.com/sqlalchemy/sqlalchemy/blob/main/setup.cfg#L43 -opentelemetry-instrumentation-asyncpg yarl -shortuuid diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt index 1f9b551837b..da22d8d822c 100644 --- a/packages/postgres-database/requirements/_base.txt +++ b/packages/postgres-database/requirements/_base.txt @@ -1,11 +1,5 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile _base.in -# alembic==1.13.3 - # via -r _base.in + # via -r requirements/_base.in annotated-types==0.7.0 # via pydantic async-timeout==4.0.3 @@ -24,8 +18,8 @@ importlib-metadata==8.4.0 # via opentelemetry-api mako==1.3.5 # via - # -c ../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c ../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt # alembic markupsafe==2.1.5 # via mako @@ -39,34 +33,36 @@ opentelemetry-api==1.27.0 opentelemetry-instrumentation==0.48b0 # via opentelemetry-instrumentation-asyncpg opentelemetry-instrumentation-asyncpg==0.48b0 - # via -r _base.in + # via -r requirements/_base.in opentelemetry-semantic-conventions==0.48b0 # via opentelemetry-instrumentation-asyncpg orjson==3.10.11 # via - # -c ../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c ../../../requirements/constraints.txt - # -r ../../../packages/common-library/requirements/_base.in + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in psycopg2-binary==2.9.9 # via sqlalchemy pydantic==2.9.2 # via - # -c ../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c ../../../requirements/constraints.txt - # -r ../../../packages/common-library/requirements/_base.in - # -r _base.in + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/_base.in # pydantic-extra-types pydantic-core==2.23.4 # via pydantic pydantic-extra-types==2.10.0 - # via -r ../../../packages/common-library/requirements/_base.in + # via -r requirements/../../../packages/common-library/requirements/_base.in +setuptools==75.6.0 + # via opentelemetry-instrumentation shortuuid==1.0.13 - # via -r _base.in -sqlalchemy[postgresql-asyncpg,postgresql-psycopg2binary,postgresql_asyncpg,postgresql_psycopg2binary]==1.4.54 + # via -r requirements/_base.in +sqlalchemy==1.4.54 # via - # -c ../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c ../../../requirements/constraints.txt - # -r _base.in + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_base.in # alembic typing-extensions==4.12.2 # via @@ -79,9 +75,6 @@ wrapt==1.16.0 # deprecated # opentelemetry-instrumentation yarl==1.12.1 - # via -r _base.in + # via -r requirements/_base.in zipp==3.20.2 # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/postgres-database/requirements/_tools.txt b/packages/postgres-database/requirements/_tools.txt index d86067d0d10..98fce79f69a 100644 --- a/packages/postgres-database/requirements/_tools.txt +++ b/packages/postgres-database/requirements/_tools.txt @@ -69,7 +69,9 @@ pyyaml==6.0.2 ruff==0.8.1 # via -r requirements/../../../requirements/devenv.txt setuptools==75.6.0 - # via pip-tools + # via + # -c requirements/_base.txt + # pip-tools tomlkit==0.13.2 # via pylint typing-extensions==4.12.2 diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index d2fa58f9494..6911194bb09 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -352,7 +352,6 @@ redis==5.0.4 # -r requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via - # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications repro-zipfile==0.3.1 @@ -372,6 +371,8 @@ setuptools==75.1.0 # via opentelemetry-instrumentation shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 @@ -450,6 +451,7 @@ wrapt==1.16.0 yarl==1.12.1 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index e7bfdb265fc..725c0e70610 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -819,6 +819,10 @@ setuptools==74.0.0 # via opentelemetry-instrumentation shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via + # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in simple-websocket==1.0.0 # via python-engineio six==1.16.0 diff --git a/services/director/requirements/_base.txt b/services/director/requirements/_base.txt index 656861c1ba1..8ce2d2419cc 100644 --- a/services/director/requirements/_base.txt +++ b/services/director/requirements/_base.txt @@ -388,7 +388,6 @@ redis==5.2.0 # -r requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via - # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications repro-zipfile==0.3.1 @@ -501,6 +500,7 @@ wrapt==1.16.0 # opentelemetry-instrumentation-redis yarl==1.17.1 # via + # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index 6cf4dc07c90..461395dda83 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -471,6 +471,8 @@ rpds-py==0.21.0 # referencing shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in simple-websocket==1.1.0 # via python-engineio six==1.16.0 diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index 4f6ebdf0893..f181f62c1cf 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -641,6 +641,10 @@ rpds-py==0.21.0 # referencing shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via + # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in simple-websocket==1.1.0 # via python-engineio six==1.16.0 diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index f0ac604d836..a6150db5ca7 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -580,8 +580,6 @@ redis==5.1.1 # -r requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/./constraints.txt - # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications repro-zipfile==0.3.1 @@ -609,6 +607,8 @@ sh==2.1.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 @@ -758,7 +758,9 @@ wrapt==1.16.0 # opentelemetry-instrumentation-redis yarl==1.15.4 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt index c2f91b9459d..c7b93773e98 100644 --- a/services/payments/requirements/_base.txt +++ b/services/payments/requirements/_base.txt @@ -492,6 +492,8 @@ rsa==4.9 # python-jose shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in simple-websocket==1.1.0 # via python-engineio six==1.16.0 diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index 0f0c9c3592e..911b9c7b482 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -620,8 +620,6 @@ redis==5.0.4 # -r requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/./constraints.txt - # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications regex==2023.12.25 @@ -655,7 +653,9 @@ sh==2.0.6 shellingham==1.5.4 # via typer shortuuid==1.0.13 - # via -r requirements/_base.in + # via + # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 @@ -819,7 +819,9 @@ wrapt==1.16.0 # opentelemetry-instrumentation-redis yarl==1.9.4 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index a3513a00a8f..e264b18a095 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -605,6 +605,8 @@ sh==2.0.6 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 @@ -761,7 +763,9 @@ wrapt==1.16.0 # opentelemetry-instrumentation-redis yarl==1.9.4 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index 881b9db0ba3..4d534a542c1 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -120,7 +120,7 @@ certifi==2024.8.30 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # requests @@ -565,8 +565,6 @@ redis==5.0.4 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via - # -c requirements/../../../packages/service-library/requirements/./constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications repro-zipfile==0.3.1 @@ -593,6 +591,8 @@ setuptools==75.1.0 # via opentelemetry-instrumentation shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 @@ -707,7 +707,9 @@ wrapt==1.16.0 # opentelemetry-instrumentation-redis yarl==1.12.1 # via + # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq From b8c918b5141bfc829b6ce51bcd8354b558ee27b6 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 5 Dec 2024 11:36:04 +0100 Subject: [PATCH 19/26] PR reviews --- .../web-server/_catalog_licensed_items.py | 65 +++++ api/specs/web-server/_license_goods.py | 60 ----- api/specs/web-server/openapi.py | 2 +- packages/aws-library/requirements/_base.txt | 2 +- .../{license_goods.py => licensed_items.py} | 12 +- .../{license_goods.py => licensed_items.py} | 12 +- .../4901050f94f4_add_license_db_tables.py | 10 +- .../{license_goods.py => licensed_items.py} | 12 +- .../resource_tracker_license_purchases.py | 2 +- .../simcore_postgres_database/utils_repos.py | 6 +- services/api-server/requirements/_base.txt | 6 + services/autoscaling/requirements/_base.txt | 4 +- services/catalog/requirements/_base.txt | 4 +- .../clusters-keeper/requirements/_base.txt | 4 +- .../datcore-adapter/requirements/_base.txt | 2 +- .../api/v0/openapi.yaml | 227 +++++++++--------- .../simcore_service_webserver/application.py | 4 - .../{ => catalog}/licenses/__init__.py | 0 .../licenses/_exceptions_handlers.py | 8 +- .../catalog/licenses/_licensed_items_api.py | 77 ++++++ .../licenses/_licensed_items_db.py} | 94 ++++---- .../licenses/_licensed_items_handlers.py | 112 +++++++++ .../{ => catalog}/licenses/_models.py | 18 +- .../{ => catalog}/licenses/api.py | 0 .../catalog/licenses/errors.py | 9 + .../{ => catalog}/licenses/plugin.py | 4 +- .../catalog/plugin.py | 3 + .../licenses/_license_goods_api.py | 77 ------ .../licenses/_license_goods_handlers.py | 110 --------- .../licenses/errors.py | 9 - .../security/_authz_access_roles.py | 2 +- .../unit/with_dbs/04/licenses/conftest.py | 4 +- .../04/licenses/test_license_goods_db.py | 52 ++-- .../licenses/test_license_goods_handlers.py | 30 +-- tests/e2e-playwright/requirements/_test.txt | 32 +-- 35 files changed, 545 insertions(+), 530 deletions(-) create mode 100644 api/specs/web-server/_catalog_licensed_items.py delete mode 100644 api/specs/web-server/_license_goods.py rename packages/models-library/src/models_library/api_schemas_webserver/{license_goods.py => licensed_items.py} (53%) rename packages/models-library/src/models_library/{license_goods.py => licensed_items.py} (75%) rename packages/postgres-database/src/simcore_postgres_database/models/{license_goods.py => licensed_items.py} (87%) rename services/web/server/src/simcore_service_webserver/{ => catalog}/licenses/__init__.py (100%) rename services/web/server/src/simcore_service_webserver/{ => catalog}/licenses/_exceptions_handlers.py (70%) create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py rename services/web/server/src/simcore_service_webserver/{licenses/_license_goods_db.py => catalog/licenses/_licensed_items_db.py} (60%) create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_handlers.py rename services/web/server/src/simcore_service_webserver/{ => catalog}/licenses/_models.py (72%) rename services/web/server/src/simcore_service_webserver/{ => catalog}/licenses/api.py (100%) create mode 100644 services/web/server/src/simcore_service_webserver/catalog/licenses/errors.py rename services/web/server/src/simcore_service_webserver/{ => catalog}/licenses/plugin.py (84%) delete mode 100644 services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py delete mode 100644 services/web/server/src/simcore_service_webserver/licenses/_license_goods_handlers.py delete mode 100644 services/web/server/src/simcore_service_webserver/licenses/errors.py diff --git a/api/specs/web-server/_catalog_licensed_items.py b/api/specs/web-server/_catalog_licensed_items.py new file mode 100644 index 00000000000..111f6da7f63 --- /dev/null +++ b/api/specs/web-server/_catalog_licensed_items.py @@ -0,0 +1,65 @@ +""" Helper script to generate OAS automatically +""" + +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + +from typing import Annotated + +from _common import as_query +from fastapi import APIRouter, Depends, status +from models_library.api_schemas_webserver.licensed_items import LicensedItemGet +from models_library.generics import Envelope +from models_library.rest_error import EnvelopedError +from simcore_service_webserver._meta import API_VTAG +from simcore_service_webserver.catalog.licenses._exceptions_handlers import ( + _TO_HTTP_ERROR_MAP, +) +from simcore_service_webserver.catalog.licenses._models import ( + LicensedItemsBodyParams, + LicensedItemsListQueryParams, + LicensedItemsPathParams, +) + +router = APIRouter( + prefix=f"/{API_VTAG}", + tags=[ + "licenses", + "catalog", + ], + responses={ + i.status_code: {"model": EnvelopedError} for i in _TO_HTTP_ERROR_MAP.values() + }, +) + + +@router.get( + "/catalog/licensed-items", + response_model=Envelope[list[LicensedItemGet]], +) +async def list_licensed_items( + _query: Annotated[as_query(LicensedItemsListQueryParams), Depends()], +): + ... + + +@router.get( + "/catalog/licensed-items/{licensed_item_id}", + response_model=Envelope[LicensedItemGet], +) +async def get_licensed_item( + _path: Annotated[LicensedItemsPathParams, Depends()], +): + ... + + +@router.post( + "/catalog/licensed-items/{licensed_item_id}", status_code=status.HTTP_204_NO_CONTENT +) +async def purchase_licensed_item( + _path: Annotated[LicensedItemsPathParams, Depends()], + _body: LicensedItemsBodyParams, +): + ... diff --git a/api/specs/web-server/_license_goods.py b/api/specs/web-server/_license_goods.py deleted file mode 100644 index 3327a85132c..00000000000 --- a/api/specs/web-server/_license_goods.py +++ /dev/null @@ -1,60 +0,0 @@ -""" Helper script to generate OAS automatically -""" - -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable -# pylint: disable=too-many-arguments - -from typing import Annotated - -from _common import as_query -from fastapi import APIRouter, Depends, status -from models_library.api_schemas_webserver.license_goods import LicenseGoodGet -from models_library.generics import Envelope -from models_library.rest_error import EnvelopedError -from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.licenses._exceptions_handlers import _TO_HTTP_ERROR_MAP -from simcore_service_webserver.licenses._models import ( - LicenseGoodsBodyParams, - LicenseGoodsListQueryParams, - LicenseGoodsPathParams, -) - -router = APIRouter( - prefix=f"/{API_VTAG}", - tags=[ - "licenses", - ], - responses={ - i.status_code: {"model": EnvelopedError} for i in _TO_HTTP_ERROR_MAP.values() - }, -) - - -@router.get( - "/license-goods", - response_model=Envelope[list[LicenseGoodGet]], -) -async def list_license_goods( - _query: Annotated[as_query(LicenseGoodsListQueryParams), Depends()], -): - ... - - -@router.get( - "/license-goods/{license_good_id}", - response_model=Envelope[LicenseGoodGet], -) -async def get_license_good( - _path: Annotated[LicenseGoodsPathParams, Depends()], -): - ... - - -@router.post("/license-goods/{license_good_id}", status_code=status.HTTP_204_NO_CONTENT) -async def purchase_license_good( - _path: Annotated[LicenseGoodsPathParams, Depends()], - _body: LicenseGoodsBodyParams, -): - ... diff --git a/api/specs/web-server/openapi.py b/api/specs/web-server/openapi.py index e5b9e12d485..5a679b75713 100644 --- a/api/specs/web-server/openapi.py +++ b/api/specs/web-server/openapi.py @@ -31,11 +31,11 @@ "_announcements", "_catalog", "_catalog_tags", # MUST BE after _catalog + "_catalog_licensed_items", "_computations", "_exporter", "_folders", "_long_running_tasks", - "_license_goods", "_metamodeling", "_nih_sparc", "_nih_sparc_redirections", diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt index a3a10ea494a..cc65edd06c1 100644 --- a/packages/aws-library/requirements/_base.txt +++ b/packages/aws-library/requirements/_base.txt @@ -307,7 +307,6 @@ redis==5.0.4 # -r requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via - # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications repro-zipfile==0.3.1 @@ -393,6 +392,7 @@ wrapt==1.16.0 # opentelemetry-instrumentation-redis yarl==1.12.1 # via + # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/packages/models-library/src/models_library/api_schemas_webserver/license_goods.py b/packages/models-library/src/models_library/api_schemas_webserver/licensed_items.py similarity index 53% rename from packages/models-library/src/models_library/api_schemas_webserver/license_goods.py rename to packages/models-library/src/models_library/api_schemas_webserver/licensed_items.py index fb074fce9e2..7d26b717505 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/license_goods.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/licensed_items.py @@ -1,22 +1,22 @@ from datetime import datetime from typing import NamedTuple -from models_library.license_goods import LicenseGoodID, LicenseResourceType +from models_library.licensed_items import LicensedItemID, LicensedResourceType from models_library.resource_tracker import PricingPlanId from pydantic import PositiveInt from ._base import OutputSchema -class LicenseGoodGet(OutputSchema): - license_good_id: LicenseGoodID +class LicensedItemGet(OutputSchema): + licensed_item_id: LicensedItemID name: str - license_resource_type: LicenseResourceType + license_resource_type: LicensedResourceType pricing_plan_id: PricingPlanId created_at: datetime modified_at: datetime -class LicenseGoodGetPage(NamedTuple): - items: list[LicenseGoodGet] +class LicensedItemGetPage(NamedTuple): + items: list[LicensedItemGet] total: PositiveInt diff --git a/packages/models-library/src/models_library/license_goods.py b/packages/models-library/src/models_library/licensed_items.py similarity index 75% rename from packages/models-library/src/models_library/license_goods.py rename to packages/models-library/src/models_library/licensed_items.py index d5d710ce61d..da4795d92e7 100644 --- a/packages/models-library/src/models_library/license_goods.py +++ b/packages/models-library/src/models_library/licensed_items.py @@ -8,10 +8,10 @@ from .resource_tracker import PricingPlanId from .utils.enums import StrAutoEnum -LicenseGoodID: TypeAlias = str +LicensedItemID: TypeAlias = str -class LicenseResourceType(StrAutoEnum): +class LicensedResourceType(StrAutoEnum): VIP_MODEL = auto() @@ -20,10 +20,10 @@ class LicenseResourceType(StrAutoEnum): # -class LicenseGoodDB(BaseModel): - license_good_id: LicenseGoodID +class LicensedItemDB(BaseModel): + licensed_item_id: LicensedItemID name: str - license_resource_type: LicenseResourceType + license_resource_type: LicensedResourceType pricing_plan_id: PricingPlanId product_name: ProductName created: datetime = Field( @@ -38,6 +38,6 @@ class LicenseGoodDB(BaseModel): model_config = ConfigDict(from_attributes=True) -class LicenseGoodUpdateDB(BaseModel): +class LicensedItemUpdateDB(BaseModel): name: str | None = None pricing_plan_id: PricingPlanId | None = None diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/4901050f94f4_add_license_db_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/4901050f94f4_add_license_db_tables.py index c5dd3fa5bc5..0e0eb412f4e 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/4901050f94f4_add_license_db_tables.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/4901050f94f4_add_license_db_tables.py @@ -21,7 +21,7 @@ def upgrade(): "resource_tracker_license_purchases", sa.Column("license_purchase_id", sa.String(), nullable=False), sa.Column("product_name", sa.String(), nullable=False), - sa.Column("license_good_id", sa.BigInteger(), nullable=False), + sa.Column("licensed_item_id", sa.BigInteger(), nullable=False), sa.Column("wallet_id", sa.BigInteger(), nullable=False), sa.Column( "start_at", @@ -87,8 +87,8 @@ def upgrade(): unique=False, ) op.create_table( - "license_goods", - sa.Column("license_good_id", sa.String(), nullable=False), + "licensed_items", + sa.Column("licensed_item_id", sa.String(), nullable=False), sa.Column("name", sa.String(), nullable=False), sa.Column( "license_resource_type", @@ -123,14 +123,14 @@ def upgrade(): onupdate="CASCADE", ondelete="CASCADE", ), - sa.PrimaryKeyConstraint("license_good_id"), + sa.PrimaryKeyConstraint("licensed_item_id"), ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("license_goods") + op.drop_table("licensed_items") op.drop_index( op.f("ix_resource_tracker_license_checkouts_wallet_id"), table_name="resource_tracker_license_checkouts", diff --git a/packages/postgres-database/src/simcore_postgres_database/models/license_goods.py b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py similarity index 87% rename from packages/postgres-database/src/simcore_postgres_database/models/license_goods.py rename to packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py index 9ae1b735430..580ab780adb 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/license_goods.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py @@ -14,15 +14,15 @@ def _custom_id_generator(): return f"lgo_{shortuuid.uuid()}" -class LicenseResourceType(str, enum.Enum): +class LicensedResourceType(str, enum.Enum): VIP_MODEL = "VIP_MODEL" -license_goods = sa.Table( - "license_goods", +licensed_items = sa.Table( + "licensed_items", metadata, sa.Column( - "license_good_id", + "licensed_item_id", sa.String, nullable=False, primary_key=True, @@ -34,8 +34,8 @@ class LicenseResourceType(str, enum.Enum): nullable=False, ), sa.Column( - "license_resource_type", - sa.Enum(LicenseResourceType), + "licensed_resource_type", + sa.Enum(LicensedResourceType), nullable=False, doc="Item type, ex. VIP_MODEL", ), diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py index a9f699e8962..948de131d0d 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py @@ -30,7 +30,7 @@ def _custom_id_generator(): doc="Product name", ), sa.Column( - "license_good_id", + "licensed_item_id", sa.BigInteger, nullable=False, ), diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_repos.py b/packages/postgres-database/src/simcore_postgres_database/utils_repos.py index b304b3c0053..e013a09b526 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_repos.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_repos.py @@ -12,7 +12,8 @@ async def pass_or_acquire_connection( engine: AsyncEngine, connection: AsyncConnection | None = None ) -> AsyncIterator[AsyncConnection]: """ - When to use: For READ operations only! + When to use: For READ operations! + It ensures that a connection is available for use within the context, either by using an existing connection passed as a parameter or by acquiring a new one from the engine. The caller must manage the lifecycle of any connection explicitly passed in, but the function handles the cleanup for connections it creates itself. This function **does not open new transactions** and therefore is recommended only for read-only database operations. """ # NOTE: When connection is passed, the engine is actually not needed # NOTE: Creator is responsible of closing connection @@ -34,7 +35,8 @@ async def transaction_context( engine: AsyncEngine, connection: AsyncConnection | None = None ): """ - When to use: For WRITE operations only! + When to use: For WRITE operations! + This function manages the database connection and ensures that a transaction context is established for write operations. It supports both outer and nested transactions, providing flexibility for scenarios where transactions may already exist in the calling context. """ async with pass_or_acquire_connection(engine, connection) as conn: if conn.in_transaction(): diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index 50bb56b4e69..bdf54cabc9e 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -711,6 +711,10 @@ setuptools==69.2.0 # opentelemetry-instrumentation shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via + # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via # jsonschema @@ -907,7 +911,9 @@ wrapt==1.16.0 yarl==1.9.4 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt index 39676c07d5b..9fcbb0283c1 100644 --- a/services/autoscaling/requirements/_base.txt +++ b/services/autoscaling/requirements/_base.txt @@ -592,8 +592,6 @@ redis==5.0.4 # -r requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/./constraints.txt - # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications repro-zipfile==0.3.1 @@ -754,6 +752,8 @@ wrapt==1.16.0 # opentelemetry-instrumentation-redis yarl==1.9.4 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt index 7f61e93f32a..9b5643c703f 100644 --- a/services/catalog/requirements/_base.txt +++ b/services/catalog/requirements/_base.txt @@ -432,7 +432,6 @@ redis==5.0.4 # aiocache referencing==0.29.3 # via - # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications repro-zipfile==0.3.1 @@ -452,6 +451,8 @@ setuptools==74.0.0 # via opentelemetry-instrumentation shellingham==1.5.4 # via typer +shortuuid==1.0.13 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 @@ -572,6 +573,7 @@ wrapt==1.16.0 yarl==1.9.4 # via # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index c642e30aa64..602b5b421bc 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -590,8 +590,6 @@ redis==5.0.4 # -r requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/./constraints.txt - # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications repro-zipfile==0.3.1 @@ -752,6 +750,8 @@ wrapt==1.16.0 # opentelemetry-instrumentation-redis yarl==1.9.4 # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/services/datcore-adapter/requirements/_base.txt b/services/datcore-adapter/requirements/_base.txt index 476901f832b..5316e532a5e 100644 --- a/services/datcore-adapter/requirements/_base.txt +++ b/services/datcore-adapter/requirements/_base.txt @@ -365,7 +365,6 @@ redis==5.0.4 # -r requirements/../../../packages/service-library/requirements/_base.in referencing==0.29.3 # via - # -c requirements/../../../packages/service-library/requirements/./constraints.txt # jsonschema # jsonschema-specifications repro-zipfile==0.3.1 @@ -464,6 +463,7 @@ wrapt==1.16.0 # opentelemetry-instrumentation-redis yarl==1.9.4 # via + # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp # aiormq diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 72d11c20aa8..bd3f7a69659 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2347,6 +2347,105 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_CatalogServiceGet_' + /v0/catalog/licensed-items: + get: + tags: + - licenses + - catalog + summary: List Licensed Items + operationId: list_licensed_items + parameters: + - name: order_by + in: query + required: false + schema: + type: string + contentMediaType: application/json + contentSchema: {} + default: '{"field":"modified","direction":"desc"}' + title: Order By + - name: limit + in: query + required: false + schema: + type: integer + default: 20 + title: Limit + - name: offset + in: query + required: false + schema: + type: integer + default: 0 + title: Offset + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_list_LicensedItemGet__' + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + /v0/catalog/licensed-items/{licensed_item_id}: + get: + tags: + - licenses + - catalog + summary: Get Licensed Item + operationId: get_licensed_item + parameters: + - name: licensed_item_id + in: path + required: true + schema: + type: string + title: Licensed Item Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Envelope_LicensedItemGet_' + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found + post: + tags: + - licenses + - catalog + summary: Purchase Licensed Item + operationId: purchase_licensed_item + parameters: + - name: licensed_item_id + in: path + required: true + schema: + type: string + title: Licensed Item Id + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/LicensedItemsBodyParams' + responses: + '204': + description: Successful Response + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/EnvelopedError' + description: Not Found /v0/computations/{project_id}: get: tags: @@ -2915,102 +3014,6 @@ paths: content: application/json: schema: {} - /v0/license-goods: - get: - tags: - - licenses - summary: List License Goods - operationId: list_license_goods - parameters: - - name: order_by - in: query - required: false - schema: - type: string - contentMediaType: application/json - contentSchema: {} - default: '{"field":"modified","direction":"desc"}' - title: Order By - - name: limit - in: query - required: false - schema: - type: integer - default: 20 - title: Limit - - name: offset - in: query - required: false - schema: - type: integer - default: 0 - title: Offset - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_list_LicenseGoodGet__' - '404': - content: - application/json: - schema: - $ref: '#/components/schemas/EnvelopedError' - description: Not Found - /v0/license-goods/{license_good_id}: - get: - tags: - - licenses - summary: Get License Good - operationId: get_license_good - parameters: - - name: license_good_id - in: path - required: true - schema: - type: string - title: License Good Id - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_LicenseGoodGet_' - '404': - content: - application/json: - schema: - $ref: '#/components/schemas/EnvelopedError' - description: Not Found - post: - tags: - - licenses - summary: Purchase License Good - operationId: purchase_license_good - parameters: - - name: license_good_id - in: path - required: true - schema: - type: string - title: License Good Id - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/LicenseGoodsBodyParams' - responses: - '204': - description: Successful Response - '404': - content: - application/json: - schema: - $ref: '#/components/schemas/EnvelopedError' - description: Not Found /v0/projects/{project_uuid}/checkpoint/{ref_id}/iterations: get: tags: @@ -7806,11 +7809,11 @@ components: title: Error type: object title: Envelope[InvitationInfo] - Envelope_LicenseGoodGet_: + Envelope_LicensedItemGet_: properties: data: anyOf: - - $ref: '#/components/schemas/LicenseGoodGet' + - $ref: '#/components/schemas/LicensedItemGet' - type: 'null' error: anyOf: @@ -7818,7 +7821,7 @@ components: - type: 'null' title: Error type: object - title: Envelope[LicenseGoodGet] + title: Envelope[LicensedItemGet] Envelope_Log_: properties: data: @@ -8592,12 +8595,12 @@ components: title: Error type: object title: Envelope[list[GroupUserGet]] - Envelope_list_LicenseGoodGet__: + Envelope_list_LicensedItemGet__: properties: data: anyOf: - items: - $ref: '#/components/schemas/LicenseGoodGet' + $ref: '#/components/schemas/LicensedItemGet' type: array - type: 'null' title: Data @@ -8607,7 +8610,7 @@ components: - type: 'null' title: Error type: object - title: Envelope[list[LicenseGoodGet]] + title: Envelope[list[LicensedItemGet]] Envelope_list_OsparcCreditsAggregatedByServiceGet__: properties: data: @@ -10003,16 +10006,16 @@ components: additionalProperties: false type: object title: InvitationInfo - LicenseGoodGet: + LicensedItemGet: properties: - licenseGoodId: + licensedItemId: type: string - title: Licensegoodid + title: Licenseditemid name: type: string title: Name licenseResourceType: - $ref: '#/components/schemas/LicenseResourceType' + $ref: '#/components/schemas/LicensedResourceType' pricingPlanId: type: integer exclusiveMinimum: true @@ -10028,14 +10031,14 @@ components: title: Modifiedat type: object required: - - licenseGoodId + - licensedItemId - name - licenseResourceType - pricingPlanId - createdAt - modifiedAt - title: LicenseGoodGet - LicenseGoodsBodyParams: + title: LicensedItemGet + LicensedItemsBodyParams: properties: wallet_id: type: integer @@ -10050,13 +10053,13 @@ components: required: - wallet_id - num_of_seeds - title: LicenseGoodsBodyParams - LicenseResourceType: + title: LicensedItemsBodyParams + LicensedResourceType: type: string enum: - VIP_MODEL const: VIP_MODEL - title: LicenseResourceType + title: LicensedResourceType Limits: properties: cpus: diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index 359f7189403..79477051ddb 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -26,7 +26,6 @@ from .garbage_collector.plugin import setup_garbage_collector from .groups.plugin import setup_groups from .invitations.plugin import setup_invitations -from .licenses.plugin import setup_licenses from .login.plugin import setup_login from .long_running_tasks import setup_long_running_tasks from .meta_modeling.plugin import setup_meta_modeling @@ -144,9 +143,6 @@ def create_application() -> web.Application: setup_scicrunch(app) setup_tags(app) - # licenses - setup_licenses(app) - setup_announcements(app) setup_publications(app) setup_studies_dispatcher(app) diff --git a/services/web/server/src/simcore_service_webserver/licenses/__init__.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/__init__.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/licenses/__init__.py rename to services/web/server/src/simcore_service_webserver/catalog/licenses/__init__.py diff --git a/services/web/server/src/simcore_service_webserver/licenses/_exceptions_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_exceptions_handlers.py similarity index 70% rename from services/web/server/src/simcore_service_webserver/licenses/_exceptions_handlers.py rename to services/web/server/src/simcore_service_webserver/catalog/licenses/_exceptions_handlers.py index d4f0e45d1d2..0abb7671b16 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_exceptions_handlers.py +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_exceptions_handlers.py @@ -2,21 +2,21 @@ from servicelib.aiohttp import status -from ..exception_handling import ( +from ...exception_handling import ( ExceptionToHttpErrorMap, HttpErrorInfo, exception_handling_decorator, to_exceptions_handlers_map, ) -from .errors import LicenseGoodNotFoundError +from .errors import LicensedItemNotFoundError _logger = logging.getLogger(__name__) _TO_HTTP_ERROR_MAP: ExceptionToHttpErrorMap = { - LicenseGoodNotFoundError: HttpErrorInfo( + LicensedItemNotFoundError: HttpErrorInfo( status.HTTP_404_NOT_FOUND, - "Market item {license_good_id} not found.", + "Market item {licensed_item_id} not found.", ) } diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py new file mode 100644 index 00000000000..b18855db3c7 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py @@ -0,0 +1,77 @@ +# pylint: disable=unused-argument + +import logging + +from aiohttp import web +from models_library.api_schemas_webserver.licensed_items import ( + LicensedItemGet, + LicensedItemGetPage, +) +from models_library.licensed_items import LicensedItemID +from models_library.products import ProductName +from models_library.rest_ordering import OrderBy +from models_library.users import UserID +from pydantic import NonNegativeInt + +from . import _licensed_items_db +from ._models import LicensedItemsBodyParams + +_logger = logging.getLogger(__name__) + + +async def get_licensed_item( + app: web.Application, + *, + licensed_item_id: LicensedItemID, + product_name: ProductName, +) -> LicensedItemGet: + + licensed_item_db = await _licensed_items_db.get( + app, licensed_item_id=licensed_item_id, product_name=product_name + ) + return LicensedItemGet( + licensed_item_id=licensed_item_db.licensed_item_id, + name=licensed_item_db.name, + license_resource_type=licensed_item_db.license_resource_type, + pricing_plan_id=licensed_item_db.pricing_plan_id, + created_at=licensed_item_db.created, + modified_at=licensed_item_db.modified, + ) + + +async def list_licensed_items( + app: web.Application, + *, + product_name: ProductName, + offset: NonNegativeInt, + limit: int, + order_by: OrderBy, +) -> LicensedItemGetPage: + total_count, licensed_item_db_list = await _licensed_items_db.list_( + app, product_name=product_name, offset=offset, limit=limit, order_by=order_by + ) + return LicensedItemGetPage( + items=[ + LicensedItemGet( + licensed_item_id=licensed_item_db.licensed_item_id, + name=licensed_item_db.name, + license_resource_type=licensed_item_db.license_resource_type, + pricing_plan_id=licensed_item_db.pricing_plan_id, + created_at=licensed_item_db.created, + modified_at=licensed_item_db.modified, + ) + for licensed_item_db in licensed_item_db_list + ], + total=total_count, + ) + + +async def purchase_licensed_item( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + licensed_item_id: LicensedItemID, + body_params: LicensedItemsBodyParams, +) -> None: + raise NotImplementedError diff --git a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_db.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_db.py similarity index 60% rename from services/web/server/src/simcore_service_webserver/licenses/_license_goods_db.py rename to services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_db.py index 9e6828ed055..56528760a4c 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_db.py +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_db.py @@ -8,17 +8,17 @@ from typing import cast from aiohttp import web -from models_library.license_goods import ( - LicenseGoodDB, - LicenseGoodID, - LicenseGoodUpdateDB, - LicenseResourceType, +from models_library.licensed_items import ( + LicensedItemDB, + LicensedItemID, + LicensedItemUpdateDB, + LicensedResourceType, ) from models_library.products import ProductName from models_library.resource_tracker import PricingPlanId from models_library.rest_ordering import OrderBy, OrderDirection from pydantic import NonNegativeInt -from simcore_postgres_database.models.license_goods import license_goods +from simcore_postgres_database.models.licensed_items import licensed_items from simcore_postgres_database.utils_repos import ( pass_or_acquire_connection, transaction_context, @@ -27,23 +27,23 @@ from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.sql import select -from ..db.plugin import get_asyncpg_engine -from .errors import LicenseGoodNotFoundError +from ...db.plugin import get_asyncpg_engine +from .errors import LicensedItemNotFoundError _logger = logging.getLogger(__name__) _SELECTION_ARGS = ( - license_goods.c.license_good_id, - license_goods.c.name, - license_goods.c.license_resource_type, - license_goods.c.pricing_plan_id, - license_goods.c.product_name, - license_goods.c.created, - license_goods.c.modified, + licensed_items.c.licensed_item_id, + licensed_items.c.name, + licensed_items.c.license_resource_type, + licensed_items.c.pricing_plan_id, + licensed_items.c.product_name, + licensed_items.c.created, + licensed_items.c.modified, ) -assert set(LicenseGoodDB.model_fields) == {c.name for c in _SELECTION_ARGS} # nosec +assert set(LicensedItemDB.model_fields) == {c.name for c in _SELECTION_ARGS} # nosec async def create( @@ -52,12 +52,12 @@ async def create( *, product_name: ProductName, name: str, - license_resource_type: LicenseResourceType, + license_resource_type: LicensedResourceType, pricing_plan_id: PricingPlanId, -) -> LicenseGoodDB: +) -> LicensedItemDB: async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.stream( - license_goods.insert() + licensed_items.insert() .values( name=name, license_resource_type=license_resource_type, @@ -69,7 +69,7 @@ async def create( .returning(*_SELECTION_ARGS) ) row = await result.first() - return LicenseGoodDB.model_validate(row) + return LicensedItemDB.model_validate(row) async def list_( @@ -80,11 +80,11 @@ async def list_( offset: NonNegativeInt, limit: NonNegativeInt, order_by: OrderBy, -) -> tuple[int, list[LicenseGoodDB]]: +) -> tuple[int, list[LicensedItemDB]]: base_query = ( select(*_SELECTION_ARGS) - .select_from(license_goods) - .where(license_goods.c.product_name == product_name) + .select_from(licensed_items) + .where(licensed_items.c.product_name == product_name) ) # Select total count from base_query @@ -93,17 +93,19 @@ async def list_( # Ordering and pagination if order_by.direction == OrderDirection.ASC: - list_query = base_query.order_by(asc(getattr(license_goods.c, order_by.field))) + list_query = base_query.order_by(asc(getattr(licensed_items.c, order_by.field))) else: - list_query = base_query.order_by(desc(getattr(license_goods.c, order_by.field))) + list_query = base_query.order_by( + desc(getattr(licensed_items.c, order_by.field)) + ) list_query = list_query.offset(offset).limit(limit) async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: total_count = await conn.scalar(count_query) result = await conn.stream(list_query) - items: list[LicenseGoodDB] = [ - LicenseGoodDB.model_validate(row) async for row in result + items: list[LicensedItemDB] = [ + LicensedItemDB.model_validate(row) async for row in result ] return cast(int, total_count), items @@ -113,15 +115,15 @@ async def get( app: web.Application, connection: AsyncConnection | None = None, *, - license_good_id: LicenseGoodID, + licensed_item_id: LicensedItemID, product_name: ProductName, -) -> LicenseGoodDB: +) -> LicensedItemDB: base_query = ( select(*_SELECTION_ARGS) - .select_from(license_goods) + .select_from(licensed_items) .where( - (license_goods.c.license_good_id == license_good_id) - & (license_goods.c.product_name == product_name) + (licensed_items.c.licensed_item_id == licensed_item_id) + & (licensed_items.c.product_name == product_name) ) ) @@ -129,8 +131,8 @@ async def get( result = await conn.stream(base_query) row = await result.first() if row is None: - raise LicenseGoodNotFoundError(license_good_id=license_good_id) - return LicenseGoodDB.model_validate(row) + raise LicensedItemNotFoundError(licensed_item_id=licensed_item_id) + return LicensedItemDB.model_validate(row) async def update( @@ -138,9 +140,9 @@ async def update( connection: AsyncConnection | None = None, *, product_name: ProductName, - license_good_id: LicenseGoodID, - updates: LicenseGoodUpdateDB, -) -> LicenseGoodDB: + licensed_item_id: LicensedItemID, + updates: LicensedItemUpdateDB, +) -> LicensedItemDB: # NOTE: at least 'touch' if updated_values is empty _updates = { **updates.dict(exclude_unset=True), @@ -149,31 +151,31 @@ async def update( async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.stream( - license_goods.update() + licensed_items.update() .values(**_updates) .where( - (license_goods.c.license_good_id == license_good_id) - & (license_goods.c.product_name == product_name) + (licensed_items.c.licensed_item_id == licensed_item_id) + & (licensed_items.c.product_name == product_name) ) .returning(*_SELECTION_ARGS) ) row = await result.first() if row is None: - raise LicenseGoodNotFoundError(license_good_id=license_good_id) - return LicenseGoodDB.model_validate(row) + raise LicensedItemNotFoundError(licensed_item_id=licensed_item_id) + return LicensedItemDB.model_validate(row) async def delete( app: web.Application, connection: AsyncConnection | None = None, *, - license_good_id: LicenseGoodID, + licensed_item_id: LicensedItemID, product_name: ProductName, ) -> None: async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.execute( - license_goods.delete().where( - (license_goods.c.license_good_id == license_good_id) - & (license_goods.c.product_name == product_name) + licensed_items.delete().where( + (licensed_items.c.licensed_item_id == licensed_item_id) + & (licensed_items.c.product_name == product_name) ) ) diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_handlers.py new file mode 100644 index 00000000000..6ed227500e5 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_handlers.py @@ -0,0 +1,112 @@ +import logging + +from aiohttp import web +from models_library.api_schemas_webserver.licensed_items import ( + LicensedItemGet, + LicensedItemGetPage, +) +from models_library.rest_ordering import OrderBy +from models_library.rest_pagination import Page +from models_library.rest_pagination_utils import paginate_data +from servicelib.aiohttp import status +from servicelib.aiohttp.requests_validation import ( + parse_request_body_as, + parse_request_path_parameters_as, + parse_request_query_parameters_as, +) +from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON +from servicelib.rest_constants import RESPONSE_MODEL_POLICY + +from ..._meta import API_VTAG as VTAG +from ...login.decorators import login_required +from ...security.decorators import permission_required +from ...utils_aiohttp import envelope_json_response +from . import _licensed_items_api +from ._exceptions_handlers import handle_plugin_requests_exceptions +from ._models import ( + LicensedItemsBodyParams, + LicensedItemsListQueryParams, + LicensedItemsPathParams, + LicensedItemsRequestContext, +) + +_logger = logging.getLogger(__name__) + + +routes = web.RouteTableDef() + + +@routes.get(f"/{VTAG}/catalog/licensed-items", name="list_licensed_items") +@login_required +@permission_required("catalog/licensed-items.*") +@handle_plugin_requests_exceptions +async def list_workspaces(request: web.Request): + req_ctx = LicensedItemsRequestContext.model_validate(request) + query_params: LicensedItemsListQueryParams = parse_request_query_parameters_as( + LicensedItemsListQueryParams, request + ) + + licensed_item_get_page: LicensedItemGetPage = ( + await _licensed_items_api.list_licensed_items( + app=request.app, + product_name=req_ctx.product_name, + offset=query_params.offset, + limit=query_params.limit, + order_by=OrderBy.model_construct(**query_params.order_by.model_dump()), + ) + ) + + page = Page[LicensedItemGet].model_validate( + paginate_data( + chunk=licensed_item_get_page.items, + request_url=request.url, + total=licensed_item_get_page.total, + limit=query_params.limit, + offset=query_params.offset, + ) + ) + return web.Response( + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), + content_type=MIMETYPE_APPLICATION_JSON, + ) + + +@routes.get( + f"/{VTAG}/catalog/licensed-items/{{licensed_item_id}}", name="get_licensed_item" +) +@login_required +@permission_required("catalog/licensed-items.*") +@handle_plugin_requests_exceptions +async def get_workspace(request: web.Request): + req_ctx = LicensedItemsRequestContext.model_validate(request) + path_params = parse_request_path_parameters_as(LicensedItemsPathParams, request) + + licensed_item_get: LicensedItemGet = await _licensed_items_api.get_licensed_item( + app=request.app, + licensed_item_id=path_params.licensed_item_id, + product_name=req_ctx.product_name, + ) + + return envelope_json_response(licensed_item_get) + + +@routes.post( + f"/{VTAG}/catalog/licensed-items/{{licensed_item_id}}:purchase", + name="purchase_licensed_item", +) +@login_required +@permission_required("catalog/licensed-items.*") +@handle_plugin_requests_exceptions +async def purchase_licensed_item(request: web.Request): + req_ctx = LicensedItemsRequestContext.model_validate(request) + path_params = parse_request_path_parameters_as(LicensedItemsPathParams, request) + body_params = await parse_request_body_as(LicensedItemsBodyParams, request) + + await _licensed_items_api.purchase_licensed_item( + app=request.app, + user_id=req_ctx.user_id, + licensed_item_id=path_params.licensed_item_id, + product_name=req_ctx.product_name, + body_params=body_params, + ) + return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_models.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_models.py similarity index 72% rename from services/web/server/src/simcore_service_webserver/licenses/_models.py rename to services/web/server/src/simcore_service_webserver/catalog/licenses/_models.py index fb5579dc34f..40d287faa92 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/_models.py +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_models.py @@ -1,7 +1,7 @@ import logging from models_library.basic_types import IDStr -from models_library.license_goods import LicenseGoodID +from models_library.licensed_items import LicensedItemID from models_library.rest_base import RequestParameters, StrictRequestParameters from models_library.rest_ordering import ( OrderBy, @@ -14,21 +14,21 @@ from pydantic import BaseModel, ConfigDict, Field from servicelib.request_keys import RQT_USERID_KEY -from .._constants import RQ_PRODUCT_KEY +from ..._constants import RQ_PRODUCT_KEY _logger = logging.getLogger(__name__) -class LicenseGoodsRequestContext(RequestParameters): +class LicensedItemsRequestContext(RequestParameters): user_id: UserID = Field(..., alias=RQT_USERID_KEY) # type: ignore[literal-required] product_name: str = Field(..., alias=RQ_PRODUCT_KEY) # type: ignore[literal-required] -class LicenseGoodsPathParams(StrictRequestParameters): - license_good_id: LicenseGoodID +class LicensedItemsPathParams(StrictRequestParameters): + licensed_item_id: LicensedItemID -_LicenseGoodsListOrderQueryParams: type[ +_LicensedItemsListOrderQueryParams: type[ RequestParameters ] = create_ordering_query_model_class( ordering_fields={ @@ -40,14 +40,14 @@ class LicenseGoodsPathParams(StrictRequestParameters): ) -class LicenseGoodsListQueryParams( +class LicensedItemsListQueryParams( PageQueryParameters, - _LicenseGoodsListOrderQueryParams, # type: ignore[misc, valid-type] + _LicensedItemsListOrderQueryParams, # type: ignore[misc, valid-type] ): ... -class LicenseGoodsBodyParams(BaseModel): +class LicensedItemsBodyParams(BaseModel): wallet_id: WalletID num_of_seeds: int diff --git a/services/web/server/src/simcore_service_webserver/licenses/api.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/api.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/licenses/api.py rename to services/web/server/src/simcore_service_webserver/catalog/licenses/api.py diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/errors.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/errors.py new file mode 100644 index 00000000000..0c8bae69b03 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/errors.py @@ -0,0 +1,9 @@ +from ...errors import WebServerBaseError + + +class LicensesValueError(WebServerBaseError, ValueError): + ... + + +class LicensedItemNotFoundError(LicensesValueError): + msg_template = "License good {licensed_item_id} not found" diff --git a/services/web/server/src/simcore_service_webserver/licenses/plugin.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/plugin.py similarity index 84% rename from services/web/server/src/simcore_service_webserver/licenses/plugin.py rename to services/web/server/src/simcore_service_webserver/catalog/licenses/plugin.py index dbd76fdc52e..ef124c69fad 100644 --- a/services/web/server/src/simcore_service_webserver/licenses/plugin.py +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/plugin.py @@ -7,7 +7,7 @@ from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _license_goods_handlers +from . import _licensed_items_handlers _logger = logging.getLogger(__name__) @@ -23,4 +23,4 @@ def setup_licenses(app: web.Application): assert app[APP_SETTINGS_KEY].WEBSERVER_LICENSES # nosec # routes - app.router.add_routes(_license_goods_handlers.routes) + app.router.add_routes(_licensed_items_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/catalog/plugin.py b/services/web/server/src/simcore_service_webserver/catalog/plugin.py index 2af8da917f0..74c36bcbcb4 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/plugin.py +++ b/services/web/server/src/simcore_service_webserver/catalog/plugin.py @@ -9,6 +9,7 @@ from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup from . import _handlers, _tags_handlers +from .licenses.plugin import setup_licenses _logger = logging.getLogger(__name__) @@ -27,6 +28,8 @@ def setup_catalog(app: web.Application): for route_def in _handlers.routes ) + setup_licenses(app) + app.add_routes(_handlers.routes) app.add_routes(_tags_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py deleted file mode 100644 index 3dad109fbaf..00000000000 --- a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_api.py +++ /dev/null @@ -1,77 +0,0 @@ -# pylint: disable=unused-argument - -import logging - -from aiohttp import web -from models_library.api_schemas_webserver.license_goods import ( - LicenseGoodGet, - LicenseGoodGetPage, -) -from models_library.license_goods import LicenseGoodID -from models_library.products import ProductName -from models_library.rest_ordering import OrderBy -from models_library.users import UserID -from pydantic import NonNegativeInt - -from . import _license_goods_db -from ._models import LicenseGoodsBodyParams - -_logger = logging.getLogger(__name__) - - -async def get_license_good( - app: web.Application, - *, - license_good_id: LicenseGoodID, - product_name: ProductName, -) -> LicenseGoodGet: - - license_good_db = await _license_goods_db.get( - app, license_good_id=license_good_id, product_name=product_name - ) - return LicenseGoodGet( - license_good_id=license_good_db.license_good_id, - name=license_good_db.name, - license_resource_type=license_good_db.license_resource_type, - pricing_plan_id=license_good_db.pricing_plan_id, - created_at=license_good_db.created, - modified_at=license_good_db.modified, - ) - - -async def list_license_goods( - app: web.Application, - *, - product_name: ProductName, - offset: NonNegativeInt, - limit: int, - order_by: OrderBy, -) -> LicenseGoodGetPage: - total_count, license_good_db_list = await _license_goods_db.list_( - app, product_name=product_name, offset=offset, limit=limit, order_by=order_by - ) - return LicenseGoodGetPage( - items=[ - LicenseGoodGet( - license_good_id=license_good_db.license_good_id, - name=license_good_db.name, - license_resource_type=license_good_db.license_resource_type, - pricing_plan_id=license_good_db.pricing_plan_id, - created_at=license_good_db.created, - modified_at=license_good_db.modified, - ) - for license_good_db in license_good_db_list - ], - total=total_count, - ) - - -async def purchase_license_good( - app: web.Application, - *, - product_name: ProductName, - user_id: UserID, - license_good_id: LicenseGoodID, - body_params: LicenseGoodsBodyParams, -) -> None: - raise NotImplementedError diff --git a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_handlers.py b/services/web/server/src/simcore_service_webserver/licenses/_license_goods_handlers.py deleted file mode 100644 index cf44888520e..00000000000 --- a/services/web/server/src/simcore_service_webserver/licenses/_license_goods_handlers.py +++ /dev/null @@ -1,110 +0,0 @@ -import logging - -from aiohttp import web -from models_library.api_schemas_webserver.license_goods import ( - LicenseGoodGet, - LicenseGoodGetPage, -) -from models_library.rest_ordering import OrderBy -from models_library.rest_pagination import Page -from models_library.rest_pagination_utils import paginate_data -from servicelib.aiohttp import status -from servicelib.aiohttp.requests_validation import ( - parse_request_body_as, - parse_request_path_parameters_as, - parse_request_query_parameters_as, -) -from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON -from servicelib.rest_constants import RESPONSE_MODEL_POLICY - -from .._meta import API_VTAG as VTAG -from ..login.decorators import login_required -from ..security.decorators import permission_required -from ..utils_aiohttp import envelope_json_response -from . import _license_goods_api -from ._exceptions_handlers import handle_plugin_requests_exceptions -from ._models import ( - LicenseGoodsBodyParams, - LicenseGoodsListQueryParams, - LicenseGoodsPathParams, - LicenseGoodsRequestContext, -) - -_logger = logging.getLogger(__name__) - - -routes = web.RouteTableDef() - - -@routes.get(f"/{VTAG}/license-goods", name="list_license_goods") -@login_required -@permission_required("license-goods.*") -@handle_plugin_requests_exceptions -async def list_workspaces(request: web.Request): - req_ctx = LicenseGoodsRequestContext.model_validate(request) - query_params: LicenseGoodsListQueryParams = parse_request_query_parameters_as( - LicenseGoodsListQueryParams, request - ) - - license_good_get_page: LicenseGoodGetPage = ( - await _license_goods_api.list_license_goods( - app=request.app, - product_name=req_ctx.product_name, - offset=query_params.offset, - limit=query_params.limit, - order_by=OrderBy.model_construct(**query_params.order_by.model_dump()), - ) - ) - - page = Page[LicenseGoodGet].model_validate( - paginate_data( - chunk=license_good_get_page.items, - request_url=request.url, - total=license_good_get_page.total, - limit=query_params.limit, - offset=query_params.offset, - ) - ) - return web.Response( - text=page.model_dump_json(**RESPONSE_MODEL_POLICY), - content_type=MIMETYPE_APPLICATION_JSON, - ) - - -@routes.get(f"/{VTAG}/license-goods/{{license_good_id}}", name="get_license_good") -@login_required -@permission_required("license-goods.*") -@handle_plugin_requests_exceptions -async def get_workspace(request: web.Request): - req_ctx = LicenseGoodsRequestContext.model_validate(request) - path_params = parse_request_path_parameters_as(LicenseGoodsPathParams, request) - - license_good_get: LicenseGoodGet = await _license_goods_api.get_license_good( - app=request.app, - license_good_id=path_params.license_good_id, - product_name=req_ctx.product_name, - ) - - return envelope_json_response(license_good_get) - - -@routes.post( - f"/{VTAG}/license-goods/{{license_good_id}}:purchase", - name="purchase_license_good", -) -@login_required -@permission_required("license-goods.*") -@handle_plugin_requests_exceptions -async def purchase_license_good(request: web.Request): - req_ctx = LicenseGoodsRequestContext.model_validate(request) - path_params = parse_request_path_parameters_as(LicenseGoodsPathParams, request) - body_params = await parse_request_body_as(LicenseGoodsBodyParams, request) - - await _license_goods_api.purchase_license_good( - app=request.app, - user_id=req_ctx.user_id, - license_good_id=path_params.license_good_id, - product_name=req_ctx.product_name, - body_params=body_params, - ) - return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/licenses/errors.py b/services/web/server/src/simcore_service_webserver/licenses/errors.py deleted file mode 100644 index 3e1550fdbf1..00000000000 --- a/services/web/server/src/simcore_service_webserver/licenses/errors.py +++ /dev/null @@ -1,9 +0,0 @@ -from ..errors import WebServerBaseError - - -class LicensesValueError(WebServerBaseError, ValueError): - ... - - -class LicenseGoodNotFoundError(LicensesValueError): - msg_template = "License good {license_good_id} not found" diff --git a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py index b0cba4579a7..da342e1b996 100644 --- a/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py +++ b/services/web/server/src/simcore_service_webserver/security/_authz_access_roles.py @@ -59,7 +59,7 @@ class PermissionDict(TypedDict, total=False): "folder.delete", "folder.access_rights.update", "groups.*", - "license-goods.*", + "catalog/licensed-items.*", "product.price.read", "project.folders.*", "project.access_rights.update", diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py index b16538764d4..5971ed9f168 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/conftest.py @@ -5,7 +5,7 @@ # pylint:disable=redefined-outer-name import pytest from aiohttp.test_utils import TestClient -from simcore_postgres_database.models.license_goods import license_goods +from simcore_postgres_database.models.licensed_items import licensed_items from simcore_postgres_database.models.resource_tracker_pricing_plans import ( resource_tracker_pricing_plans, ) @@ -40,5 +40,5 @@ async def pricing_plan_id( yield int(row[0]) async with transaction_context(get_asyncpg_engine(client.app)) as conn: - await conn.execute(license_goods.delete()) + await conn.execute(licensed_items.delete()) await conn.execute(resource_tracker_pricing_plans.delete()) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py index 083ec23c146..4d63832ae6c 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py @@ -7,22 +7,22 @@ import pytest from aiohttp.test_utils import TestClient -from models_library.license_goods import ( - LicenseGoodDB, - LicenseGoodUpdateDB, - LicenseResourceType, +from models_library.licensed_items import ( + LicensedItemDB, + LicensedItemUpdateDB, + LicensedResourceType, ) from models_library.rest_ordering import OrderBy from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status +from simcore_service_webserver.catalog.licenses import _licensed_items_db +from simcore_service_webserver.catalog.licenses.errors import LicensedItemNotFoundError from simcore_service_webserver.db.models import UserRole -from simcore_service_webserver.licenses import _license_goods_db -from simcore_service_webserver.licenses.errors import LicenseGoodNotFoundError from simcore_service_webserver.projects.models import ProjectDict @pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) -async def test_license_goods_db_crud( +async def test_licensed_items_db_crud( client: TestClient, logged_user: UserInfoDict, user_project: ProjectDict, @@ -32,7 +32,7 @@ async def test_license_goods_db_crud( ): assert client.app - output: tuple[int, list[LicenseGoodDB]] = await _license_goods_db.list_( + output: tuple[int, list[LicensedItemDB]] = await _licensed_items_db.list_( client.app, product_name=osparc_product_name, offset=0, @@ -41,16 +41,16 @@ async def test_license_goods_db_crud( ) assert output[0] == 0 - license_good_db = await _license_goods_db.create( + licensed_item_db = await _licensed_items_db.create( client.app, product_name=osparc_product_name, name="Model A", - license_resource_type=LicenseResourceType.VIP_MODEL, + license_resource_type=LicensedResourceType.VIP_MODEL, pricing_plan_id=pricing_plan_id, ) - _license_good_id = license_good_db.license_good_id + _licensed_item_id = licensed_item_db.licensed_item_id - output: tuple[int, list[LicenseGoodDB]] = await _license_goods_db.list_( + output: tuple[int, list[LicensedItemDB]] = await _licensed_items_db.list_( client.app, product_name=osparc_product_name, offset=0, @@ -59,36 +59,36 @@ async def test_license_goods_db_crud( ) assert output[0] == 1 - license_good_db = await _license_goods_db.get( + licensed_item_db = await _licensed_items_db.get( client.app, - license_good_id=_license_good_id, + licensed_item_id=_licensed_item_id, product_name=osparc_product_name, ) - assert license_good_db.name == "Model A" + assert licensed_item_db.name == "Model A" - await _license_goods_db.update( + await _licensed_items_db.update( client.app, - license_good_id=_license_good_id, + licensed_item_id=_licensed_item_id, product_name=osparc_product_name, - updates=LicenseGoodUpdateDB(name="Model B"), + updates=LicensedItemUpdateDB(name="Model B"), ) - license_good_db = await _license_goods_db.get( + licensed_item_db = await _licensed_items_db.get( client.app, - license_good_id=_license_good_id, + licensed_item_id=_licensed_item_id, product_name=osparc_product_name, ) - assert license_good_db.name == "Model B" + assert licensed_item_db.name == "Model B" - license_good_db = await _license_goods_db.delete( + licensed_item_db = await _licensed_items_db.delete( client.app, - license_good_id=_license_good_id, + licensed_item_id=_licensed_item_id, product_name=osparc_product_name, ) - with pytest.raises(LicenseGoodNotFoundError): - await _license_goods_db.get( + with pytest.raises(LicensedItemNotFoundError): + await _licensed_items_db.get( client.app, - license_good_id=_license_good_id, + licensed_item_id=_licensed_item_id, product_name=osparc_product_name, ) diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_handlers.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_handlers.py index b4fea027a7f..cbcc6254fa1 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_handlers.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_handlers.py @@ -7,18 +7,18 @@ import pytest from aiohttp.test_utils import TestClient -from models_library.api_schemas_webserver.license_goods import LicenseGoodGet -from models_library.license_goods import LicenseResourceType +from models_library.api_schemas_webserver.licensed_items import LicensedItemGet +from models_library.licensed_items import LicensedResourceType from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status +from simcore_service_webserver.catalog.licenses import _licensed_items_db from simcore_service_webserver.db.models import UserRole -from simcore_service_webserver.licenses import _license_goods_db from simcore_service_webserver.projects.models import ProjectDict @pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) -async def test_license_goods_db_crud( +async def test_licensed_items_db_crud( client: TestClient, logged_user: UserInfoDict, user_project: ProjectDict, @@ -29,38 +29,38 @@ async def test_license_goods_db_crud( assert client.app # list - url = client.app.router["list_license_goods"].url_for() + url = client.app.router["list_licensed_items"].url_for() resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert data == [] - license_good_db = await _license_goods_db.create( + licensed_item_db = await _licensed_items_db.create( client.app, product_name=osparc_product_name, name="Model A", - license_resource_type=LicenseResourceType.VIP_MODEL, + license_resource_type=LicensedResourceType.VIP_MODEL, pricing_plan_id=pricing_plan_id, ) - _license_good_id = license_good_db.license_good_id + _licensed_item_id = licensed_item_db.licensed_item_id # list - url = client.app.router["list_license_goods"].url_for() + url = client.app.router["list_licensed_items"].url_for() resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 - assert LicenseGoodGet(**data[0]) + assert LicensedItemGet(**data[0]) # get - url = client.app.router["get_license_good"].url_for( - license_good_id=_license_good_id + url = client.app.router["get_licensed_item"].url_for( + licensed_item_id=_licensed_item_id ) resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) - assert LicenseGoodGet(**data) + assert LicensedItemGet(**data) # purchase - url = client.app.router["purchase_license_good"].url_for( - license_good_id=_license_good_id + url = client.app.router["purchase_licensed_item"].url_for( + licensed_item_id=_licensed_item_id ) resp = await client.post(f"{url}", json={"wallet_id": 1, "num_of_seeds": 5}) # NOTE: Not yet implemented diff --git a/tests/e2e-playwright/requirements/_test.txt b/tests/e2e-playwright/requirements/_test.txt index 011cb6fbd7c..a54fad46969 100644 --- a/tests/e2e-playwright/requirements/_test.txt +++ b/tests/e2e-playwright/requirements/_test.txt @@ -1,15 +1,9 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile _test.in -# annotated-types==0.7.0 # via pydantic anyio==4.6.2.post1 # via httpx arrow==1.3.0 - # via -r _test.in + # via -r requirements/_test.in certifi==2024.8.30 # via # httpcore @@ -20,11 +14,11 @@ charset-normalizer==3.3.2 dnspython==2.6.1 # via email-validator docker==7.1.0 - # via -r _test.in + # via -r requirements/_test.in email-validator==2.2.0 # via pydantic faker==29.0.0 - # via -r _test.in + # via -r requirements/_test.in greenlet==3.0.3 # via playwright h11==0.14.0 @@ -32,7 +26,7 @@ h11==0.14.0 httpcore==1.0.7 # via httpx httpx==0.27.2 - # via -r _test.in + # via -r requirements/_test.in idna==3.10 # via # anyio @@ -53,8 +47,8 @@ playwright==1.47.0 # via pytest-playwright pluggy==1.5.0 # via pytest -pydantic[email]==2.9.2 - # via -r _test.in +pydantic==2.9.2 + # via -r requirements/_test.in pydantic-core==2.23.4 # via pydantic pyee==12.0.0 @@ -70,17 +64,17 @@ pytest==8.3.3 pytest-base-url==2.1.0 # via pytest-playwright pytest-html==4.1.1 - # via -r _test.in + # via -r requirements/_test.in pytest-instafail==0.5.0 - # via -r _test.in + # via -r requirements/_test.in pytest-metadata==3.1.1 # via pytest-html pytest-playwright==0.5.2 - # via -r _test.in + # via -r requirements/_test.in pytest-runner==6.0.1 - # via -r _test.in + # via -r requirements/_test.in pytest-sugar==1.0.0 - # via -r _test.in + # via -r requirements/_test.in python-dateutil==2.9.0.post0 # via # arrow @@ -88,7 +82,7 @@ python-dateutil==2.9.0.post0 python-slugify==8.0.4 # via pytest-playwright pyyaml==6.0.2 - # via -r _test.in + # via -r requirements/_test.in requests==2.32.3 # via # docker @@ -100,7 +94,7 @@ sniffio==1.3.1 # anyio # httpx tenacity==9.0.0 - # via -r _test.in + # via -r requirements/_test.in termcolor==2.4.0 # via pytest-sugar text-unidecode==1.3 From c629b0a00d019b7e63be9578bd22b2a71a0718a9 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 5 Dec 2024 11:47:38 +0100 Subject: [PATCH 20/26] PR reviews --- .../web-server/_catalog_licensed_items.py | 3 +- ... => 24a877297edc_add_license_db_tables.py} | 36 +++++++++---------- ...ource_tracker_licensed_items_purchases.py} | 6 ++-- ... resource_tracker_licensed_items_usage.py} | 18 +++------- .../api/v0/openapi.yaml | 1 + 5 files changed, 28 insertions(+), 36 deletions(-) rename packages/postgres-database/src/simcore_postgres_database/migration/versions/{4901050f94f4_add_license_db_tables.py => 24a877297edc_add_license_db_tables.py} (79%) rename packages/postgres-database/src/simcore_postgres_database/models/{resource_tracker_license_purchases.py => resource_tracker_licensed_items_purchases.py} (89%) rename packages/postgres-database/src/simcore_postgres_database/models/{resource_tracker_license_checkouts.py => resource_tracker_licensed_items_usage.py} (78%) diff --git a/api/specs/web-server/_catalog_licensed_items.py b/api/specs/web-server/_catalog_licensed_items.py index 111f6da7f63..29b39853c95 100644 --- a/api/specs/web-server/_catalog_licensed_items.py +++ b/api/specs/web-server/_catalog_licensed_items.py @@ -56,7 +56,8 @@ async def get_licensed_item( @router.post( - "/catalog/licensed-items/{licensed_item_id}", status_code=status.HTTP_204_NO_CONTENT + "/catalog/licensed-items/{licensed_item_id}:purchase", + status_code=status.HTTP_204_NO_CONTENT, ) async def purchase_licensed_item( _path: Annotated[LicensedItemsPathParams, Depends()], diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/4901050f94f4_add_license_db_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/24a877297edc_add_license_db_tables.py similarity index 79% rename from packages/postgres-database/src/simcore_postgres_database/migration/versions/4901050f94f4_add_license_db_tables.py rename to packages/postgres-database/src/simcore_postgres_database/migration/versions/24a877297edc_add_license_db_tables.py index 0e0eb412f4e..c264a831d78 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/4901050f94f4_add_license_db_tables.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/24a877297edc_add_license_db_tables.py @@ -1,15 +1,15 @@ """add license db tables -Revision ID: 4901050f94f4 +Revision ID: 24a877297edc Revises: e05bdc5b3c7b -Create Date: 2024-12-04 10:54:13.440309+00:00 +Create Date: 2024-12-05 10:47:31.251248+00:00 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. -revision = "4901050f94f4" +revision = "24a877297edc" down_revision = "e05bdc5b3c7b" branch_labels = None depends_on = None @@ -18,8 +18,8 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table( - "resource_tracker_license_purchases", - sa.Column("license_purchase_id", sa.String(), nullable=False), + "resource_tracker_licensed_items_purchases", + sa.Column("licensed_item_purchase_id", sa.String(), nullable=False), sa.Column("product_name", sa.String(), nullable=False), sa.Column("licensed_item_id", sa.BigInteger(), nullable=False), sa.Column("wallet_id", sa.BigInteger(), nullable=False), @@ -48,12 +48,12 @@ def upgrade(): server_default=sa.text("now()"), nullable=False, ), - sa.PrimaryKeyConstraint("license_purchase_id"), + sa.PrimaryKeyConstraint("licensed_item_purchase_id"), ) op.create_table( - "resource_tracker_license_checkouts", - sa.Column("license_checkout_id", sa.String(), nullable=False), - sa.Column("license_package_id", sa.String(), nullable=True), + "resource_tracker_licensed_items_usage", + sa.Column("licensed_item_usage_id", sa.String(), nullable=False), + sa.Column("licensed_item_id", sa.String(), nullable=True), sa.Column("wallet_id", sa.BigInteger(), nullable=False), sa.Column("user_id", sa.BigInteger(), nullable=False), sa.Column("user_email", sa.String(), nullable=True), @@ -78,11 +78,11 @@ def upgrade(): onupdate="CASCADE", ondelete="RESTRICT", ), - sa.PrimaryKeyConstraint("license_checkout_id"), + sa.PrimaryKeyConstraint("licensed_item_usage_id"), ) op.create_index( - op.f("ix_resource_tracker_license_checkouts_wallet_id"), - "resource_tracker_license_checkouts", + op.f("ix_resource_tracker_licensed_items_usage_wallet_id"), + "resource_tracker_licensed_items_usage", ["wallet_id"], unique=False, ) @@ -91,8 +91,8 @@ def upgrade(): sa.Column("licensed_item_id", sa.String(), nullable=False), sa.Column("name", sa.String(), nullable=False), sa.Column( - "license_resource_type", - sa.Enum("VIP_MODEL", name="licenseresourcetype"), + "licensed_resource_type", + sa.Enum("VIP_MODEL", name="licensedresourcetype"), nullable=False, ), sa.Column("pricing_plan_id", sa.BigInteger(), nullable=False), @@ -132,9 +132,9 @@ def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.drop_table("licensed_items") op.drop_index( - op.f("ix_resource_tracker_license_checkouts_wallet_id"), - table_name="resource_tracker_license_checkouts", + op.f("ix_resource_tracker_licensed_items_usage_wallet_id"), + table_name="resource_tracker_licensed_items_usage", ) - op.drop_table("resource_tracker_license_checkouts") - op.drop_table("resource_tracker_license_purchases") + op.drop_table("resource_tracker_licensed_items_usage") + op.drop_table("resource_tracker_licensed_items_purchases") # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py similarity index 89% rename from packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py rename to packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py index 948de131d0d..ee7ebd9e5c6 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_purchases.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py @@ -13,11 +13,11 @@ def _custom_id_generator(): return f"rlp_{shortuuid.uuid()}" -resource_tracker_license_purchases = sa.Table( - "resource_tracker_license_purchases", +resource_tracker_licensed_items_purchases = sa.Table( + "resource_tracker_licensed_items_purchases", metadata, sa.Column( - "license_purchase_id", + "licensed_item_purchase_id", sa.String, nullable=False, primary_key=True, diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_checkouts.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py similarity index 78% rename from packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_checkouts.py rename to packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py index a97e31494b1..6d6f84852bf 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_license_checkouts.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py @@ -12,18 +12,18 @@ def _custom_id_generator(): return f"rlc_{shortuuid.uuid()}" -resource_tracker_license_checkouts = sa.Table( - "resource_tracker_license_checkouts", +resource_tracker_licensed_items_usage = sa.Table( + "resource_tracker_licensed_items_usage", metadata, sa.Column( - "license_checkout_id", + "licensed_item_usage_id", sa.String, nullable=False, primary_key=True, default=_custom_id_generator, ), sa.Column( - "license_package_id", + "licensed_item_id", sa.String, nullable=True, ), @@ -79,13 +79,3 @@ def _custom_id_generator(): ondelete=RefActions.RESTRICT, ), ) - -# We define the partial index -# sa.Index( -# "ix_resource_tracker_credit_transactions_status_running", -# resource_tracker_service_runs.c.service_run_status, -# postgresql_where=( -# resource_tracker_service_runs.c.service_run_status -# == ResourceTrackerServiceRunStatus.RUNNING -# ), -# ) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index bd3f7a69659..dc6c68bed39 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2418,6 +2418,7 @@ paths: schema: $ref: '#/components/schemas/EnvelopedError' description: Not Found + /v0/catalog/licensed-items/{licensed_item_id}:purchase: post: tags: - licenses From 50204c3f8ffade790ec7c3047a05cc822f493d0b Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 5 Dec 2024 11:50:13 +0100 Subject: [PATCH 21/26] fix test --- .../models_library/api_schemas_webserver/licensed_items.py | 2 +- .../models-library/src/models_library/licensed_items.py | 2 +- .../catalog/licenses/_licensed_items_api.py | 4 ++-- .../catalog/licenses/_licensed_items_db.py | 6 +++--- .../{test_license_goods_db.py => test_licensed_items_db.py} | 2 +- ...se_goods_handlers.py => test_licensed_items_handlers.py} | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) rename services/web/server/tests/unit/with_dbs/04/licenses/{test_license_goods_db.py => test_licensed_items_db.py} (97%) rename services/web/server/tests/unit/with_dbs/04/licenses/{test_license_goods_handlers.py => test_licensed_items_handlers.py} (97%) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/licensed_items.py b/packages/models-library/src/models_library/api_schemas_webserver/licensed_items.py index 7d26b717505..5c170588856 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/licensed_items.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/licensed_items.py @@ -11,7 +11,7 @@ class LicensedItemGet(OutputSchema): licensed_item_id: LicensedItemID name: str - license_resource_type: LicensedResourceType + licensed_resource_type: LicensedResourceType pricing_plan_id: PricingPlanId created_at: datetime modified_at: datetime diff --git a/packages/models-library/src/models_library/licensed_items.py b/packages/models-library/src/models_library/licensed_items.py index da4795d92e7..b65cd86805a 100644 --- a/packages/models-library/src/models_library/licensed_items.py +++ b/packages/models-library/src/models_library/licensed_items.py @@ -23,7 +23,7 @@ class LicensedResourceType(StrAutoEnum): class LicensedItemDB(BaseModel): licensed_item_id: LicensedItemID name: str - license_resource_type: LicensedResourceType + licensed_resource_type: LicensedResourceType pricing_plan_id: PricingPlanId product_name: ProductName created: datetime = Field( diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py index b18855db3c7..bb024b0423b 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_api.py @@ -32,7 +32,7 @@ async def get_licensed_item( return LicensedItemGet( licensed_item_id=licensed_item_db.licensed_item_id, name=licensed_item_db.name, - license_resource_type=licensed_item_db.license_resource_type, + licensed_resource_type=licensed_item_db.licensed_resource_type, pricing_plan_id=licensed_item_db.pricing_plan_id, created_at=licensed_item_db.created, modified_at=licensed_item_db.modified, @@ -55,7 +55,7 @@ async def list_licensed_items( LicensedItemGet( licensed_item_id=licensed_item_db.licensed_item_id, name=licensed_item_db.name, - license_resource_type=licensed_item_db.license_resource_type, + licensed_resource_type=licensed_item_db.licensed_resource_type, pricing_plan_id=licensed_item_db.pricing_plan_id, created_at=licensed_item_db.created, modified_at=licensed_item_db.modified, diff --git a/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_db.py b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_db.py index 56528760a4c..fc14221ff91 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_db.py +++ b/services/web/server/src/simcore_service_webserver/catalog/licenses/_licensed_items_db.py @@ -36,7 +36,7 @@ _SELECTION_ARGS = ( licensed_items.c.licensed_item_id, licensed_items.c.name, - licensed_items.c.license_resource_type, + licensed_items.c.licensed_resource_type, licensed_items.c.pricing_plan_id, licensed_items.c.product_name, licensed_items.c.created, @@ -52,7 +52,7 @@ async def create( *, product_name: ProductName, name: str, - license_resource_type: LicensedResourceType, + licensed_resource_type: LicensedResourceType, pricing_plan_id: PricingPlanId, ) -> LicensedItemDB: async with transaction_context(get_asyncpg_engine(app), connection) as conn: @@ -60,7 +60,7 @@ async def create( licensed_items.insert() .values( name=name, - license_resource_type=license_resource_type, + licensed_resource_type=licensed_resource_type, pricing_plan_id=pricing_plan_id, product_name=product_name, created=func.now(), diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_db.py similarity index 97% rename from services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py rename to services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_db.py index 4d63832ae6c..5455c280cd7 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_db.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_db.py @@ -45,7 +45,7 @@ async def test_licensed_items_db_crud( client.app, product_name=osparc_product_name, name="Model A", - license_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_type=LicensedResourceType.VIP_MODEL, pricing_plan_id=pricing_plan_id, ) _licensed_item_id = licensed_item_db.licensed_item_id diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_handlers.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_handlers.py similarity index 97% rename from services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_handlers.py rename to services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_handlers.py index cbcc6254fa1..535032c31a2 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_license_goods_handlers.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_handlers.py @@ -38,7 +38,7 @@ async def test_licensed_items_db_crud( client.app, product_name=osparc_product_name, name="Model A", - license_resource_type=LicensedResourceType.VIP_MODEL, + licensed_resource_type=LicensedResourceType.VIP_MODEL, pricing_plan_id=pricing_plan_id, ) _licensed_item_id = licensed_item_db.licensed_item_id From 68ecfe5d2ee13e3accf62ac5451cd5e4eaa201c7 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 5 Dec 2024 11:57:25 +0100 Subject: [PATCH 22/26] fix test --- ... => e5555076ef50_add_license_db_tables.py} | 28 +++++++++++++++---- .../models/licensed_items.py | 10 ++----- ...source_tracker_licensed_items_purchases.py | 11 ++------ .../resource_tracker_licensed_items_usage.py | 11 ++------ 4 files changed, 31 insertions(+), 29 deletions(-) rename packages/postgres-database/src/simcore_postgres_database/migration/versions/{24a877297edc_add_license_db_tables.py => e5555076ef50_add_license_db_tables.py} (86%) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/24a877297edc_add_license_db_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e5555076ef50_add_license_db_tables.py similarity index 86% rename from packages/postgres-database/src/simcore_postgres_database/migration/versions/24a877297edc_add_license_db_tables.py rename to packages/postgres-database/src/simcore_postgres_database/migration/versions/e5555076ef50_add_license_db_tables.py index c264a831d78..ba4c6007be9 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/24a877297edc_add_license_db_tables.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e5555076ef50_add_license_db_tables.py @@ -1,15 +1,16 @@ """add license db tables -Revision ID: 24a877297edc +Revision ID: e5555076ef50 Revises: e05bdc5b3c7b -Create Date: 2024-12-05 10:47:31.251248+00:00 +Create Date: 2024-12-05 10:57:16.867891+00:00 """ import sqlalchemy as sa from alembic import op +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = "24a877297edc" +revision = "e5555076ef50" down_revision = "e05bdc5b3c7b" branch_labels = None depends_on = None @@ -19,7 +20,12 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.create_table( "resource_tracker_licensed_items_purchases", - sa.Column("licensed_item_purchase_id", sa.String(), nullable=False), + sa.Column( + "licensed_item_purchase_id", + postgresql.UUID(as_uuid=True), + server_default="gen_random_uuid()", + nullable=False, + ), sa.Column("product_name", sa.String(), nullable=False), sa.Column("licensed_item_id", sa.BigInteger(), nullable=False), sa.Column("wallet_id", sa.BigInteger(), nullable=False), @@ -52,7 +58,12 @@ def upgrade(): ) op.create_table( "resource_tracker_licensed_items_usage", - sa.Column("licensed_item_usage_id", sa.String(), nullable=False), + sa.Column( + "licensed_item_usage_id", + postgresql.UUID(as_uuid=True), + server_default="gen_random_uuid()", + nullable=False, + ), sa.Column("licensed_item_id", sa.String(), nullable=True), sa.Column("wallet_id", sa.BigInteger(), nullable=False), sa.Column("user_id", sa.BigInteger(), nullable=False), @@ -88,7 +99,12 @@ def upgrade(): ) op.create_table( "licensed_items", - sa.Column("licensed_item_id", sa.String(), nullable=False), + sa.Column( + "licensed_item_id", + postgresql.UUID(as_uuid=True), + server_default="gen_random_uuid()", + nullable=False, + ), sa.Column("name", sa.String(), nullable=False), sa.Column( "licensed_resource_type", diff --git a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py index 580ab780adb..0baed5ad37e 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py @@ -3,17 +3,13 @@ import enum -import shortuuid import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID from ._common import RefActions, column_created_datetime, column_modified_datetime from .base import metadata -def _custom_id_generator(): - return f"lgo_{shortuuid.uuid()}" - - class LicensedResourceType(str, enum.Enum): VIP_MODEL = "VIP_MODEL" @@ -23,10 +19,10 @@ class LicensedResourceType(str, enum.Enum): metadata, sa.Column( "licensed_item_id", - sa.String, + UUID(as_uuid=True), nullable=False, primary_key=True, - default=_custom_id_generator, + server_default="gen_random_uuid()", ), sa.Column( "name", diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py index ee7ebd9e5c6..44f574e7a28 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py @@ -2,26 +2,21 @@ """ -import shortuuid import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID from ._common import column_modified_datetime from .base import metadata - -def _custom_id_generator(): - return f"rlp_{shortuuid.uuid()}" - - resource_tracker_licensed_items_purchases = sa.Table( "resource_tracker_licensed_items_purchases", metadata, sa.Column( "licensed_item_purchase_id", - sa.String, + UUID(as_uuid=True), nullable=False, primary_key=True, - default=_custom_id_generator, + server_default="gen_random_uuid()", ), sa.Column( "product_name", diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py index 6d6f84852bf..9b8f5dc618a 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py @@ -1,26 +1,21 @@ """ resource_tracker_service_runs table """ -import shortuuid import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID from ._common import RefActions, column_modified_datetime from .base import metadata - -def _custom_id_generator(): - return f"rlc_{shortuuid.uuid()}" - - resource_tracker_licensed_items_usage = sa.Table( "resource_tracker_licensed_items_usage", metadata, sa.Column( "licensed_item_usage_id", - sa.String, + UUID(as_uuid=True), nullable=False, primary_key=True, - default=_custom_id_generator, + server_default="gen_random_uuid()", ), sa.Column( "licensed_item_id", From 9896a916c0cad94d68cd59c0ced05cebf95fd16a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 5 Dec 2024 13:14:46 +0100 Subject: [PATCH 23/26] PR reviews --- .../src/models_library/licensed_items.py | 3 ++- .../e5555076ef50_add_license_db_tables.py | 21 ++++++++++++++++--- .../api/v0/openapi.yaml | 7 +++++-- .../licenses/test_licensed_items_handlers.py | 4 ++-- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/packages/models-library/src/models_library/licensed_items.py b/packages/models-library/src/models_library/licensed_items.py index b65cd86805a..021cf214ce5 100644 --- a/packages/models-library/src/models_library/licensed_items.py +++ b/packages/models-library/src/models_library/licensed_items.py @@ -1,6 +1,7 @@ from datetime import datetime from enum import auto from typing import TypeAlias +from uuid import UUID from pydantic import BaseModel, ConfigDict, Field @@ -8,7 +9,7 @@ from .resource_tracker import PricingPlanId from .utils.enums import StrAutoEnum -LicensedItemID: TypeAlias = str +LicensedItemID: TypeAlias = UUID class LicensedResourceType(StrAutoEnum): diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e5555076ef50_add_license_db_tables.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e5555076ef50_add_license_db_tables.py index ba4c6007be9..abdfafefdf7 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e5555076ef50_add_license_db_tables.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e5555076ef50_add_license_db_tables.py @@ -17,13 +17,28 @@ def upgrade(): + # CREATE EXTENSION pgcrypto; + op.execute( + """ + DO + $$ + BEGIN + IF EXISTS(SELECT * FROM pg_available_extensions WHERE name = 'pgcrypto') THEN + -- Create the extension + CREATE EXTENSION if not exists pgcrypto; + END IF; + END + $$; + """ + ) + # ### commands auto generated by Alembic - please adjust! ### op.create_table( "resource_tracker_licensed_items_purchases", sa.Column( "licensed_item_purchase_id", postgresql.UUID(as_uuid=True), - server_default="gen_random_uuid()", + server_default=sa.text("gen_random_uuid()"), nullable=False, ), sa.Column("product_name", sa.String(), nullable=False), @@ -61,7 +76,7 @@ def upgrade(): sa.Column( "licensed_item_usage_id", postgresql.UUID(as_uuid=True), - server_default="gen_random_uuid()", + server_default=sa.text("gen_random_uuid()"), nullable=False, ), sa.Column("licensed_item_id", sa.String(), nullable=True), @@ -102,7 +117,7 @@ def upgrade(): sa.Column( "licensed_item_id", postgresql.UUID(as_uuid=True), - server_default="gen_random_uuid()", + server_default=sa.text("gen_random_uuid()"), nullable=False, ), sa.Column("name", sa.String(), nullable=False), diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index dc6c68bed39..829aa2be9c2 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2404,6 +2404,7 @@ paths: required: true schema: type: string + format: uuid title: Licensed Item Id responses: '200': @@ -2431,6 +2432,7 @@ paths: required: true schema: type: string + format: uuid title: Licensed Item Id requestBody: required: true @@ -10011,11 +10013,12 @@ components: properties: licensedItemId: type: string + format: uuid title: Licenseditemid name: type: string title: Name - licenseResourceType: + licensedResourceType: $ref: '#/components/schemas/LicensedResourceType' pricingPlanId: type: integer @@ -10034,7 +10037,7 @@ components: required: - licensedItemId - name - - licenseResourceType + - licensedResourceType - pricingPlanId - createdAt - modifiedAt diff --git a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_handlers.py b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_handlers.py index 535032c31a2..eb63d9bb75a 100644 --- a/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_handlers.py +++ b/services/web/server/tests/unit/with_dbs/04/licenses/test_licensed_items_handlers.py @@ -52,7 +52,7 @@ async def test_licensed_items_db_crud( # get url = client.app.router["get_licensed_item"].url_for( - licensed_item_id=_licensed_item_id + licensed_item_id=f"{_licensed_item_id}" ) resp = await client.get(f"{url}") data, _ = await assert_status(resp, status.HTTP_200_OK) @@ -60,7 +60,7 @@ async def test_licensed_items_db_crud( # purchase url = client.app.router["purchase_licensed_item"].url_for( - licensed_item_id=_licensed_item_id + licensed_item_id=f"{_licensed_item_id}" ) resp = await client.post(f"{url}", json={"wallet_id": 1, "num_of_seeds": 5}) # NOTE: Not yet implemented From f919e4cfe76d73b014836b1c04fe146a27e70c76 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 5 Dec 2024 13:37:35 +0100 Subject: [PATCH 24/26] remove shortuuid --- packages/notifications-library/requirements/_base.txt | 2 -- packages/postgres-database/requirements/_base.in | 1 - packages/postgres-database/requirements/_base.txt | 2 -- packages/simcore-sdk/requirements/_base.txt | 2 -- services/api-server/requirements/_base.txt | 4 ---- services/catalog/requirements/_base.txt | 2 -- services/director-v2/requirements/_base.txt | 4 ---- services/dynamic-scheduler/requirements/_base.txt | 2 -- services/dynamic-sidecar/requirements/_base.txt | 4 ---- services/efs-guardian/requirements/_base.txt | 2 -- services/payments/requirements/_base.txt | 2 -- services/resource-usage-tracker/requirements/_base.txt | 4 +--- services/storage/requirements/_base.txt | 2 -- services/web/server/requirements/_base.txt | 4 ---- tests/swarm-deploy/requirements/_test.txt | 2 -- 15 files changed, 1 insertion(+), 38 deletions(-) diff --git a/packages/notifications-library/requirements/_base.txt b/packages/notifications-library/requirements/_base.txt index bdf2b72c07b..560e3e1e3b6 100644 --- a/packages/notifications-library/requirements/_base.txt +++ b/packages/notifications-library/requirements/_base.txt @@ -154,8 +154,6 @@ rpds-py==0.20.0 # referencing shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sqlalchemy==1.4.54 diff --git a/packages/postgres-database/requirements/_base.in b/packages/postgres-database/requirements/_base.in index 042a33cbd73..c5aa128b710 100644 --- a/packages/postgres-database/requirements/_base.in +++ b/packages/postgres-database/requirements/_base.in @@ -8,6 +8,5 @@ alembic opentelemetry-instrumentation-asyncpg pydantic -shortuuid sqlalchemy[postgresql_psycopg2binary,postgresql_asyncpg] # SEE extras in https://github.com/sqlalchemy/sqlalchemy/blob/main/setup.cfg#L43 yarl diff --git a/packages/postgres-database/requirements/_base.txt b/packages/postgres-database/requirements/_base.txt index da22d8d822c..a5d6b092f2a 100644 --- a/packages/postgres-database/requirements/_base.txt +++ b/packages/postgres-database/requirements/_base.txt @@ -56,8 +56,6 @@ pydantic-extra-types==2.10.0 # via -r requirements/../../../packages/common-library/requirements/_base.in setuptools==75.6.0 # via opentelemetry-instrumentation -shortuuid==1.0.13 - # via -r requirements/_base.in sqlalchemy==1.4.54 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 6911194bb09..91d7099c7d3 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -371,8 +371,6 @@ setuptools==75.1.0 # via opentelemetry-instrumentation shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index bdf54cabc9e..eee7085c551 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -711,10 +711,6 @@ setuptools==69.2.0 # opentelemetry-instrumentation shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via - # -r requirements/../../../packages/postgres-database/requirements/_base.in - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via # jsonschema diff --git a/services/catalog/requirements/_base.txt b/services/catalog/requirements/_base.txt index 9b5643c703f..5067e459082 100644 --- a/services/catalog/requirements/_base.txt +++ b/services/catalog/requirements/_base.txt @@ -451,8 +451,6 @@ setuptools==74.0.0 # via opentelemetry-instrumentation shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index 725c0e70610..e7bfdb265fc 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -819,10 +819,6 @@ setuptools==74.0.0 # via opentelemetry-instrumentation shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via - # -r requirements/../../../packages/postgres-database/requirements/_base.in - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in simple-websocket==1.0.0 # via python-engineio six==1.16.0 diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index 461395dda83..6cf4dc07c90 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -471,8 +471,6 @@ rpds-py==0.21.0 # referencing shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in simple-websocket==1.1.0 # via python-engineio six==1.16.0 diff --git a/services/dynamic-sidecar/requirements/_base.txt b/services/dynamic-sidecar/requirements/_base.txt index f181f62c1cf..4f6ebdf0893 100644 --- a/services/dynamic-sidecar/requirements/_base.txt +++ b/services/dynamic-sidecar/requirements/_base.txt @@ -641,10 +641,6 @@ rpds-py==0.21.0 # referencing shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via - # -r requirements/../../../packages/postgres-database/requirements/_base.in - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in simple-websocket==1.1.0 # via python-engineio six==1.16.0 diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index a6150db5ca7..8840b5eab83 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -607,8 +607,6 @@ sh==2.1.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 diff --git a/services/payments/requirements/_base.txt b/services/payments/requirements/_base.txt index c7b93773e98..c2f91b9459d 100644 --- a/services/payments/requirements/_base.txt +++ b/services/payments/requirements/_base.txt @@ -492,8 +492,6 @@ rsa==4.9 # python-jose shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in simple-websocket==1.1.0 # via python-engineio six==1.16.0 diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index 911b9c7b482..d59daedf943 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -653,9 +653,7 @@ sh==2.0.6 shellingham==1.5.4 # via typer shortuuid==1.0.13 - # via - # -r requirements/../../../packages/postgres-database/requirements/_base.in - # -r requirements/_base.in + # via -r requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index e264b18a095..1c95f6b95e8 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -605,8 +605,6 @@ sh==2.0.6 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 diff --git a/services/web/server/requirements/_base.txt b/services/web/server/requirements/_base.txt index 5d607aa94fe..a11d65a49c1 100644 --- a/services/web/server/requirements/_base.txt +++ b/services/web/server/requirements/_base.txt @@ -702,10 +702,6 @@ setuptools==69.1.1 # opentelemetry-instrumentation shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via - # -r requirements/../../../../packages/postgres-database/requirements/_base.in - # -r requirements/../../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via # jsonschema diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index 4d534a542c1..946cc1f8853 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -591,8 +591,6 @@ setuptools==75.1.0 # via opentelemetry-instrumentation shellingham==1.5.4 # via typer -shortuuid==1.0.13 - # via -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.in six==1.16.0 # via python-dateutil sniffio==1.3.1 From 519c2ed1345ed17fd82803b842697dc5ddeead52 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 5 Dec 2024 13:58:25 +0100 Subject: [PATCH 25/26] fix --- tests/e2e-playwright/requirements/_test.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e-playwright/requirements/_test.txt b/tests/e2e-playwright/requirements/_test.txt index a54fad46969..2c499672c85 100644 --- a/tests/e2e-playwright/requirements/_test.txt +++ b/tests/e2e-playwright/requirements/_test.txt @@ -47,9 +47,9 @@ playwright==1.47.0 # via pytest-playwright pluggy==1.5.0 # via pytest -pydantic==2.9.2 +pydantic==2.10.3 # via -r requirements/_test.in -pydantic-core==2.23.4 +pydantic-core==2.27.1 # via pydantic pyee==12.0.0 # via playwright From 038ea39d7ef394aba119ecfdb85fcc94cb62f9b0 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Thu, 5 Dec 2024 14:18:03 +0100 Subject: [PATCH 26/26] fix --- .../src/simcore_postgres_database/models/licensed_items.py | 2 +- .../models/resource_tracker_licensed_items_purchases.py | 2 +- .../models/resource_tracker_licensed_items_usage.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py index 0baed5ad37e..63301eb9c1d 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/licensed_items.py @@ -22,7 +22,7 @@ class LicensedResourceType(str, enum.Enum): UUID(as_uuid=True), nullable=False, primary_key=True, - server_default="gen_random_uuid()", + server_default=sa.text("gen_random_uuid()"), ), sa.Column( "name", diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py index 44f574e7a28..2a13e3d718e 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_purchases.py @@ -16,7 +16,7 @@ UUID(as_uuid=True), nullable=False, primary_key=True, - server_default="gen_random_uuid()", + server_default=sa.text("gen_random_uuid()"), ), sa.Column( "product_name", diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py index 9b8f5dc618a..27d6afe8250 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_licensed_items_usage.py @@ -15,7 +15,7 @@ UUID(as_uuid=True), nullable=False, primary_key=True, - server_default="gen_random_uuid()", + server_default=sa.text("gen_random_uuid()"), ), sa.Column( "licensed_item_id",