From 0442238028be3a006893fa9a26dcf92f1394dacb Mon Sep 17 00:00:00 2001 From: pdmurray Date: Tue, 17 Sep 2024 13:22:18 -0700 Subject: [PATCH 1/2] Sort envs returned by REST API by current build's scheduled_on time --- .../conda_store_server/_internal/server/views/api.py | 12 +++++++----- conda-store-server/conda_store_server/api.py | 9 +++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/conda-store-server/conda_store_server/_internal/server/views/api.py b/conda-store-server/conda_store_server/_internal/server/views/api.py index 496e46f6..31522de5 100644 --- a/conda-store-server/conda_store_server/_internal/server/views/api.py +++ b/conda-store-server/conda_store_server/_internal/server/views/api.py @@ -676,8 +676,10 @@ async def api_list_environments( Returns ------- Dict - Paginated JSON response containing the requested environments - + Paginated JSON response containing the requested environments. Results are sorted by each + envrionment's build's scheduled_on time to ensure all results are returned when iterating + over pages in systems where the number of environments is changing while results are being + requested; see https://github.com/conda-incubator/conda-store/issues/859 for context """ with conda_store.get_db() as db: if jwt: @@ -712,10 +714,10 @@ async def api_list_environments( schema.Environment, exclude={"current_build"}, allowed_sort_bys={ - "namespace": orm.Namespace.name, - "name": orm.Environment.name, + "scheduled_on": orm.Environment.current_build.scheduled_on, }, - default_sort_by=["namespace", "name"], + default_sort_by=["scheduled_on"], + default_order="asc", ) diff --git a/conda-store-server/conda_store_server/api.py b/conda-store-server/conda_store_server/api.py index 7b937046..0b582c5d 100644 --- a/conda-store-server/conda_store_server/api.py +++ b/conda-store-server/conda_store_server/api.py @@ -324,7 +324,11 @@ def list_environments( Query Sqlalchemy query containing the requested environments """ - query = db.query(orm.Environment).join(orm.Environment.namespace) + query = ( + db.query(orm.Environment) + .join(orm.Environment.namespace) + .join(orm.Environment.current_build) + ) if namespace: query = query.filter(orm.Namespace.name == namespace) @@ -343,9 +347,6 @@ def list_environments( if not show_soft_deleted: query = query.filter(orm.Environment.deleted_on == null()) - if status or artifact or packages: - query = query.join(orm.Environment.current_build) - if status: query = query.filter(orm.Build.status == status) From 6cf6c9229ca6ae1dee23751148b2f3a91f6e686f Mon Sep 17 00:00:00 2001 From: pdmurray Date: Fri, 18 Oct 2024 10:29:39 -0700 Subject: [PATCH 2/2] Add fastapi_pagination --- .../_internal/server/app.py | 3 + .../_internal/server/views/api.py | 86 ++++++++++++++----- conda-store-server/pyproject.toml | 2 + 3 files changed, 68 insertions(+), 23 deletions(-) diff --git a/conda-store-server/conda_store_server/_internal/server/app.py b/conda-store-server/conda_store_server/_internal/server/app.py index ffe66fd1..9ba0e7ef 100644 --- a/conda-store-server/conda_store_server/_internal/server/app.py +++ b/conda-store-server/conda_store_server/_internal/server/app.py @@ -18,6 +18,7 @@ from fastapi.responses import FileResponse, JSONResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates +from fastapi_pagination import add_pagination from sqlalchemy.pool import QueuePool from starlette.middleware.sessions import SessionMiddleware from traitlets import ( @@ -239,6 +240,8 @@ def trim_slash(url): }, ) + add_pagination(app) + app.add_middleware( CORSMiddleware, allow_origins=self.cors_allow_origins, diff --git a/conda-store-server/conda_store_server/_internal/server/views/api.py b/conda-store-server/conda_store_server/_internal/server/views/api.py index 31522de5..218a7ecd 100644 --- a/conda-store-server/conda_store_server/_internal/server/views/api.py +++ b/conda-store-server/conda_store_server/_internal/server/views/api.py @@ -4,13 +4,18 @@ import datetime -from typing import Any, Dict, List, Optional, TypedDict +from typing import Any, Dict, List, Optional, Tuple, TypedDict import pydantic import yaml from fastapi import APIRouter, Body, Depends, HTTPException, Query, Request from fastapi.responses import PlainTextResponse, RedirectResponse +from fastapi_pagination import set_params +from fastapi_pagination.cursor import CursorPage, CursorParams +from fastapi_pagination.ext.sqlalchemy import paginate +from pydantic import BaseModel +from sqlalchemy.orm import Query as SqlQuery from conda_store_server import __version__, api, app from conda_store_server._internal import orm, schema, utils @@ -20,6 +25,9 @@ from conda_store_server.server.auth import Authentication +set_params(CursorParams(size=10, cursor=None)) + + class PaginatedArgs(TypedDict): """Dictionary type holding information about paginated requests.""" @@ -55,19 +63,40 @@ def get_paginated_args( def filter_distinct_on( - query, - distinct_on: List[str] = [], - allowed_distinct_ons: Dict = {}, - default_distinct_on: List[str] = [], -): - distinct_on = distinct_on or default_distinct_on - distinct_on = [ - allowed_distinct_ons[d] for d in distinct_on if d in allowed_distinct_ons - ] + query: SqlQuery, + distinct_on: List[str] | None = None, + allowed_distinct_ons: Dict | None = None, +) -> Tuple[List[str], SqlQuery]: + """Filter the query using the distinct fields. + + Parameters + ---------- + query : SqlQuery + Query to filter + distinct_on : List[str] | None + Parameter to pass to the FILTER DISTINCT statement + allowed_distinct_ons : Dict | None + Allowed values of the parameter + + Returns + ------- + SqlQuery + Query containing filtered results + """ + if distinct_on is None: + distinct_on = [] + + disallowed = set(distinct_on) - set(allowed_distinct_ons) + if disallowed: + raise HTTPException( + status_code=400, + detail=( + f"Requested distinct_on terms ({disallowed}) are not allowed. " + f"Valid terms are {set(allowed_distinct_ons)}" + ), + ) - if distinct_on: - return distinct_on, query.distinct(*distinct_on) - return distinct_on, query + return query.distinct(*[allowed_distinct_ons[item] for item in distinct_on]) def get_sorts( @@ -94,6 +123,15 @@ def get_sorts( return [order_mapping[order](k) for k in sort_by] +def paginate_response( + query: SqlQuery, + obj_schema: BaseModel, + order: str = "asc", + sort_by: List[str] = None, +) -> CursorPage: + return + + def paginated_api_response( query, paginated_args, @@ -104,7 +142,7 @@ def paginated_api_response( required_sort_bys: List = [], default_sort_by: List = [], default_order: str = "asc", -): +) -> CursorPage: sorts = get_sorts( order=paginated_args["order"], sort_by=paginated_args["sort_by"], @@ -114,15 +152,17 @@ def paginated_api_response( default_order=default_order, ) - count = query.count() - query = ( - query.order_by(*sorts) - .limit(paginated_args["limit"]) - .offset(paginated_args["offset"]) + print( + query, + paginated_args, + object_schema, + sorts, ) + + count = query.count() return { "status": "ok", - "data": [object_schema.from_orm(_).dict(exclude=exclude) for _ in query.all()], + "data": paginate(query.order_by(*sorts)), "page": (paginated_args["offset"] // paginated_args["limit"]) + 1, "size": paginated_args["limit"], "count": count, @@ -713,9 +753,9 @@ async def api_list_environments( paginated_args, schema.Environment, exclude={"current_build"}, - allowed_sort_bys={ - "scheduled_on": orm.Environment.current_build.scheduled_on, - }, + # allowed_sort_bys={ + # "scheduled_on": orm.Environment.current_build.scheduled_on, + # }, default_sort_by=["scheduled_on"], default_order="asc", ) diff --git a/conda-store-server/pyproject.toml b/conda-store-server/pyproject.toml index d2e2a970..7085fae6 100644 --- a/conda-store-server/pyproject.toml +++ b/conda-store-server/pyproject.toml @@ -52,6 +52,7 @@ dependencies = [ "alembic", "celery", "fastapi", + "fastapi_pagination", "filelock", "flower", "itsdangerous", @@ -65,6 +66,7 @@ dependencies = [ "pydantic >=1.10.16,<2.0a0", "python-multipart", "sqlalchemy<2.0a0", + "sqlakeyset", "traitlets", "uvicorn", "yarl",