From 95412760bed6414be8245b0939418e971c8677eb Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Fri, 31 May 2024 12:15:52 -0400 Subject: [PATCH 1/3] Remove pagination from roms endpoints --- backend/endpoints/rom.py | 53 +++++++++++++------ backend/main.py | 2 - frontend/src/__generated__/index.ts | 1 - .../models/CursorPage_RomSchema_.ts | 31 ----------- frontend/src/components/Dashboard/Recent.vue | 2 +- .../src/components/Dialog/Rom/SearchRom.vue | 4 +- frontend/src/services/api/rom.ts | 16 ++---- frontend/src/views/Gallery/Base.vue | 4 +- poetry.lock | 35 +----------- pyproject.toml | 1 - 10 files changed, 49 insertions(+), 100 deletions(-) delete mode 100644 frontend/src/__generated__/models/CursorPage_RomSchema_.ts diff --git a/backend/endpoints/rom.py b/backend/endpoints/rom.py index 6df3644b6..6ef6f5594 100644 --- a/backend/endpoints/rom.py +++ b/backend/endpoints/rom.py @@ -16,8 +16,6 @@ from exceptions.fs_exceptions import RomAlreadyExistsException from fastapi import APIRouter, File, HTTPException, Query, Request, UploadFile, status from fastapi.responses import FileResponse -from fastapi_pagination.cursor import CursorPage, CursorParams -from fastapi_pagination.ext.sqlalchemy import paginate from handler.database import db_platform_handler, db_rom_handler from handler.filesystem import fs_resource_handler, fs_rom_handler from handler.filesystem.base_handler import CoverSize @@ -25,6 +23,7 @@ from logger.logger import log from stream_zip import ZIP_AUTO, stream_zip # type: ignore[import] from urllib.parse import quote + router = APIRouter() @@ -88,12 +87,10 @@ def add_roms( def get_roms( request: Request, platform_id: int = None, - size: int = 60, - cursor: str = "", search_term: str = "", order_by: str = "name", order_dir: str = "asc", -) -> CursorPage[RomSchema]: +) -> list[RomSchema]: """Get roms endpoint Args: @@ -101,18 +98,42 @@ def get_roms( id (int, optional): Rom internal id Returns: - RomSchema: Rom stored in the database + list[RomSchema]: List of roms stored in the database """ with db_rom_handler.session.begin() as session: - cursor_params = CursorParams(size=size, cursor=cursor) - qq = db_rom_handler.get_roms( - platform_id=platform_id, - search_term=search_term.lower(), - order_by=order_by.lower(), - order_dir=order_dir.lower(), - ) - return paginate(session, qq, cursor_params) + return session.scalars( + db_rom_handler.get_roms( + platform_id=platform_id, + search_term=search_term.lower(), + order_by=order_by.lower(), + order_dir=order_dir.lower(), + ) + ).all() + + +@protected_route(router.get, "/roms/recent", ["roms.read"]) +def get_recent_roms( + request: Request, + size: int = 15, +) -> list[RomSchema]: + """Get the most recent roms endpoint + + Args: + request (Request): Fastapi Request object + id (int, optional): Rom internal id + + Returns: + list[RomSchema]: List of roms stored in the database + """ + + with db_rom_handler.session.begin() as session: + return session.scalars( + db_rom_handler.get_roms( + order_by="id", + order_dir="desc", + ).limit(size) + ).all() @protected_route( @@ -237,7 +258,9 @@ def contents(f): return CustomStreamingResponse( zipped_chunks, media_type="application/zip", - headers={"Content-Disposition": f'attachment; filename="{quote(file_name)}.zip"'}, + headers={ + "Content-Disposition": f'attachment; filename="{quote(file_name)}.zip"' + }, emit_body={"id": rom.id}, ) diff --git a/backend/main.py b/backend/main.py index 199d9ba0c..8293b1ac1 100644 --- a/backend/main.py +++ b/backend/main.py @@ -25,7 +25,6 @@ import endpoints.sockets.scan # noqa from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi_pagination import add_pagination from handler.database import db_user_handler from handler.socket_handler import socket_handler from handler.auth import auth_handler @@ -95,7 +94,6 @@ async def lifespan(app: FastAPI): app.include_router(screenshots.router) app.include_router(firmware.router) -add_pagination(app) app.mount("/ws", socket_handler.socket_app) diff --git a/frontend/src/__generated__/index.ts b/frontend/src/__generated__/index.ts index 1fc2b9cf4..fc5b6f174 100644 --- a/frontend/src/__generated__/index.ts +++ b/frontend/src/__generated__/index.ts @@ -14,7 +14,6 @@ export type { Body_token_token_post } from './models/Body_token_token_post'; export type { Body_update_rom_roms__id__put } from './models/Body_update_rom_roms__id__put'; export type { Body_update_user_users__id__put } from './models/Body_update_user_users__id__put'; export type { ConfigResponse } from './models/ConfigResponse'; -export type { CursorPage_RomSchema_ } from './models/CursorPage_RomSchema_'; export type { DetailedRomSchema } from './models/DetailedRomSchema'; export type { FirmwareSchema } from './models/FirmwareSchema'; export type { HeartbeatResponse } from './models/HeartbeatResponse'; diff --git a/frontend/src/__generated__/models/CursorPage_RomSchema_.ts b/frontend/src/__generated__/models/CursorPage_RomSchema_.ts deleted file mode 100644 index 3a95eeb11..000000000 --- a/frontend/src/__generated__/models/CursorPage_RomSchema_.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* generated using openapi-typescript-codegen -- do no edit */ -/* istanbul ignore file */ -/* tslint:disable */ -/* eslint-disable */ - -import type { RomSchema } from './RomSchema'; - -export type CursorPage_RomSchema_ = { - items: Array; - /** - * Total items - */ - total?: (number | null); - /** - * Cursor to refetch the current page - */ - current_page?: (string | null); - /** - * Cursor to refetch the current page starting from the last item - */ - current_page_backwards?: (string | null); - /** - * Cursor for the previous page - */ - previous_page?: (string | null); - /** - * Cursor for the next page - */ - next_page?: (string | null); -}; - diff --git a/frontend/src/components/Dashboard/Recent.vue b/frontend/src/components/Dashboard/Recent.vue index 3d6850846..00de68712 100644 --- a/frontend/src/components/Dashboard/Recent.vue +++ b/frontend/src/components/Dashboard/Recent.vue @@ -19,7 +19,7 @@ onMounted(() => { romApi .getRecentRoms() .then(({ data: recentData }) => { - romsStore.setRecentRoms(recentData.items); + romsStore.setRecentRoms(recentData); }) .catch((error) => { console.error(error); diff --git a/frontend/src/components/Dialog/Rom/SearchRom.vue b/frontend/src/components/Dialog/Rom/SearchRom.vue index 2f0572906..e652f456f 100644 --- a/frontend/src/components/Dialog/Rom/SearchRom.vue +++ b/frontend/src/components/Dialog/Rom/SearchRom.vue @@ -47,8 +47,8 @@ async function searchRoms() { inputElement?.blur(); searching.value = true; searchedRoms.value = ( - await romApi.getRoms({ searchTerm: searchValue.value, size: 250 }) - ).data.items.sort((a, b) => { + await romApi.getRoms({ searchTerm: searchValue.value }) + ).data.sort((a, b) => { return a.platform_name.localeCompare(b.platform_name); }); platforms.value = [ diff --git a/frontend/src/services/api/rom.ts b/frontend/src/services/api/rom.ts index 1663a8b0d..f24125dcf 100644 --- a/frontend/src/services/api/rom.ts +++ b/frontend/src/services/api/rom.ts @@ -1,7 +1,7 @@ import type { AddRomsResponse, - CursorPage_RomSchema_, MessageResponse, + RomSchema, SearchRomSchema, } from "@/__generated__"; import api from "@/services/api/index"; @@ -32,24 +32,18 @@ async function uploadRoms({ async function getRoms({ platformId = null, - size = 5000, - cursor = "", searchTerm = "", orderBy = "name", orderDir = "asc", }: { platformId?: number | null; - size?: number | null; - cursor?: string | null; searchTerm?: string | null; orderBy?: string | null; orderDir?: string | null; -}): Promise<{ data: CursorPage_RomSchema_ }> { +}): Promise<{ data: SimpleRom[] }> { return api.get(`/roms`, { params: { platform_id: platformId, - size: size, - cursor: cursor, search_term: searchTerm, order_by: orderBy, order_dir: orderDir, @@ -57,9 +51,9 @@ async function getRoms({ }); } -async function getRecentRoms(): Promise<{ data: CursorPage_RomSchema_ }> { - return api.get("/roms", { - params: { size: 15, order_by: "id", order_dir: "desc" }, +async function getRecentRoms(): Promise<{ data: SimpleRom[] }> { + return api.get("/roms/recent", { + params: { size: 15 }, }); } diff --git a/frontend/src/views/Gallery/Base.vue b/frontend/src/views/Gallery/Base.vue index a9538f93f..7f95f7577 100644 --- a/frontend/src/views/Gallery/Base.vue +++ b/frontend/src/views/Gallery/Base.vue @@ -61,12 +61,12 @@ async function fetchRoms() { }) .then(({ data }) => { // Add any new roms to the store - const allRomsSet = [...allRoms.value, ...data.items]; + const allRomsSet = [...allRoms.value, ...data]; romsStore.set(allRomsSet); romsStore.setFiltered(allRomsSet, galleryFilterStore); if (galleryFilterStore.isFiltered()) { - const serchedRomsSet = [...searchRoms.value, ...data.items]; + const serchedRomsSet = [...searchRoms.value, ...data]; romsStore.setSearch(serchedRomsSet); romsStore.setFiltered(serchedRomsSet, galleryFilterStore); } diff --git a/poetry.lock b/poetry.lock index 4f8ce7782..e50ac91ed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -491,39 +491,6 @@ typing-extensions = ">=4.8.0" [package.extras] all = ["email-validator (>=2.0.0)", "httpx (>=0.23.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=2.11.2)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.7)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] -[[package]] -name = "fastapi-pagination" -version = "0.12.24" -description = "FastAPI pagination" -optional = false -python-versions = "<4.0,>=3.8" -files = [ - {file = "fastapi_pagination-0.12.24-py3-none-any.whl", hash = "sha256:a639df7301a89414244c6763bb97cff043815cb839070b8a38c58c007cf75d48"}, - {file = "fastapi_pagination-0.12.24.tar.gz", hash = "sha256:c9c6508e0182aab679a13b1de261d4923e3b530b410500dcb271638ff714fb14"}, -] - -[package.dependencies] -fastapi = ">=0.93.0" -pydantic = ">=1.9.1" -typing-extensions = ">=4.8.0,<5.0.0" - -[package.extras] -all = ["SQLAlchemy (>=1.3.20)", "asyncpg (>=0.24.0)", "beanie (>=1.25.0)", "bunnet (>=1.1.0,<2.0.0)", "databases (>=0.6.0)", "django (<5.0.0)", "mongoengine (>=0.23.1,<0.29.0)", "motor (>=2.5.1,<4.0.0)", "orm (>=0.3.1)", "ormar (>=0.11.2)", "piccolo (>=0.89,<0.122)", "pony (>=0.7.16,<0.8.0)", "scylla-driver (>=3.25.6,<4.0.0)", "sqlakeyset (>=2.0.1680321678,<3.0.0)", "sqlmodel (>=0.0.8,<0.0.15)", "tortoise-orm (>=0.16.18,<0.21.0)"] -asyncpg = ["SQLAlchemy (>=1.3.20)", "asyncpg (>=0.24.0)"] -beanie = ["beanie (>=1.25.0)"] -bunnet = ["bunnet (>=1.1.0,<2.0.0)"] -databases = ["databases (>=0.6.0)"] -django = ["databases (>=0.6.0)", "django (<5.0.0)"] -mongoengine = ["mongoengine (>=0.23.1,<0.29.0)"] -motor = ["motor (>=2.5.1,<4.0.0)"] -orm = ["databases (>=0.6.0)", "orm (>=0.3.1)"] -ormar = ["ormar (>=0.11.2)"] -piccolo = ["piccolo (>=0.89,<0.122)"] -scylla-driver = ["scylla-driver (>=3.25.6,<4.0.0)"] -sqlalchemy = ["SQLAlchemy (>=1.3.20)", "sqlakeyset (>=2.0.1680321678,<3.0.0)"] -sqlmodel = ["sqlakeyset (>=2.0.1680321678,<3.0.0)", "sqlmodel (>=0.0.8,<0.0.15)"] -tortoise = ["tortoise-orm (>=0.16.18,<0.21.0)"] - [[package]] name = "freezegun" version = "1.5.1" @@ -2518,4 +2485,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "2512d625904e0d46e08726537a7d9a71267ab8bfd69eeff9b60a62493873cddc" +content-hash = "74172854654d4aea35bdc0dd15ac2f9c77ed6b11541bed0f11a6d0c5bb408ca2" diff --git a/pyproject.toml b/pyproject.toml index 1b9c8d721..0ef557bde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ PyYAML = "6.0.1" Unidecode = "1.3.8" emoji = "2.10.1" python-dotenv = "1.0.1" -fastapi-pagination = "^0.12.19" sqlakeyset = "^2.0.1708907391" pydash = "^7.0.7" mariadb = "1.1.10" From 2d24b762b4ebbd6da03a3f679693b6113f1605bf Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Fri, 31 May 2024 12:22:20 -0400 Subject: [PATCH 2/3] merge endpoints again --- backend/endpoints/rom.py | 27 ++------------------------- frontend/src/services/api/rom.ts | 4 ++-- 2 files changed, 4 insertions(+), 27 deletions(-) diff --git a/backend/endpoints/rom.py b/backend/endpoints/rom.py index 6ef6f5594..1d987f803 100644 --- a/backend/endpoints/rom.py +++ b/backend/endpoints/rom.py @@ -88,6 +88,7 @@ def get_roms( request: Request, platform_id: int = None, search_term: str = "", + limit: int = None, order_by: str = "name", order_dir: str = "asc", ) -> list[RomSchema]: @@ -108,31 +109,7 @@ def get_roms( search_term=search_term.lower(), order_by=order_by.lower(), order_dir=order_dir.lower(), - ) - ).all() - - -@protected_route(router.get, "/roms/recent", ["roms.read"]) -def get_recent_roms( - request: Request, - size: int = 15, -) -> list[RomSchema]: - """Get the most recent roms endpoint - - Args: - request (Request): Fastapi Request object - id (int, optional): Rom internal id - - Returns: - list[RomSchema]: List of roms stored in the database - """ - - with db_rom_handler.session.begin() as session: - return session.scalars( - db_rom_handler.get_roms( - order_by="id", - order_dir="desc", - ).limit(size) + ).limit(limit) ).all() diff --git a/frontend/src/services/api/rom.ts b/frontend/src/services/api/rom.ts index f24125dcf..426729ae3 100644 --- a/frontend/src/services/api/rom.ts +++ b/frontend/src/services/api/rom.ts @@ -52,8 +52,8 @@ async function getRoms({ } async function getRecentRoms(): Promise<{ data: SimpleRom[] }> { - return api.get("/roms/recent", { - params: { size: 15 }, + return api.get("/roms", { + params: { order_by: "id", order_dir: "desc", limit: 15 }, }); } From b8c29b756e2767d80ba28a6678aed419cf7079af Mon Sep 17 00:00:00 2001 From: Zurdi Date: Fri, 31 May 2024 18:30:43 +0200 Subject: [PATCH 3/3] unit tests fixed --- backend/endpoints/tests/test_rom.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/endpoints/tests/test_rom.py b/backend/endpoints/tests/test_rom.py index afdec9fb7..66314f13e 100644 --- a/backend/endpoints/tests/test_rom.py +++ b/backend/endpoints/tests/test_rom.py @@ -26,8 +26,8 @@ def test_get_all_roms(access_token, rom, platform): assert response.status_code == 200 body = response.json() - assert len(body["items"]) == 1 - assert body["items"][0]["id"] == rom.id + assert len(body) == 1 + assert body[0]["id"] == rom.id @patch("endpoints.rom.fs_rom_handler.rename_file")