Feat: Add Filter, Sort and Pagination Support (#20)

* chore: resolve fastapi exception handlers' signature errors

* refactor: extend response data serialization function

* fix: resolve user schema type issues

* chore: remove leftover print statement

* refactor: separate user 404 error condition from user fetch function

* chore: move ItemIdType enum to schemas

* feat: add pagination schema

* feat: add utility function to create paginated responses

* feat: add sort input dependency's  schema

* feat: enable sort on database queries

* feat: enable query filtering

* refactor: remove extra json body nesting for filter-sort input

* feat: allow chaining filter-sort and pagination queries

* fix: check for apt negative condition for filter-sort objects

src/config/constants/app.py

@@ -1,6 +1,11 @@
+from typing import Literal
 from uuid import uuid4
 import datetime as dt
 
+from beanie.odm.interfaces.find import FindType, DocumentProjectionType
+from beanie.odm.queries.find import FindMany
+
+
 PROJECT_NAME = "backend_burger"
 S3_BUCKET_NAME = "backend-burger"
 S3_FOLDER_NAME = "logs"
@@ -18,3 +23,11 @@
 USER_CACHE_KEY = "users"
 SINGLE_USER_CACHE_DURATION = 60 * 60
 USERS_CACHE_DURATION = 5 * 60
+
+ITEMS_PER_PAGE = 100
+MAXIMUM_ITEMS_PER_PAGE = 500
+
+SORT_OPERATIONS = Literal["asc", "desc"]
+FIND_MANY_QUERY = FindMany[FindType] | FindMany[DocumentProjectionType]
+
+FILTER_OPERATIONS = Literal["=", "!=", ">", ">=", "<", "<=", "like"]

src/main.py

@@ -14,7 +14,7 @@
 )
 from src.config.middleware import ExceptionHandlerMiddleware, LoggingMiddleware
 from src.config.services import setup_services
-from src.routers import users, auth
+from src.routers import poe, users, auth
 from src.schemas.responses import BaseResponse
 
 
@@ -27,6 +27,7 @@
 
 app.include_router(users.router)
 app.include_router(auth.router)
+app.include_router(poe.router)
 
 app.add_exception_handler(status.HTTP_400_BAD_REQUEST, handle_invalid_input_exception)
 app.add_exception_handler(RequestValidationError, handle_validation_exception)

src/models/poe.py

@@ -1,15 +1,7 @@
-from enum import Enum
-
 from beanie import Link
-from pydantic import Field, Json
 
 from src.models.common import DateMetadataDocument
-from src.schemas.poe import ItemPrice
-
-
-class ItemIdType(str, Enum):
-    pay = "pay"
-    receive = "receive"
+from src.schemas.poe import ItemBase
 
 
 class ItemCategory(DateMetadataDocument):
@@ -26,19 +18,11 @@ class Settings:
         name = "poe_item_categories"
 
 
-class Item(DateMetadataDocument):
+class Item(ItemBase, DateMetadataDocument):
     """Item represents a Path of Exile in-game item. Each item belongs to a category. It contains information such as
     item type and the current, past and predicted pricing, encapsulated in the `ItemPrice` schema."""
 
-    poe_ninja_id: int
-    id_type: ItemIdType | None = None
-    name: str
     category: Link[ItemCategory]
-    price: Json[ItemPrice] | None = None
-    type_: str | None = Field(None, serialization_alias="type")
-    variant: str | None = None
-    icon_url: str | None = None
-    enabled: bool = True
 
     class Settings:
         """Defines the settings for the collection."""

src/routers/poe.py

@@ -0,0 +1,29 @@
+from fastapi import APIRouter, Depends, Query
+
+from src import dependencies as deps
+from src.schemas.web_responses import users as resp
+from src.schemas.responses import AppResponse, BaseResponse
+from src.services import poe as service
+
+
+dependencies = [Depends(deps.check_access_token)]
+router = APIRouter(prefix="/poe", tags=["Path of Exile"], dependencies=dependencies)
+
+
+# TODO: add responses
+@router.get("/categories", responses=resp.GET_USERS_RESPONSES)
+async def get_all_categories():
+    """Gets a list of all item categories from the database, mapped by their group names."""
+
+    item_categories = await service.get_item_categories()
+    item_category_mapping = service.group_item_categories(item_categories)
+
+    return AppResponse(BaseResponse(data=item_category_mapping))
+
+
+@router.get("/items", responses=resp.GET_USERS_RESPONSES)
+async def get_items_by_group(category_group: str = Query(..., min_length=3, max_length=50)):
+    """Gets a list of all items belonging to the given category group."""
+
+    items = await service.get_items_by_group(category_group)
+    return AppResponse(BaseResponse(data=items))

src/schemas/poe.py

@@ -2,14 +2,19 @@
 from decimal import Decimal
 from enum import Enum
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field, Json
 
 
 class Currency(str, Enum):
     chaos = "chaos"
     divines = "divines"
 
 
+class ItemIdType(str, Enum):
+    pay = "pay"
+    receive = "receive"
+
+
 class ItemPrice(BaseModel):
     """ItemPrice holds information regarding the current, past and future price of an item.
     It stores the recent and predicted prices in a dictionary, with the date as the key."""
@@ -20,3 +25,24 @@ class ItemPrice(BaseModel):
     price_history_currency: Currency
     price_prediction: dict[dt.datetime, Decimal]
     price_prediction_currency: Currency
+
+
+class ItemCategoryResponse(BaseModel):
+    """ItemCategoryResponse holds the requisite subset of ItemCategory's data for API responses."""
+
+    name: str
+    internal_name: str
+    group: str
+
+
+class ItemBase(BaseModel):
+    """ItemBase encapsulates core fields of the Item document."""
+
+    poe_ninja_id: int
+    id_type: ItemIdType | None = None
+    name: str
+    price: Json[ItemPrice] | None = None
+    type_: str | None = Field(None, serialization_alias="type")
+    variant: str | None = None
+    icon_url: str | None = None
+    enabled: bool = True

src/schemas/requests.py

@@ -0,0 +1,45 @@
+from typing import Annotated
+from fastapi import Body, Query
+from pydantic import BaseModel, BeforeValidator, Field, computed_field
+
+from src.config.constants.app import FILTER_OPERATIONS, ITEMS_PER_PAGE, MAXIMUM_ITEMS_PER_PAGE, SORT_OPERATIONS
+
+
+lowercase_validator = BeforeValidator(lambda v: v.lower())
+
+
+class PaginationInput(BaseModel):
+    """PaginationInput encapsulates query parameters required for a paginated response."""
+
+    page: int = Query(1, gt=0)
+    per_page: int = Query(ITEMS_PER_PAGE, gt=0, lte=MAXIMUM_ITEMS_PER_PAGE)
+
+    @computed_field
+    @property
+    def offset(self) -> int:
+        """Calculates the offset value for use in database queries."""
+
+        return (self.page - 1) * self.per_page
+
+
+class SortSchema(BaseModel):
+    """SortInput encapsulates the sorting schema model, requiring the field to sort on, and the sort operation type."""
+
+    field: Annotated[str, lowercase_validator]
+    operation: Annotated[SORT_OPERATIONS, lowercase_validator]
+
+
+class FilterSchema(BaseModel):
+    """FilterSchema encapsulates the filter schema model, requiring a field, a valid operation and the value to filter on the field by."""
+
+    field: Annotated[str, lowercase_validator]
+    operation: Annotated[FILTER_OPERATIONS, lowercase_validator]
+    value: str = Field(min_length=1, max_length=200)
+
+
+class FilterSortInput(BaseModel):
+    """FilterSortInput wraps filter and sort schema implementations, enabling them to be embedded as JSON body
+    parameters for FastAPI request handler functions."""
+
+    filter_: list[FilterSchema] | None = Field(Body(None, embed=True), alias="filter")
+    sort: list[SortSchema] | None = Field(Body(None, embed=True))

src/schemas/responses.py

@@ -4,6 +4,8 @@
 from starlette import status
 from pydantic import BaseModel, Field, root_validator
 
+from src.config.constants.app import MAXIMUM_ITEMS_PER_PAGE
+
 
 # T represents any Pydantic BaseModel or Beanie Document, dict or list of BaseModel/Document or dict return types
 # TODO: define apt type constraints, currently failing with BaseModel constraint
@@ -53,7 +55,7 @@ def __init__(
         """Dumps Pydantic models or keeps content as is, passing it to the parent `__init__` function."""
 
         if isinstance(content, BaseResponse):
-            data = content.model_dump()
+            data = content.model_dump(mode="json")
         else:
             data = content
 
@@ -66,3 +68,12 @@ def render(self, content: Any) -> bytes:
             return content
 
         return super().render(content)
+
+
+class PaginationResponse(BaseModel):
+    """PaginationResponse encapsulates pagination values required by the client."""
+
+    page: int = Field(gt=0)
+    per_page: int = Field(gt=0, le=MAXIMUM_ITEMS_PER_PAGE)
+    total_items: int
+    total_pages: int

src/services/poe.py

@@ -0,0 +1,61 @@
+from collections import defaultdict
+
+from fastapi import HTTPException
+from loguru import logger
+from starlette import status
+
+from src.models.poe import Item, ItemCategory
+from src.schemas.poe import ItemBase, ItemCategoryResponse
+
+
+async def get_item_categories() -> list[ItemCategoryResponse]:
+    """Gets all item category documents from the database, extracting only the required fields from the documents."""
+
+    try:
+        item_categories = await ItemCategory.find_all().project(ItemCategoryResponse).to_list()
+    except Exception as exc:
+        logger.error(f"error getting item categories: {exc}")
+        raise
+
+    return item_categories
+
+
+def group_item_categories(item_categories: list[ItemCategoryResponse]) -> dict[str, list[ItemCategoryResponse]]:
+    """Groups item category documents by their category group."""
+
+    item_category_mapping = defaultdict(list)
+
+    for category in item_categories:
+        item_category_mapping[category.group].append(category)
+
+    return item_category_mapping
+
+
+async def get_items_by_group(category_group: str) -> list[ItemBase]:
+    """
+    Gets items by given category group. Raises a 400 error if category group is invalid.
+    """
+
+    try:
+        item_category = await ItemCategory.find_one(ItemCategory.group == category_group)
+    except Exception as exc:
+        logger.error(f"error getting item category by group '{category_group}': {exc} ")
+        raise
+
+    if item_category is None:
+        raise HTTPException(status.HTTP_400_BAD_REQUEST, "Invalid category group.")
+
+    try:
+        items = (
+            await Item.find(
+                Item.category.group == category_group,  # type: ignore
+                fetch_links=True,
+            )
+            .project(ItemBase)
+            .to_list()
+        )
+    except Exception as exc:
+        logger.error(f"error getting item by category group '{category_group}': {exc}")
+        raise
+
+    return items

src/utils/routers.py

@@ -1,9 +1,11 @@
-from typing import Awaitable
+import math
+from typing import Any, Awaitable
 
 from loguru import logger
 from redis.asyncio import Redis
 
-from src.schemas.responses import BaseResponse
+from src.schemas.requests import PaginationInput
+from src.schemas.responses import BaseResponse, PaginationResponse
 from src.utils.services import cache_data, get_cached_data, serialize_response
 
 
@@ -41,3 +43,18 @@ async def get_or_cache_serialized_entity(
     logger.debug(f"cached '{redis_key}' data")
 
     return serialized_entity
+
+
+def create_pagination_response(data: Any, total_items: int, pagination: PaginationInput, data_key: str) -> BaseResponse:
+    """Creates a response structure warpped in a `BaseResponse`, calculating and assigning pagination values."""
+
+    per_page = pagination.per_page
+    total_pages = math.ceil(total_items / per_page)
+
+    pagination_response = PaginationResponse(
+        page=pagination.page, per_page=pagination.per_page, total_items=total_items, total_pages=total_pages
+    )
+    response_data = {data_key: data, "pagination": pagination_response}
+
+    response = BaseResponse(data=response_data)
+    return response