diff --git a/API/auth/__init__.py b/API/auth/__init__.py index 0ff71d79..04e1bf13 100644 --- a/API/auth/__init__.py +++ b/API/auth/__init__.py @@ -1,13 +1,18 @@ from enum import Enum from typing import Union -from fastapi import Depends, Header, HTTPException +from fastapi.security import APIKeyHeader +from fastapi import Depends, HTTPException from osm_login_python.core import Auth from pydantic import BaseModel, Field from src.app import Users from src.config import get_oauth_credentials +API_Access_Token = APIKeyHeader( + name="Access_Token", description="Access Token to Authorize User", auto_error=False +) + class UserRole(Enum): ADMIN = 1 @@ -21,6 +26,16 @@ class AuthUser(BaseModel): img_url: Union[str, None] role: UserRole = Field(default=UserRole.GUEST.value) + class Config: + json_schema_extra = { + "example": { + "id": "123", + "username": "HOT Team", + "img_url": "https://hotosm/image.jpg", + "role": UserRole.GUEST.value, + } + } + osm_auth = Auth(*get_oauth_credentials()) @@ -34,7 +49,7 @@ def get_user_from_db(osm_id: int): def get_osm_auth_user(access_token): try: user = AuthUser(**osm_auth.deserialize_access_token(access_token)) - except Exception as ex: + except Exception: raise HTTPException( status_code=403, detail=[{"msg": "OSM Authentication failed"}] ) @@ -43,11 +58,11 @@ def get_osm_auth_user(access_token): return user -def login_required(access_token: str = Header(...)): +def login_required(access_token: str = Depends(API_Access_Token)): return get_osm_auth_user(access_token) -def get_optional_user(access_token: str = Header(default=None)) -> AuthUser: +def get_optional_user(access_token: str | None = Depends(API_Access_Token)) -> AuthUser: if access_token: return get_osm_auth_user(access_token) else: @@ -57,8 +72,8 @@ def get_optional_user(access_token: str = Header(default=None)) -> AuthUser: def admin_required(user: AuthUser = Depends(login_required)): db_user = get_user_from_db(user.id) - if not db_user["role"] is UserRole.ADMIN.value: - raise HTTPException(status_code=403, detail="User is not an admin") + if db_user["role"] is not UserRole.ADMIN.value: + raise HTTPException(status_code=403, detail=[{"msg": "User is not an admin"}]) return user @@ -70,5 +85,5 @@ def staff_required(user: AuthUser = Depends(login_required)): db_user["role"] is UserRole.STAFF.value or db_user["role"] is UserRole.ADMIN.value ): - raise HTTPException(status_code=403, detail="User is not a staff") + raise HTTPException(status_code=403, detail=[{"msg": "User is not a staff"}]) return user diff --git a/API/auth/routers.py b/API/auth/routers.py index 438a28e4..a6aadb0a 100644 --- a/API/auth/routers.py +++ b/API/auth/routers.py @@ -1,18 +1,33 @@ -import json - -from fastapi import APIRouter, Depends, Request +from fastapi import APIRouter, Depends, Request, Path, Query from pydantic import BaseModel from src.app import Users +from src.validation.models import ErrorMessage, common_responses from . import AuthUser, admin_required, login_required, osm_auth, staff_required router = APIRouter(prefix="/auth", tags=["Auth"]) -@router.get("/login/") +@router.get( + "/login", + responses={ + 200: { + "description": "A Login URL", + "content": { + "application/json": { + "example": { + "login_url": "https://www.openstreetmap.org/oauth2/authorize/..." + } + } + }, + }, + 500: {"model": ErrorMessage}, + }, +) def login_url(request: Request): - """Generate Login URL for authentication using OAuth2 Application registered with OpenStreetMap. + """ + Generate Login URL for authentication using OAuth2 Application registered with OpenStreetMap. Click on the download url returned to get access_token. Parameters: None @@ -25,7 +40,7 @@ def login_url(request: Request): return login_url -@router.get("/callback/") +@router.get("/callback", responses={500: {"model": ErrorMessage}}) def callback(request: Request): """Performs token exchange between OpenStreetMap and Raw Data API @@ -42,18 +57,27 @@ def callback(request: Request): return access_token -@router.get("/me/", response_model=AuthUser) +@router.get( + "/me", + response_model=AuthUser, + responses={**common_responses}, + response_description="User Information", +) def my_data(user_data: AuthUser = Depends(login_required)): """Read the access token and provide user details from OSM user's API endpoint, also integrated with underpass . Parameters:None - Returns: user_data + Returns: user_data\n User Role : ADMIN = 1 STAFF = 2 GUEST = 3 + + Raises: + - HTTPException 403: Due to authentication error(Wrong access token). + - HTTPException 500: Internal server error """ return user_data @@ -62,9 +86,19 @@ class User(BaseModel): osm_id: int role: int + class Config: + json_schema_extra = {"example": {"osm_id": 123, "role": 1}} + # Create user -@router.post("/users/", response_model=dict) +@router.post( + "/users", + response_model=dict, + responses={ + **common_responses, + "200": {"content": {"application/json": {"example": {"osm_id": 123}}}}, + }, +) async def create_user(params: User, user_data: AuthUser = Depends(admin_required)): """ Creates a new user and returns the user's information. @@ -80,15 +114,26 @@ async def create_user(params: User, user_data: AuthUser = Depends(admin_required - Dict[str, Any]: A dictionary containing the osm_id of the newly created user. Raises: - - HTTPException: If the user creation fails. + - HTTPException 403: If the user creation fails due to insufficient permission. + - HTTPException 500: If the user creation fails due to internal server error. """ auth = Users() return auth.create_user(params.osm_id, params.role) # Read user by osm_id -@router.get("/users/{osm_id}", response_model=dict) -async def read_user(osm_id: int, user_data: AuthUser = Depends(staff_required)): +@router.get( + "/users/{osm_id}", + responses={ + **common_responses, + "200": {"content": {"application/json": {"example": {"osm_id": 1, "role": 2}}}}, + "404": {"model": ErrorMessage}, + }, +) +async def read_user( + osm_id: int = Path(description="The OSM ID of the User to Retrieve"), + user_data: AuthUser = Depends(staff_required), +): """ Retrieves user information based on the given osm_id. User Role : @@ -103,7 +148,9 @@ async def read_user(osm_id: int, user_data: AuthUser = Depends(staff_required)): - Dict[str, Any]: A dictionary containing user information. Raises: - - HTTPException: If the user with the given osm_id is not found. + - HTTPException 403: If the user has insufficient permission. + - HTTPException 404: If the user with the given osm_id is not found. + - HTTPException 500: If it fails due to internal server error. """ auth = Users() @@ -111,9 +158,18 @@ async def read_user(osm_id: int, user_data: AuthUser = Depends(staff_required)): # Update user by osm_id -@router.put("/users/{osm_id}", response_model=dict) +@router.put( + "/users/{osm_id}", + responses={ + **common_responses, + "200": {"content": {"application/json": {"example": {"osm_id": 1, "role": 1}}}}, + "404": {"model": ErrorMessage}, + }, +) async def update_user( - osm_id: int, update_data: User, user_data: AuthUser = Depends(admin_required) + update_data: User, + user_data: AuthUser = Depends(admin_required), + osm_id: int = Path(description="The OSM ID of the User to Update"), ): """ Updates user information based on the given osm_id. @@ -129,15 +185,27 @@ async def update_user( - Dict[str, Any]: A dictionary containing the updated user information. Raises: - - HTTPException: If the user with the given osm_id is not found. + - HTTPException 403: If the user has insufficient permission. + - HTTPException 404: If the user with the given osm_id is not found. + - HTTPException 500: If it fails due to internal server error. """ auth = Users() return auth.update_user(osm_id, update_data) # Delete user by osm_id -@router.delete("/users/{osm_id}", response_model=dict) -async def delete_user(osm_id: int, user_data: AuthUser = Depends(admin_required)): +@router.delete( + "/users/{osm_id}", + responses={ + **common_responses, + "200": {"content": {"application/json": {"example": {"osm_id": 1, "role": 1}}}}, + "404": {"model": ErrorMessage}, + }, +) +async def delete_user( + user_data: AuthUser = Depends(admin_required), + osm_id: int = Path(description="The OSM ID of the User to Delete"), +): """ Deletes a user based on the given osm_id. @@ -148,16 +216,29 @@ async def delete_user(osm_id: int, user_data: AuthUser = Depends(admin_required) - Dict[str, Any]: A dictionary containing the deleted user information. Raises: - - HTTPException: If the user with the given osm_id is not found. + - HTTPException 403: If the user has insufficient permission. + - HTTPException 404: If the user with the given osm_id is not found. + - HTTPException 500: If it fails due to internal server error. """ auth = Users() return auth.delete_user(osm_id) # Get all users -@router.get("/users/", response_model=list) +@router.get( + "/users", + response_model=list, + responses={ + **common_responses, + "200": { + "content": {"application/json": {"example": [{"osm_id": 1, "role": 2}]}} + }, + }, +) async def read_users( - skip: int = 0, limit: int = 10, user_data: AuthUser = Depends(staff_required) + skip: int = Query(0, description="The Number of Users to Skip"), + limit: int = Query(10, description="The Maximum Number of Users to Retrieve"), + user_data: AuthUser = Depends(staff_required), ): """ Retrieves a list of users with optional pagination. @@ -168,6 +249,9 @@ async def read_users( Returns: - List[Dict[str, Any]]: A list of dictionaries containing user information. + + - HTTPException 403: If it fails due to insufficient permission. + - HTTPException 500: If it fails due to internal server error. """ auth = Users() return auth.read_users(skip, limit) diff --git a/API/custom_exports.py b/API/custom_exports.py index 089317a6..9e21e776 100644 --- a/API/custom_exports.py +++ b/API/custom_exports.py @@ -1,6 +1,4 @@ # Standard library imports -import json -from typing import Dict # Third party imports import yaml @@ -13,7 +11,11 @@ from src.config import DEFAULT_QUEUE_NAME from src.config import LIMITER as limiter from src.config import RATE_LIMIT_PER_MIN -from src.validation.models import CustomRequestsYaml, DynamicCategoriesModel +from src.validation.models import ( + CustomRequestsYaml, + DynamicCategoriesModel, + common_responses, +) from .api_worker import process_custom_request from .auth import AuthUser, UserRole, staff_required @@ -21,7 +23,22 @@ router = APIRouter(prefix="/custom", tags=["Custom Exports"]) -@router.post("/snapshot/") +@router.post( + "/snapshot", + responses={ + **common_responses, + "200": { + "content": { + "application/json": { + "example": { + "task_id": "3fded368-456f-4ef4-a1b8-c099a7f77ca4", + "track_link": "/tasks/status/3fded368-456f-4ef4-a1b8-c099a7f77ca4/", + } + } + } + }, + }, +) @limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute") @version(1) async def process_custom_requests( @@ -827,7 +844,7 @@ async def process_custom_requests( @router.post( - "/snapshot/yaml/", + "/snapshot/yaml", openapi_extra={ "requestBody": { "content": { @@ -847,11 +864,13 @@ async def process_custom_requests_yaml( try: data = yaml.safe_load(raw_body) except yaml.YAMLError: - raise HTTPException(status_code=422, detail="Invalid YAML") + raise HTTPException(status_code=422, detail=[{"msg": "Invalid YAML"}]) try: validated_data = DynamicCategoriesModel.model_validate(data) except ValidationError as e: - raise HTTPException(status_code=422, detail=e.errors(include_url=False)) + raise HTTPException( + status_code=422, detail=[{"msg": e.errors(include_url=False)}] + ) queue_name = validated_data.queue if validated_data.queue != DEFAULT_QUEUE_NAME and user.role != UserRole.ADMIN.value: diff --git a/API/hdx.py b/API/hdx.py index b5d89a20..e0b47b72 100644 --- a/API/hdx.py +++ b/API/hdx.py @@ -1,6 +1,6 @@ from typing import Dict, List -from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Path from fastapi_versioning import version from src.app import HDX @@ -9,13 +9,20 @@ from .auth import AuthUser, admin_required, staff_required -# from src.validation.models import DynamicCategoriesModel +from src.validation.models import ErrorMessage, common_responses router = APIRouter(prefix="/hdx", tags=["HDX"]) -@router.post("/", response_model=dict) +@router.post( + "", + response_model=dict, + responses={ + "200": {"content": {"application/json": {"example": {"create": True}}}}, + **common_responses, + }, +) @limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute") @version(1) async def create_hdx( @@ -25,8 +32,8 @@ async def create_hdx( Create a new HDX entry. Args: - request (Request): The request object. - hdx_data (dict): Data for creating the HDX entry. + request (Request): The request object.\n + hdx_data (dict): Data for creating the HDX entry.\n user_data (AuthUser): User authentication data. Returns: @@ -36,13 +43,13 @@ async def create_hdx( return hdx_instance.create_hdx(hdx_data) -@router.get("/", response_model=List[dict]) +@router.get("", response_model=List[dict], responses={"500": {"model": ErrorMessage}}) @limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute") @version(1) async def read_hdx_list( request: Request, - skip: int = 0, - limit: int = 10, + skip: int = Query(0, description="Number of entries to skip."), + limit: int = Query(10, description="Maximum number of entries to retrieve."), ): """ Retrieve a list of HDX entries based on provided filters. @@ -65,12 +72,16 @@ async def read_hdx_list( filters[f"dataset->>'{key}' = %s"] = values try: hdx_list = hdx_instance.get_hdx_list_with_filters(skip, limit, filters) - except Exception as ex: - raise HTTPException(status_code=422, detail="Couldn't process query") + except Exception: + raise HTTPException(status_code=422, detail=[{"msg": "Couldn't process query"}]) return hdx_list -@router.get("/search/", response_model=List[dict]) +@router.get( + "/search", + response_model=List[dict], + responses={"404": {"model": ErrorMessage}, "500": {"model": ErrorMessage}}, +) @limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute") @version(1) async def search_hdx( @@ -98,10 +109,16 @@ async def search_hdx( return hdx_list -@router.get("/{hdx_id}", response_model=dict) +@router.get( + "/{hdx_id}", + response_model=dict, + responses={"404": {"model": ErrorMessage}, "500": {"model": ErrorMessage}}, +) @limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute") @version(1) -async def read_hdx(request: Request, hdx_id: int): +async def read_hdx( + request: Request, hdx_id: int = Path(description="ID of the HDX entry to retrieve") +): """ Retrieve a specific HDX entry by its ID. @@ -113,102 +130,116 @@ async def read_hdx(request: Request, hdx_id: int): dict: Details of the requested HDX entry. Raises: - HTTPException: If the HDX entry is not found. + HTTPException 404: If the HDX entry is not found. """ hdx_instance = HDX() hdx = hdx_instance.get_hdx_by_id(hdx_id) if hdx: return hdx - raise HTTPException(status_code=404, detail="HDX not found") + raise HTTPException(status_code=404, detail=[{"msg": "HDX not found"}]) -@router.put("/{hdx_id}", response_model=dict) +@router.put( + "/{hdx_id}", + response_model=dict, + responses={**common_responses, "404": {"model": ErrorMessage}}, +) @limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute") @version(1) async def update_hdx( request: Request, - hdx_id: int, hdx_data: dict, + hdx_id: int = Path(description="ID of the HDX entry to update"), user_data: AuthUser = Depends(staff_required), ): """ Update an existing HDX entry. Args: - request (Request): The request object. - hdx_id (int): ID of the HDX entry to update. - hdx_data (dict): Data for updating the HDX entry. + request (Request): The request object.\n + hdx_id (int): ID of the HDX entry to update.\n + hdx_data (dict): Data for updating the HDX entry.\n user_data (AuthUser): User authentication data. Returns: dict: Result of the HDX update process. Raises: - HTTPException: If the HDX entry is not found. + HTTPException 404: If the HDX entry is not found. """ hdx_instance = HDX() existing_hdx = hdx_instance.get_hdx_by_id(hdx_id) if not existing_hdx: - raise HTTPException(status_code=404, detail="HDX not found") + raise HTTPException(status_code=404, detail=[{"msg": "HDX not found"}]) hdx_instance_update = HDX() return hdx_instance_update.update_hdx(hdx_id, hdx_data) -@router.patch("/{hdx_id}", response_model=Dict) +@router.patch( + "/{hdx_id}", + response_model=Dict, + responses={**common_responses, "404": {"model": ErrorMessage}}, +) @limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute") @version(1) async def patch_hdx( request: Request, - hdx_id: int, hdx_data: Dict, + hdx_id: int = Path(description="ID of the HDX entry to update"), user_data: AuthUser = Depends(staff_required), ): """ Partially update an existing HDX entry. Args: - request (Request): The request object. - hdx_id (int): ID of the HDX entry to update. - hdx_data (Dict): Data for partially updating the HDX entry. + request (Request): The request object.\n + hdx_id (int): ID of the HDX entry to update.\n + hdx_data (Dict): Data for partially updating the HDX entry.\n user_data (AuthUser): User authentication data. Returns: Dict: Result of the HDX update process. Raises: - HTTPException: If the HDX entry is not found. + HTTPException 404: If the HDX entry is not found. """ hdx_instance = HDX() existing_hdx = hdx_instance.get_hdx_by_id(hdx_id) if not existing_hdx: - raise HTTPException(status_code=404, detail="HDX not found") + raise HTTPException(status_code=404, detail=[{"msg": "HDX not found"}]) patch_instance = HDX() return patch_instance.patch_hdx(hdx_id, hdx_data) -@router.delete("/{hdx_id}", response_model=dict) +@router.delete( + "/{hdx_id}", + response_model=dict, + responses={**common_responses, "404": {"model": ErrorMessage}}, +) @limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute") @version(1) async def delete_hdx( - request: Request, hdx_id: int, user_data: AuthUser = Depends(admin_required) + request: Request, + hdx_id: int = Path(description="ID of the HDX entry to delete"), + user_data: AuthUser = Depends(admin_required), ): """ Delete an existing HDX entry. Args: - request (Request): The request object. - hdx_id (int): ID of the HDX entry to delete. + request (Request): The request object.\n + hdx_id (int): ID of the HDX entry to delete.\n user_data (AuthUser): User authentication data. Returns: dict: Result of the HDX deletion process. Raises: - HTTPException: If the HDX entry is not found. + HTTPException 404: If the HDX entry is not found. """ hdx_instance = HDX() existing_hdx = hdx_instance.get_hdx_by_id(hdx_id) if not existing_hdx: - raise HTTPException(status_code=404, detail="HDX not found") + raise HTTPException(status_code=404, detail=[{"msg": "HDX not found"}]) return hdx_instance.delete_hdx(hdx_id) diff --git a/API/main.py b/API/main.py index e46b0b81..23b19560 100644 --- a/API/main.py +++ b/API/main.py @@ -84,7 +84,12 @@ os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1" -app = FastAPI(title="Raw Data API ", swagger_ui_parameters={"syntaxHighlight": False}) +app = FastAPI( + title="Raw Data API ", + description="""The Raw Data API allows you to transform + and export OpenStreetMap (OSM) data in different GIS file formats""", + swagger_ui_parameters={"syntaxHighlight": False}, +) app.include_router(auth_router) app.include_router(raw_data_router) app.include_router(tasks_router) diff --git a/API/raw_data.py b/API/raw_data.py index eaa8f3cf..b83ed1aa 100644 --- a/API/raw_data.py +++ b/API/raw_data.py @@ -17,8 +17,8 @@ # 1100 13th Street NW Suite 800 Washington, D.C. 20005 # -"""[Router Responsible for Raw data API ] -""" +"""[Router Responsible for Raw data API ]""" + # Standard library imports import json from typing import AsyncGenerator @@ -27,7 +27,7 @@ import orjson import redis from area import area -from fastapi import APIRouter, Body, Depends, HTTPException, Request +from fastapi import APIRouter, Body, Depends, HTTPException, Request, Path, Query from fastapi.responses import JSONResponse, StreamingResponse from fastapi_versioning import version @@ -47,6 +47,8 @@ RawDataCurrentParamsBase, SnapshotResponse, StatusResponse, + ErrorMessage, + common_responses, ) from .api_worker import process_raw_data @@ -57,15 +59,25 @@ redis_client = redis.StrictRedis.from_url(CELERY_BROKER_URL) -@router.get("/status/", response_model=StatusResponse) +@router.get( + "/status", response_model=StatusResponse, responses={"500": {"model": ErrorMessage}} +) @version(1) def check_database_last_updated(): - """Gives status about how recent the osm data is , it will give the last time that database was updated completely""" + """Gives status about how recent the osm data is. It will give the last time that database was updated completely""" result = RawData().check_status() return {"last_updated": result} -@router.post("/snapshot/", response_model=SnapshotResponse) +@router.post( + "/snapshot", + response_model=SnapshotResponse, + responses={ + **common_responses, + 404: {"model": ErrorMessage}, + 429: {"model": ErrorMessage}, + }, +) @limiter.limit(f"{export_rate_limit}/minute") @version(1) def get_osm_current_snapshot_as_file( @@ -407,7 +419,10 @@ def get_osm_current_snapshot_as_file( "task_id": "your task_id", "track_link": "/tasks/task_id/" } - 2. Now navigate to /tasks/ with your task id to track progress and result + 2. Now navigate to /tasks/ with your task id to track progress and result\n + + + Authentication is optional. If no token provided, it returns a user with limited options / guest user """ if not (user.role is UserRole.STAFF.value or user.role is UserRole.ADMIN.value): @@ -456,7 +471,7 @@ def get_osm_current_snapshot_as_file( status_code=403, detail=[ { - "msg": "Insufficient Permission for extracting exports with user metadata , Please login first" + "msg": "Insufficient Permission for extracting exports with user metadata, Please login first" } ], ) @@ -476,7 +491,9 @@ def get_osm_current_snapshot_as_file( ) -@router.post("/snapshot/plain/") +@router.post( + "/snapshot/plain", responses={**common_responses, 404: {"model": ErrorMessage}} +) @version(1) async def get_osm_current_snapshot_as_plain_geojson( request: Request, @@ -490,7 +507,9 @@ async def get_osm_current_snapshot_as_plain_geojson( params (RawDataCurrentParamsBase): Same as /snapshot except multiple output format options and configurations Returns: - FeatureCollection: Geojson + FeatureCollection: Geojson\n + + Authentication is optional. If no token provided, it returns a user with limited options / guest user """ if user.id == 0 and params.include_user_metadata: raise HTTPException( @@ -544,21 +563,42 @@ async def generate_geojson() -> AsyncGenerator[bytes, None]: return StreamingResponse(generate_geojson(), media_type="application/geo+json") -@router.get("/countries/") +@router.get("/countries", responses={"500": {"model": ErrorMessage}}) @version(1) -def get_countries(q: str = ""): +def get_countries( + q: str = Query("", description="Query parameter for filtering countries"), +): + """ + Gets Countries list from the database + Args: + q (str): query parameter for filtering countries + Returns: + featurecollection: geojson of country + """ + result = RawData().get_countries_list(q) return result -@router.get("/countries/{cid}/") +@router.get("/countries/{cid}") @version(1) def get_specific_country(cid: int): result = RawData().get_country(cid) return result -@router.get("/osm_id/") +@router.get( + "/osm_id", + responses={"404": {"model": ErrorMessage}, "500": {"model": ErrorMessage}}, +) @version(1) -def get_osm_feature(osm_id: int): +def get_osm_feature(osm_id: int = Path(description="The OSM ID of feature")): + """ + Gets geometry of osm_id in geojson + Args: + osm_id (int): osm_id of feature + Returns: + featurecollection: Geojson + """ + return RawData().get_osm_feature(osm_id) diff --git a/API/s3.py b/API/s3.py index 767f2952..46741f68 100644 --- a/API/s3.py +++ b/API/s3.py @@ -1,11 +1,10 @@ import json from urllib.parse import quote -import boto3 import humanize from boto3.session import Session from botocore.exceptions import NoCredentialsError -from fastapi import APIRouter, Header, HTTPException, Path, Query, Request +from fastapi import APIRouter, HTTPException, Path, Query, Request from fastapi.encoders import jsonable_encoder from fastapi.responses import ( JSONResponse, @@ -18,6 +17,7 @@ from src.config import AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME from src.config import LIMITER as limiter from src.config import RATE_LIMIT_PER_MIN +from src.validation.models import ErrorMessage router = APIRouter(prefix="/s3", tags=["S3"]) @@ -32,12 +32,14 @@ paginator = s3.get_paginator("list_objects_v2") -@router.get("/files/") +@router.get( + "/files", responses={"404": {"model": ErrorMessage}, "500": {"model": ErrorMessage}} +) @limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute") @version(1) async def list_s3_files( request: Request, - folder: str = Query(default="/HDX"), + folder: str = Query("/HDX", description="Folder in S3"), prettify: bool = Query( default=False, description="Display size & date in human-readable format" ), @@ -80,7 +82,9 @@ async def generate(): return StreamingResponse(content=generate(), media_type="application/json") except NoCredentialsError: - raise HTTPException(status_code=500, detail="AWS credentials not available") + raise HTTPException( + status_code=500, detail=[{"msg": "AWS credentials not available"}] + ) async def check_object_existence(bucket_name, file_path): @@ -88,10 +92,12 @@ async def check_object_existence(bucket_name, file_path): try: s3.head_object(Bucket=bucket_name, Key=file_path) except NoCredentialsError: - raise HTTPException(status_code=500, detail="AWS credentials not available") - except Exception as e: raise HTTPException( - status_code=404, detail=f"File or folder not found: {file_path}" + status_code=500, detail=[{"msg": "AWS credentials not available"}] + ) + except Exception: + raise HTTPException( + status_code=404, detail=[{"msg": f"File or folder not found: {file_path}"}] ) @@ -103,11 +109,14 @@ async def read_meta_json(bucket_name, file_path): return content except Exception as e: raise HTTPException( - status_code=500, detail=f"Error reading meta.json: {str(e)}" + status_code=500, detail=[{"msg": f"Error reading meta.json: {str(e)}"}] ) -@router.head("/get/{file_path:path}") +@router.head( + "/get/{file_path:path}", + responses={"404": {"model": ErrorMessage}, "500": {"model": ErrorMessage}}, +) @limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute") @version(1) async def head_s3_file( @@ -131,10 +140,15 @@ async def head_s3_file( if e.response["Error"]["Code"] == "404": return Response(status_code=404) else: - raise HTTPException(status_code=500, detail=f"AWS Error: {str(e)}") + raise HTTPException( + status_code=500, detail=[{"msg": f"AWS Error: {str(e)}"}] + ) -@router.get("/get/{file_path:path}") +@router.get( + "/get/{file_path:path}", + responses={"404": {"model": ErrorMessage}, "500": {"model": ErrorMessage}}, +) @limiter.limit(f"{RATE_LIMIT_PER_MIN}/minute") @version(1) async def get_s3_file( diff --git a/API/stats.py b/API/stats.py index bb1522ec..1536b041 100644 --- a/API/stats.py +++ b/API/stats.py @@ -5,18 +5,21 @@ from area import area from fastapi import APIRouter, Body, Depends, HTTPException, Request from fastapi_versioning import version +from .auth import AuthUser, UserRole, get_optional_user # Reader imports from src.app import PolygonStats from src.config import LIMITER as limiter from src.config import POLYGON_STATISTICS_API_RATE_LIMIT -from src.validation.models import StatsRequestParams +from src.validation.models import StatsRequestParams, stats_response router = APIRouter(prefix="/stats", tags=["Stats"]) -from .auth import AuthUser, UserRole, get_optional_user -@router.post("/polygon/") +@router.post( + "/polygon", + responses={**stats_response}, +) @limiter.limit(f"{POLYGON_STATISTICS_API_RATE_LIMIT}/minute") @version(1) async def get_polygon_stats( diff --git a/API/tasks.py b/API/tasks.py index 9d5295ee..5b049add 100644 --- a/API/tasks.py +++ b/API/tasks.py @@ -5,24 +5,28 @@ # Third party imports import redis from celery.result import AsyncResult -from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Path from fastapi.responses import JSONResponse from fastapi_versioning import version # Reader imports from src.config import CELERY_BROKER_URL, DEFAULT_QUEUE_NAME, ONDEMAND_QUEUE_NAME -from src.validation.models import SnapshotTaskResponse +from src.validation.models import SnapshotTaskResponse, ErrorMessage, common_responses from .api_worker import celery -from .auth import AuthUser, admin_required, login_required, staff_required +from .auth import AuthUser, admin_required, staff_required router = APIRouter(prefix="/tasks", tags=["Tasks"]) -@router.get("/status/{task_id}/", response_model=SnapshotTaskResponse) +@router.get( + "/status/{task_id}", + response_model=SnapshotTaskResponse, + responses={"404": {"model": ErrorMessage}, "500": {"model": ErrorMessage}}, +) @version(1) def get_task_status( - task_id, + task_id=Path(description="Unique id provided on response from */snapshot/*"), only_args: bool = Query( default=False, description="Fetches arguments of task", @@ -82,10 +86,26 @@ def get_task_status( return JSONResponse(result) -@router.get("/revoke/{task_id}/") +@router.get( + "/revoke/{task_id}", + responses={ + **common_responses, + "404": {"model": ErrorMessage}, + "200": { + "content": { + "application/json": { + "example": {"id": "aa539af6-83d4-4aa3-879e-abf14fffa03f"} + } + } + }, + }, +) @version(1) -def revoke_task(task_id, user: AuthUser = Depends(staff_required)): - """Revokes task , Terminates if it is executing +def revoke_task( + task_id=Path(description="Unique id provided on response from */snapshot*"), + user: AuthUser = Depends(staff_required), +): + """Revokes task, Terminates if it is executing Args: task_id (_type_): task id of raw data task @@ -97,7 +117,19 @@ def revoke_task(task_id, user: AuthUser = Depends(staff_required)): return JSONResponse({"id": task_id}) -@router.get("/inspect/") +@router.get( + "/inspect", + responses={ + "500": {"model": ErrorMessage}, + "200": { + "content": { + "application/json": { + "example": {"active": [{"celery@default_worker": {}}]} + } + } + }, + }, +) @version(1) def inspect_workers( request: Request, @@ -138,7 +170,19 @@ def inspect_workers( return JSONResponse(content=response_data) -@router.get("/ping/") +@router.get( + "/ping", + responses={ + "500": {"model": ErrorMessage}, + "200": { + "content": { + "application/json": { + "example": {"celery@default_worker": {"ok": "pong"}} + } + } + }, + }, +) @version(1) def ping_workers(): """Pings available workers @@ -149,12 +193,22 @@ def ping_workers(): return JSONResponse(inspected_ping) -@router.get("/purge/") +@router.get( + "/purge", + responses={ + **common_responses, + "200": {"content": {"application/json": {"example": {"tasks_discarded": 0}}}}, + }, +) @version(1) def discard_all_waiting_tasks(user: AuthUser = Depends(admin_required)): """ Discards all waiting tasks from the queue + Returns : Number of tasks discarded + + Raises: + - HTTPException 403: If purge fails due to insufficient permission. """ purged = celery.control.purge() return JSONResponse({"tasks_discarded": purged}) @@ -163,9 +217,22 @@ def discard_all_waiting_tasks(user: AuthUser = Depends(admin_required)): queues = [DEFAULT_QUEUE_NAME, ONDEMAND_QUEUE_NAME] -@router.get("/queue/") +@router.get( + "/queue", + responses={ + "500": {"model": ErrorMessage}, + "200": { + "content": {"application/json": {"example": {"raw_daemon": {"length": 0}}}} + }, + }, +) @version(1) def get_queue_info(): + """ + Get all the queues + + Returns : The queues names and their lengths + """ queue_info = {} redis_client = redis.StrictRedis.from_url(CELERY_BROKER_URL) @@ -180,17 +247,31 @@ def get_queue_info(): return JSONResponse(content=queue_info) -@router.get("/queue/details/{queue_name}/") +@router.get( + "/queue/details/{queue_name}", + responses={**common_responses, "404": {"model": ErrorMessage}}, +) @version(1) def get_list_details( - queue_name: str, + queue_name=Path(description="Name of queue to retrieve"), args: bool = Query( default=False, description="Includes arguments of task", ), ): + """ + Retrieves queue information based on the given queue name + + Args: + - queue_name (str): The name of the queue to retrieve. + + Returns : The queue details + """ + if queue_name not in queues: - raise HTTPException(status_code=404, detail=f"Queue '{queue_name}' not found") + raise HTTPException( + status_code=404, detail=[{"msg": f"Queue '{queue_name}' not found"}] + ) redis_client = redis.StrictRedis.from_url(CELERY_BROKER_URL) list_items = redis_client.lrange(queue_name, 0, -1) diff --git a/src/config.py b/src/config.py index 78bb17e8..d71e7ee8 100644 --- a/src/config.py +++ b/src/config.py @@ -32,7 +32,17 @@ def get_bool_env_var(key, default=False): value = os.environ.get(key, default) - return bool(strtobool(str(value))) + try: + return bool(strtobool(str(value))) + except Exception: + logging.warn(f"{value} for {key} is not valid") + return False + + +def parse_list(value, delimiter=","): + if isinstance(value, str): + return value.split(delimiter) + return value or [] CONFIG_FILE_PATH = "config.txt" @@ -341,10 +351,6 @@ def not_raises(func, *args, **kwargs): from hdx.data.dataset import Dataset from hdx.data.vocabulary import Vocabulary - parse_list = lambda value, delimiter=",": ( - value.split(delimiter) if isinstance(value, str) else value or [] - ) - ALLOWED_HDX_TAGS = parse_list( os.environ.get("ALLOWED_HDX_TAGS") or config.get("HDX", "ALLOWED_HDX_TAGS", fallback=None) diff --git a/src/validation/models.py b/src/validation/models.py index 1b1b0e92..61c1da59 100644 --- a/src/validation/models.py +++ b/src/validation/models.py @@ -17,6 +17,7 @@ # 1100 13th Street NW Suite 800 Washington, D.C. 20005 # """Page contains validation models for application""" + # Standard library imports from enum import Enum from typing import Dict, List, Optional, Union @@ -229,14 +230,12 @@ def check_bind_option(cls, value, values): """Checks if cloud optimized output format or geoJSON is selected along with bind to zip file""" if value is False: if values.get("output_type") not in ( - ( - [ - RawDataOutputType.GEOJSON.value, - RawDataOutputType.FLATGEOBUF.value, - RawDataOutputType.GEOPARQUET.value, - ] - + ([RawDataOutputType.PMTILES.value] if ENABLE_TILES else []) - ) + [ + RawDataOutputType.GEOJSON.value, + RawDataOutputType.FLATGEOBUF.value, + RawDataOutputType.GEOPARQUET.value, + ] + + ([RawDataOutputType.PMTILES.value] if ENABLE_TILES else []) ): raise ValueError( "Only Cloud Optimized format and GeoJSON is supported for streaming" @@ -303,22 +302,22 @@ class StatsRequestParams(BaseModel, GeometryValidatorMixin): max_length=3, example="NPL", ) - geometry: Optional[ - Union[Polygon, MultiPolygon, Feature, FeatureCollection] - ] = Field( - default=None, - example={ - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, + geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = ( + Field( + default=None, + example={ + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + ) ) @validator("geometry", pre=True, always=True) @@ -624,22 +623,22 @@ class DynamicCategoriesModel(CategoriesBase, GeometryValidatorMixin): max_length=3, example="USA", ) - geometry: Optional[ - Union[Polygon, MultiPolygon, Feature, FeatureCollection] - ] = Field( - default=None, - example={ - "type": "Polygon", - "coordinates": [ - [ - [83.96919250488281, 28.194446860487773], - [83.99751663208006, 28.194446860487773], - [83.99751663208006, 28.214869548073377], - [83.96919250488281, 28.214869548073377], - [83.96919250488281, 28.194446860487773], - ] - ], - }, + geometry: Optional[Union[Polygon, MultiPolygon, Feature, FeatureCollection]] = ( + Field( + default=None, + example={ + "type": "Polygon", + "coordinates": [ + [ + [83.96919250488281, 28.194446860487773], + [83.99751663208006, 28.194446860487773], + [83.99751663208006, 28.214869548073377], + [83.96919250488281, 28.214869548073377], + [83.96919250488281, 28.194446860487773], + ] + ], + }, + ) ) @validator("geometry", pre=True, always=True) @@ -682,3 +681,64 @@ class CustomRequestsYaml(CategoriesBase): ], }, ) + + +class ErrorDetail(BaseModel): + msg: str + + +class ErrorMessage(BaseModel): + detail: List[ErrorDetail] + + +common_responses = { + 401: { + "model": ErrorMessage, + "content": { + "application/json": { + "example": {"detail": [{"msg": "User is not an admin"}]} + } + }, + }, + 403: { + "model": ErrorMessage, + "content": { + "application/json": { + "example": {"detail": [{"msg": "OSM Authentication failed"}]} + } + }, + }, + 500: {"model": ErrorMessage}, +} + +stats_response = { + "200": { + "content": { + "application/json": { + "example": { + "summary": {"buildings": "", "roads": ""}, + "raw": { + "population": 0, + "populatedAreaKm2": 0, + "averageEditTime": 0, + "lastEditTime": 0, + "osmUsersCount": 0, + "osmBuildingCompletenessPercentage": 0, + "osmRoadsCompletenessPercentage": 0, + "osmBuildingsCount": 0, + "osmHighwayLengthKm": 0, + "aiBuildingsCountEstimation": 0, + "aiRoadCountEstimationKm": 0, + "buildingCount6Months": 0, + "highwayLength6MonthsKm": 0, + }, + "meta": { + "indicators": "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md", + "metrics": "https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/metrics.md", + }, + } + } + } + }, + "500": {"model": ErrorMessage}, +}