From f875238130becc3fcf28fab5c65feecd011730fa Mon Sep 17 00:00:00 2001 From: florianpix Date: Thu, 26 Oct 2023 15:24:12 +0200 Subject: [PATCH] switched to gridFS API for media at climbingAPI; --- src/climbingAPI/app/routers/admin.py | 35 ++++++++---- src/climbingAPI/app/routers/media.py | 79 +++++++++++++++++----------- 2 files changed, 74 insertions(+), 40 deletions(-) diff --git a/src/climbingAPI/app/routers/admin.py b/src/climbingAPI/app/routers/admin.py index 96577e7..6a92d14 100644 --- a/src/climbingAPI/app/routers/admin.py +++ b/src/climbingAPI/app/routers/admin.py @@ -1,11 +1,14 @@ import datetime +import bson from fastapi import APIRouter, Depends, Security from fastapi_auth0 import Auth0User -from app.core.db import get_db +from app.core.db import get_db, get_db_client from app.core.auth import auth +from motor.motor_asyncio import AsyncIOMotorGridFSBucket + router = APIRouter() collection_names = ["trip", "spot", "single_pitch_route", "multi_pitch_route", "pitch", "ascent", "medium"] @@ -71,45 +74,57 @@ async def migrate_add_updated(user: Auth0User = Security(auth.get_user, scopes=[ await db[collection_name].update_many({"user_id": user.id}, {"$set": {"updated": datetime.datetime.now()}}) -@router.delete('/', description="Delete everything from this user", dependencies=[Depends(auth.implicit_scheme)]) +@router.delete('/', description="Delete everything of this user", dependencies=[Depends(auth.implicit_scheme)]) async def delete_all(user: Auth0User = Security(auth.get_user, scopes=["write:diary"])): + # delete all data db = await get_db() for collection_name in collection_names: await db[collection_name].delete_many({"user_id": user.id}) + # delete all media + client = await get_db_client() # AsyncIOMotorClient + bucket = AsyncIOMotorGridFSBucket(client.get_database('fs')) # AsyncIOMotorGridFSBucket + meta_media = await bucket.find().to_list(None) + for meta_medium in meta_media: + gridOut = await bucket.open_download_stream(meta_medium['_id']) # AsyncIOMotorGridOut + medium = bson.decode(await gridOut.read()) + if medium['user_id'] == user.id: await bucket.delete(medium['_id']) -@router.delete('/trips', description="Delete all trips from all users", dependencies=[Depends(auth.implicit_scheme)]) +@router.delete('/trips', description="Delete all trips of this user", dependencies=[Depends(auth.implicit_scheme)]) async def delete_trips(user: Auth0User = Security(auth.get_user, scopes=["write:diary"])): db = await get_db() delete_result = await db["trip"].delete_many({"user_id": user.id}) -@router.delete('/spots', description="Delete all spots from all users", dependencies=[Depends(auth.implicit_scheme)]) +@router.delete('/spots', description="Delete all spots of this user", dependencies=[Depends(auth.implicit_scheme)]) async def delete_spots(user: Auth0User = Security(auth.get_user, scopes=["write:diary"])): db = await get_db() delete_result = await db["spot"].delete_many({"user_id": user.id}) -@router.delete('/routes', description="Delete all routes from all users", dependencies=[Depends(auth.implicit_scheme)]) +@router.delete('/routes', description="Delete all routes of this user", dependencies=[Depends(auth.implicit_scheme)]) async def delete_routes(user: Auth0User = Security(auth.get_user, scopes=["write:diary"])): db = await get_db() delete_result = await db["single_pitch_route"].delete_many({"user_id": user.id}) delete_result = await db["multi_pitch_route"].delete_many({"user_id": user.id}) -@router.delete('/pitches', description="Delete all pitches from all users", dependencies=[Depends(auth.implicit_scheme)]) +@router.delete('/pitches', description="Delete all pitches of this user", dependencies=[Depends(auth.implicit_scheme)]) async def delete_pitches(user: Auth0User = Security(auth.get_user, scopes=["write:diary"])): db = await get_db() delete_result = await db["pitch"].delete_many({"user_id": user.id}) -@router.delete('/ascents', description="Delete all ascents from all users", dependencies=[Depends(auth.implicit_scheme)]) +@router.delete('/ascents', description="Delete all ascents of this user", dependencies=[Depends(auth.implicit_scheme)]) async def delete_ascents(user: Auth0User = Security(auth.get_user, scopes=["write:diary"])): db = await get_db() delete_result = await db["ascent"].delete_many({"user_id": user.id}) -@router.delete('/media', description="Delete all media from all users", dependencies=[Depends(auth.implicit_scheme)]) +@router.delete('/media', description="Delete all media of this user", dependencies=[Depends(auth.implicit_scheme)]) async def delete_media(user: Auth0User = Security(auth.get_user, scopes=["write:diary"])): - db = await get_db() - delete_result = await db["medium"].delete_many({"user_id": user.id}) + client = await get_db_client() # AsyncIOMotorClient + bucket = AsyncIOMotorGridFSBucket(client.get_database('fs')) # AsyncIOMotorGridFSBucket + media = await bucket.find({"user_id": user.id}).to_list(None) + for medium in media: + await bucket.delete(medium['_id']) diff --git a/src/climbingAPI/app/routers/media.py b/src/climbingAPI/app/routers/media.py index b6825f0..0caaedd 100644 --- a/src/climbingAPI/app/routers/media.py +++ b/src/climbingAPI/app/routers/media.py @@ -1,52 +1,64 @@ -import datetime -from typing import List, Tuple +from typing import List from fastapi import APIRouter, Body, Depends, HTTPException, Security, status from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse from fastapi_auth0 import Auth0User -from app.core.db import get_db +from app.core.db import get_db, get_db_client from app.core.auth import auth from app.models.medium.medium_model import MediumModel from app.models.medium.small_medium_model import SmallMediumModel +from motor.motor_asyncio import AsyncIOMotorGridFSBucket + +import bson +import gridfs + router = APIRouter() +@router.post('test', description="test", dependencies=[Depends(auth.implicit_scheme)]) +async def test(): + client = await get_db_client() # AsyncIOMotorClient + await client.drop_database('fs') # MotorDatabase + + @router.post('', description="Add a new medium", response_model=MediumModel, dependencies=[Depends(auth.implicit_scheme)]) async def create_medium(medium: MediumModel = Body(...), user: Auth0User = Security(auth.get_user, scopes=["write:diary"])): medium = jsonable_encoder(medium) medium["user_id"] = user.id - db = await get_db() - # check if medium already exists - if (media := await db["medium"].find({ - "title": medium["title"], - "user_id": user.id - }).to_list(None)): - raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Medium already exists") - new_medium = await db["medium"].insert_one(medium) - created_medium = await db["medium"].find_one({"_id": new_medium.inserted_id}) + client = await get_db_client() # AsyncIOMotorClient + bucket = AsyncIOMotorGridFSBucket(client.get_database('fs')) # AsyncIOMotorGridFSBucket + gridIn = bucket.open_upload_stream_with_id(file_id=medium['_id'], filename=medium['_id']) # AsyncIOMotorGridIn + try: + await gridIn.write(bson.encode(medium)) + await gridIn.close() + except gridfs.errors.FileExists: + print(f"medium {medium['_id']} {medium['title']} already exists") + gridOut = await bucket.open_download_stream(medium['_id']) # AsyncIOMotorGridOut + created_medium = bson.decode(await gridOut.read()) return JSONResponse(status_code=status.HTTP_201_CREATED, content=jsonable_encoder(MediumModel(**created_medium))) @router.get('/{medium_id}', description="Get a medium", response_model=MediumModel, dependencies=[Depends(auth.implicit_scheme)]) async def retrieve_medium_of_id(medium_id: str, user: Auth0User = Security(auth.get_user, scopes=["read:diary"])): - db = await get_db() - if (medium := await db["medium"].find_one({"_id": medium_id, "user_id": user.id})) is not None: - return medium - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Medium {medium_id} not found") + client = await get_db_client() # AsyncIOMotorClient + bucket = AsyncIOMotorGridFSBucket(client.get_database('fs')) # AsyncIOMotorGridFSBucket + gridOut = await bucket.open_download_stream(medium_id) # AsyncIOMotorGridOut + return MediumModel(**bson.decode(await gridOut.read())) @router.post('/ids', description="Get media of ids", response_model=List[MediumModel], dependencies=[Depends(auth.implicit_scheme)]) async def retrieve_media_of_ids(medium_ids: List[str] = Body(...), user: Auth0User = Security(auth.get_user, scopes=["read:diary"])): if not medium_ids: return [] - db = await get_db() + client = await get_db_client() # AsyncIOMotorClient + bucket = AsyncIOMotorGridFSBucket(client.get_database('fs')) # AsyncIOMotorGridFSBucket media = [] for medium_id in medium_ids: - if (medium := await db["medium"].find_one({"_id": medium_id, "user_id": user.id})) is not None: + if (medium := await bucket.find({"_id": medium_id})) is not None: media.append(medium) else: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Medium {medium_id} not found") @@ -57,25 +69,32 @@ async def retrieve_media_of_ids(medium_ids: List[str] = Body(...), user: Auth0Us @router.get('-small', description="Retrieve all media without the actual image", response_model=List[SmallMediumModel], dependencies=[Depends(auth.implicit_scheme)]) async def retrieve_all_media_small(user: Auth0User = Security(auth.get_user, scopes=["read:diary"])): - db = await get_db() - media = await db["medium"].find({"user_id": user.id}, {"image": 0}).to_list(None) + client = await get_db_client() # AsyncIOMotorClient + bucket = AsyncIOMotorGridFSBucket(client.get_database('fs')) # AsyncIOMotorGridFSBucket + meta_media = await bucket.find().to_list(None) + media = [] + for meta_medium in meta_media: + gridOut = await bucket.open_download_stream(meta_medium['_id']) # AsyncIOMotorGridOut + medium = SmallMediumModel(**bson.decode(await gridOut.read())) + if medium.user_id == user.id: media.append(medium) return media @router.get('', description="Retrieve all media", response_model=List[MediumModel], dependencies=[Depends(auth.implicit_scheme)]) async def retrieve_all_media(user: Auth0User = Security(auth.get_user, scopes=["read:diary"])): - db = await get_db() - media = await db["medium"].find({"user_id": user.id}).to_list(None) + client = await get_db_client() # AsyncIOMotorClient + bucket = AsyncIOMotorGridFSBucket(client.get_database('fs')) # AsyncIOMotorGridFSBucket + meta_media = await bucket.find().to_list(None) + media = [] + for meta_medium in meta_media: + gridOut = await bucket.open_download_stream(meta_medium['_id']) # AsyncIOMotorGridOut + medium = MediumModel(**bson.decode(await gridOut.read())) + if medium.user_id == user.id: media.append(medium) return media @router.delete('/{medium_id}', description="Delete a medium", response_model=MediumModel, dependencies=[Depends(auth.implicit_scheme)]) async def delete_medium(medium_id: str, user: Auth0User = Security(auth.get_user, scopes=["write:diary"])): - db = await get_db() - medium = await db["medium"].find_one({"_id": medium_id, "user_id": user.id}) - if medium is None: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Medium {medium_id} not found") - delete_result = await db["medium"].delete_one({"_id": medium_id}) - if delete_result.deleted_count == 1: - return medium - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Medium {medium_id} not found") + client = await get_db_client() # AsyncIOMotorClient + bucket = AsyncIOMotorGridFSBucket(client.get_database('fs')) # AsyncIOMotorGridFSBucket + await bucket.delete(medium_id)