Wasabi S3 integration for fetching .osu files (#9)

* Wasabi S3 integration for .osu files

* add env example

.env.example

@@ -17,4 +17,10 @@ DB_HOST=localhost
 DB_PORT=3306
 DB_NAME=akatsuki
 
+AWS_S3_ENDPOINT_URL=
+AWS_S3_REGION_NAME=
+AWS_S3_BUCKET_NAME=
+AWS_S3_ACCESS_KEY_ID=
+AWS_S3_SECRET_ACCESS_KEY=
+
 DISCORD_BEATMAP_UPDATES_WEBHOOK_URL=

app/adapters/aws_s3.py

@@ -0,0 +1,37 @@
+import logging
+
+from app import settings
+from app import state
+
+
+async def get_object_data(key: str) -> bytes | None:
+    try:
+        s3_object = await state.s3_client.get_object(
+            Bucket=settings.AWS_S3_BUCKET_NAME,
+            Key=key,
+        )
+    except Exception:
+        logging.warning(
+            "Failed to get object data from S3",
+            exc_info=True,
+            extra={"object_key": key},
+        )
+        return None
+
+    return await s3_object["Body"].read()
+
+
+async def save_object_data(key: str, data: bytes) -> None:
+    try:
+        await state.s3_client.put_object(
+            Bucket=settings.AWS_S3_BUCKET_NAME,
+            Key=key,
+            Body=data,
+        )
+    except Exception:
+        logging.warning(
+            "Failed to save object data to S3",
+            exc_info=True,
+            extra={"object_key": key},
+        )
+        return None

app/api/internal/v1/osu_api_v1.py

@@ -1,18 +1,24 @@
 from fastapi import APIRouter
 from fastapi import Response
 
+from app.adapters import aws_s3
 from app.adapters import osu_api_v1
 
 router = APIRouter(tags=["osu Files"])
 
 
 @router.get("/api/osu-api/v1/osu-files/{beatmap_id}")
 async def download_beatmap_osu_file(beatmap_id: int) -> Response:
-    beatmap_osu_file_data = await osu_api_v1.fetch_beatmap_osu_file_data(beatmap_id)
-    # TODO: consider at which points in beatmaps-service we should update
-    #       the .osu file that is currently saved in wasabi s3 storage.
+    beatmap_osu_file_data = await aws_s3.get_object_data(f"/beatmaps/{beatmap_id}.osu")
     if beatmap_osu_file_data is None:
-        return Response(status_code=404)
+        beatmap_osu_file_data = await osu_api_v1.fetch_beatmap_osu_file_data(beatmap_id)
+        # TODO: consider at which points in beatmaps-service we should update
+        #       the .osu file that is currently saved in wasabi s3 storage.
+        if beatmap_osu_file_data is None:
+            return Response(status_code=404)
+    else:
+        # TODO: consider cache expiry
+        ...
 
     return Response(
         beatmap_osu_file_data,

app/init_api.py

@@ -2,6 +2,7 @@
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
 
+import aiobotocore.session
 from databases import Database
 from fastapi import FastAPI
 from fastapi import Request
@@ -19,7 +20,19 @@
 async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     logger.configure_logging()
     await state.database.connect()
+
+    aws_session = aiobotocore.session.get_session()
+    s3_client = aws_session.create_client(
+        service_name="s3",
+        region_name=settings.AWS_S3_REGION_NAME,
+        endpoint_url=settings.AWS_S3_ENDPOINT_URL,
+        aws_access_key_id=settings.AWS_S3_ACCESS_KEY_ID,
+        aws_secret_access_key=settings.AWS_S3_SECRET_ACCESS_KEY,
+    )
+    state.s3_client = await s3_client.__aenter__()
+
     yield
+    await state.s3_client.__aexit__(None, None, None)
     await state.database.disconnect()
 
 

app/settings.py

@@ -26,4 +26,10 @@ def read_bool(s: str) -> bool:
 DB_PORT = int(os.environ["DB_PORT"])
 DB_NAME = os.environ["DB_NAME"]
 
+AWS_S3_ENDPOINT_URL = os.environ["AWS_S3_ENDPOINT_URL"]
+AWS_S3_REGION_NAME = os.environ["AWS_S3_REGION_NAME"]
+AWS_S3_BUCKET_NAME = os.environ["AWS_S3_BUCKET_NAME"]
+AWS_S3_ACCESS_KEY_ID = os.environ["AWS_S3_ACCESS_KEY_ID"]
+AWS_S3_SECRET_ACCESS_KEY = os.environ["AWS_S3_SECRET_ACCESS_KEY"]
+
 DISCORD_BEATMAP_UPDATES_WEBHOOK_URL = os.environ["DISCORD_BEATMAP_UPDATES_WEBHOOK_URL"]

app/state.py

@@ -1,3 +1,8 @@
-from databases import Database
+from typing import TYPE_CHECKING
 
-database: Database
+if TYPE_CHECKING:
+    from databases import Database
+    from types_aiobotocore_s3.client import S3Client
+
+database: "Database"
+s3_client: "S3Client"

requirements-dev.txt

@@ -1,3 +1,4 @@
 mypy
+types-aiobotocore[s3]
 types-jmespath
 types-pyyaml

requirements.txt

@@ -1,3 +1,4 @@
+aiobotocore
 cryptography
 databases[aiomysql]
 fastapi