CI deploy and general infra improvements

.env.example

@@ -1,2 +1,10 @@
+APP_ENV=
+APP_HOST=
+APP_PORT=
+
+SERVICE_READINESS_TIMEOUT=60
+
+CODE_HOTRELOAD=
+
 OSU_API_V2_CLIENT_ID=
 OSU_API_V2_CLIENT_SECRET=

.github/workflows/production-deploy.yml

@@ -0,0 +1,89 @@
+name: production-deploy
+
+on:
+  push:
+    branches:
+      - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  production-deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out latest commit
+        uses: actions/checkout@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: osuAkatsuki/beatmaps-service
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/beatmaps-service:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/beatmaps-service:${{ github.sha }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Get kubeconfig from github secrets
+        run: |
+            mkdir -p $HOME/.kube
+            echo "${{ secrets.KUBECONFIG }}" > $HOME/.kube/config
+            sudo chown $(id -u):$(id -g) $HOME/.kube/config
+            chmod 600 $HOME/.kube/config
+
+      - name: Install helm
+        uses: azure/setup-helm@v3
+        with:
+          version: "latest"
+          token: ${{ secrets.GITHUB_TOKEN }}
+        id: install
+
+      - name: Install helm-diff
+        run: helm plugin install https://github.com/databus23/helm-diff
+
+      - name: Checkout common-helm-charts repo
+        uses: actions/checkout@v3
+        with:
+          repository: osuAkatsuki/common-helm-charts
+          token: ${{ secrets.COMMON_HELM_CHARTS_PAT_2024 }}
+          path: common-helm-charts
+
+      - name: Clear pending deployments
+        run: |
+          kubectl delete secret -l 'status in (pending-install, pending-upgrade, pending-rollback),name=beatmaps-service-production'
+
+      - name: Show manifest diff since previous release
+        run: |
+          helm diff upgrade \
+          --allow-unreleased \
+          --color=true \
+          --values chart/values.yaml \
+          beatmaps-service-production \
+          common-helm-charts/microservice-base/
+
+      - name: Deploy service to production cluster
+        run: |
+          helm upgrade \
+            --install \
+            --atomic \
+            --wait --timeout 10m \
+            --cleanup-on-fail \
+            --values chart/values.yaml \
+            beatmaps-service-production \
+            common-helm-charts/microservice-base/

.pre-commit-config.yaml

@@ -0,0 +1,39 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    -   id: check-ast
+    -   id: check-builtin-literals
+    -   id: check-yaml
+    -   id: debug-statements
+    -   id: end-of-file-fixer
+    -   id: requirements-txt-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 24.4.2
+    hooks:
+    -   id: black
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.2
+    hooks:
+    -   id: pyupgrade
+        args: [--py311-plus, --keep-runtime-typing]
+-   repo: https://github.com/asottile/reorder_python_imports
+    rev: v3.12.0
+    hooks:
+    -   id: reorder-python-imports
+        args: [--py311-plus]
+-   repo: https://github.com/asottile/add-trailing-comma
+    rev: v3.1.0
+    hooks:
+    -   id: add-trailing-comma
+-   repo: https://github.com/asottile/blacken-docs
+    rev: 1.16.0
+    hooks:
+    -   id: blacken-docs
+        additional_dependencies: [black==24.4.2]
+
+default_language_version:
+    python: python3.11

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.11
+
+ENV PYTHONUNBUFFERED=1
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+RUN pip install git+https://github.com/osuAkatsuki/akatsuki-cli
+
+COPY scripts /scripts
+
+COPY . /srv/root
+WORKDIR /srv/root
+
+ENTRYPOINT ["/scripts/bootstrap.sh"]

app/adapters/osu_api_v2.py

@@ -1,8 +1,11 @@
 from datetime import datetime
 from enum import StrEnum
-from pydantic import BaseModel
-from app import oauth, settings
+
 import httpx
+from pydantic import BaseModel
+
+from app import oauth
+from app import settings
 
 http_client = httpx.AsyncClient(
     base_url="https://osu.ppy.sh/api/v2/",

app/api/v1/__init__.py

@@ -1,4 +1,5 @@
 from fastapi import APIRouter
+
 from . import cheesegull
 
 v1_router = APIRouter()

app/api/v1/cheesegull.py

@@ -1,5 +1,6 @@
 from fastapi import APIRouter
 from pydantic import BaseModel
+
 from app.adapters import osu_api_v2
 
 router = APIRouter()

app/init_api.py

@@ -0,0 +1,14 @@
+from fastapi import FastAPI
+
+from app.api.v1 import v1_router
+
+
+def init_api() -> FastAPI:
+    app = FastAPI()
+
+    app.include_router(v1_router)
+
+    return app
+
+
+asgi_app = init_api()

app/logger.py

@@ -0,0 +1,9 @@
+import logging.config
+
+import yaml
+
+
+def configure_logging() -> None:
+    with open("logging.yaml") as f:
+        config = yaml.safe_load(f.read())
+        logging.config.dictConfig(config)

app/oauth.py

@@ -1,4 +1,6 @@
-from typing import AsyncGenerator, Generator
+from collections.abc import AsyncGenerator
+from collections.abc import Generator
+
 import httpx
 
 
@@ -33,14 +35,16 @@ def update_tokens(self, response: httpx.Response) -> None:
         self.access_token = data["access_token"]
 
     def sync_auth_flow(
-        self, request: httpx.Request
+        self,
+        request: httpx.Request,
     ) -> Generator[httpx.Request, httpx.Response, None]:
         raise RuntimeError(
-            "Cannot use a sync authentication class with httpx.AsyncClient"
+            "Cannot use a sync authentication class with httpx.AsyncClient",
         )
 
     async def async_auth_flow(
-        self, request: httpx.Request
+        self,
+        request: httpx.Request,
     ) -> AsyncGenerator[httpx.Request, httpx.Response]:
         if self.requires_request_body:
             await request.aread()

app/settings.py

@@ -4,5 +4,16 @@
 
 load_dotenv()
 
+
+def read_bool(s: str) -> bool:
+    return s.lower() == "true"
+
+
+APP_ENV = os.environ["APP_ENV"]
+APP_HOST = os.environ["APP_HOST"]
+APP_PORT = int(os.environ["APP_PORT"])
+
+CODE_HOTRELOAD = read_bool(os.environ["CODE_HOTRELOAD"])
+
 OSU_API_V2_CLIENT_ID = os.environ["OSU_API_V2_CLIENT_ID"]
 OSU_API_V2_CLIENT_SECRET = os.environ["OSU_API_V2_CLIENT_SECRET"]

chart/values.yaml

@@ -0,0 +1,38 @@
+apps:
+  - name: beatmaps-service-api
+    environment: production
+    codebase: beatmaps-service
+    autoscaling:
+      enabled: true
+      minReplicas: 2
+      maxReplicas: 5
+      targetCPUUtilizationPercentage: 80
+    container:
+      image:
+        repository: osuakatsuki/beatmaps-service
+        tag: latest
+      port: 80
+      readinessProbe:
+        httpGet:
+          path: /_health
+          port: 80
+        initialDelaySeconds: 10
+        periodSeconds: 10
+        timeoutSeconds: 3
+        successThreshold: 1
+        failureThreshold: 3
+      resources:
+        limits:
+          cpu: 400m
+          memory: 400Mi
+        requests:
+          cpu: 300m
+          memory: 300Mi
+      env:
+        - name: APP_COMPONENT
+          value: api
+      imagePullSecrets:
+        - name: osuakatsuki-registry-secret
+    service:
+      type: ClusterIP
+      port: 80

logging.yaml

@@ -0,0 +1,28 @@
+version: 1
+disable_existing_loggers: false
+loggers:
+  httpx:
+    level: WARNING
+    handlers: [console]
+    propagate: no
+  httpcore:
+    level: WARNING
+    handlers: [console]
+    propagate: no
+  multipart.multipart:
+    level: ERROR
+    handlers: [console]
+    propagate: no
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: json
+    stream: ext://sys.stdout
+formatters:
+  json:
+    class: pythonjsonlogger.jsonlogger.JsonFormatter
+    format: '%(asctime)s %(name)s %(levelname)s %(message)s'
+root:
+  level: INFO
+  handlers: [console]

main.py

@@ -1,13 +1,24 @@
 #!/usr/bin/env python3
-from fastapi import FastAPI
+import uvicorn
 
-from app.api.v1 import v1_router
+from app import logger
+from app import settings
 
-app = FastAPI()
 
-app.include_router(v1_router)
+def main() -> int:
+    logger.configure_logging()
+
+    uvicorn.run(
+        "app.init_api:asgi_app",
+        reload=settings.CODE_HOTRELOAD,
+        server_header=False,
+        date_header=False,
+        host=settings.APP_HOST,
+        port=settings.APP_PORT,
+        access_log=False,
+    )
+    return 0
 
-if __name__ == "__main__":
-    import uvicorn
 
-    uvicorn.run("main:app")
+if __name__ == "__main__":
+    exit(main())

mypy.ini

@@ -0,0 +1,19 @@
+[mypy]
+exclude = (tests|scripts|venv)
+
+strict = True
+disallow_untyped_calls = True
+enable_error_code = truthy-bool, truthy-iterable, ignore-without-code, unused-awaitable, redundant-expr, possibly-undefined
+
+[mypy-tests.*]
+disable_error_code = var-annotated, has-type
+allow_untyped_defs = True
+
+[mypy-amplitude.*]
+ignore_missing_imports = True
+
+[mypy-orjson.*]
+ignore_missing_imports = True
+
+[mypy-py3rijndael.*]
+ignore_missing_imports = True

requirements-dev.txt

@@ -0,0 +1,3 @@
+mypy
+types-jmespath
+types-pyyaml

requirements.txt

@@ -1,5 +1,5 @@
 fastapi
-uvicorn
 httpx
 python-dotenv
 python-json-logger
+uvicorn