k8s deploy (#1)

* initial k8s deploy attempt

* fix pypi

* replace API_PORT with PERFORMANCE_SERVICE_BASE_URL

* fix vault pull

* fix pending install pending deploy case

* python-json-logger dep

* fix

* color helm diff, support pending install & rollback clearing, silence perms warnings, cleanup on failure, 10m timeout

* reduce system resource usage

* k8s deploy becomes production deploy

---------

Co-authored-by: Josh Smith <[email protected]>

.env.example

@@ -1,9 +1,8 @@
-APP_ENV="" # production/staging/local
-APP_COMPONENT="rework-frontend"
-APP_HOST="127.0.0.1"
+APP_ENV=local
+APP_COMPONENT=api
+APP_HOST=127.0.0.1
 APP_PORT=8787
-
-API_PORT=8665
-
-# CRITICAL: 50 | ERROR: 40 | WARNING: 30 | INFO: 20 | DEBUG: 10
-LOG_LEVEL=10
+CODE_HOTRELOAD=false
+SERVICE_READINESS_TIMEOUT=60
+PULL_SECRETS_FROM_VAULT=
+PERFORMANCE_SERVICE_BASE_URL=http://localhost:8665

.github/workflows/production-deploy.yml

@@ -0,0 +1,89 @@
+name: production-deploy
+
+on:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  production-deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out latest commit
+        uses: actions/checkout@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: osuAkatsuki/rework-frontend
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/rework-frontend:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/rework-frontend:${{ github.sha }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Get kubeconfig from github secrets
+        run: |
+          mkdir -p $HOME/.kube
+          echo "${{ secrets.KUBECONFIG }}" > $HOME/.kube/config
+          sudo chown $(id -u):$(id -g) $HOME/.kube/config
+          chmod 600 $HOME/.kube/config
+
+      - name: Install helm
+        uses: azure/setup-helm@v3
+        with:
+          version: "latest"
+          token: ${{ secrets.GITHUB_TOKEN }}
+        id: install
+
+      - name: Install helm-diff
+        run: helm plugin install https://github.com/databus23/helm-diff
+
+      - name: Checkout common-helm-charts repo
+        uses: actions/checkout@v3
+        with:
+          repository: osuAkatsuki/common-helm-charts
+          token: ${{ secrets.COMMON_HELM_CHARTS_PAT }}
+          path: common-helm-charts
+
+      - name: Clear pending deployments
+        run: |
+          kubectl delete secret -l 'status in (pending-install, pending-upgrade, pending-rollback),name=rework-frontend-production'
+
+      - name: Show manifest diff since previous release
+        run: |
+          helm diff upgrade \
+          --allow-unreleased \
+          --color=true \
+          --values chart/values.yaml \
+          rework-frontend-production \
+          common-helm-charts/microservice-base/
+
+      - name: Deploy service to production cluster
+        run: |
+          helm upgrade \
+            --install \
+            --atomic \
+            --wait --timeout 10m \
+            --cleanup-on-fail \
+            --values chart/values.yaml \
+            rework-frontend-production \
+            common-helm-charts/microservice-base/

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.10
+
+ENV PYTHONUNBUFFERED=1
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+RUN pip install -i https://pypi2.akatsuki.gg/cmyui/dev akatsuki-cli
+
+COPY scripts /scripts
+
+COPY . /srv/root
+WORKDIR /srv/root
+
+ENTRYPOINT ["/scripts/bootstrap.sh"]

app/api/rest/__init__.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
+import logging
+
 from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
-from shared_modules import logger
 from starlette.middleware.base import BaseHTTPMiddleware
 
 from app.api import middlewares
@@ -12,17 +13,17 @@
 def init_http(api: FastAPI) -> None:
     @api.on_event("startup")
     async def startup_http() -> None:
-        logger.info("Starting up http connector")
+        logging.info("Starting up http connector")
         service_http = http.ServiceHTTP()
         api.state.http = service_http
-        logger.info("HTTP connector started up")
+        logging.info("HTTP connector started up")
 
     @api.on_event("shutdown")
     async def shutdown_http() -> None:
-        logger.info("Shutting down http connector")
+        logging.info("Shutting down http connector")
         await api.state.http.aclose()
         del api.state.http
-        logger.info("HTTP connector shut down")
+        logging.info("HTTP connector shut down")
 
 
 def init_middlewares(api: FastAPI) -> None:
@@ -44,6 +45,10 @@ def init_routes(api: FastAPI) -> None:
     api.include_router(v1_router)
     api.include_router(public_router)
 
+    @api.get("/_health")
+    async def healthcheck():
+        return {"status": "ok"}
+
 
 def init_api():
     api = FastAPI()

app/api_boot.py

@@ -1,10 +1,5 @@
 from __future__ import annotations
 
-from shared_modules import logger
-
 from app.api.rest import init_api
-from app.common import settings
-
-logger.configure_logging(app_env=settings.APP_ENV, log_level=settings.LOG_LEVEL)
 
 api = init_api()

app/common/settings.py

@@ -6,11 +6,18 @@
 
 load_dotenv()
 
-LOG_LEVEL = int(os.environ["LOG_LEVEL"])
+
+def read_bool(value: str) -> bool:
+    return value.lower() in ("true", "1")
+
 
 APP_ENV = os.environ["APP_ENV"]
 APP_COMPONENT = os.environ["APP_COMPONENT"]
 APP_HOST = os.environ["APP_HOST"]
 APP_PORT = int(os.environ["APP_PORT"])
 
-API_PORT = int(os.environ["API_PORT"])
+CODE_HOTRELOAD = read_bool(os.environ["CODE_HOTRELOAD"])
+SERVICE_READINESS_TIMEOUT = int(os.environ["SERVICE_READINESS_TIMEOUT"])
+PULL_SECRETS_FROM_VAULT = read_bool(os.environ["PULL_SECRETS_FROM_VAULT"])
+
+PERFORMANCE_SERVICE_BASE_URL = os.environ["PERFORMANCE_SERVICE_BASE_URL"]

app/exception_handling.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import logging
+import sys
+import threading
+from types import TracebackType
+from typing import Any
+from typing import Callable
+from typing import Optional
+
+ExceptionHook = Callable[
+    [type[BaseException], BaseException, Optional[TracebackType]],
+    Any,
+]
+ThreadingExceptionHook = Callable[[threading.ExceptHookArgs], Any]
+
+_default_excepthook: Optional[ExceptionHook] = None
+_default_threading_excepthook: Optional[ThreadingExceptionHook] = None
+
+
+def internal_exception_handler(
+    exc_type: type[BaseException],
+    exc_value: BaseException,
+    exc_traceback: Optional[TracebackType],
+) -> None:
+    logging.exception(
+        "An unhandled exception occurred",
+        exc_info=(exc_type, exc_value, exc_traceback),
+    )
+
+
+def internal_thread_exception_handler(
+    args: threading.ExceptHookArgs,
+) -> None:
+    if args.exc_value is None:  # pragma: no cover
+        logging.warning("Exception hook called without exception value.")
+        return
+
+    logging.exception(
+        "An unhandled exception occurred",
+        exc_info=(args.exc_type, args.exc_value, args.exc_traceback),
+        extra={"thread_vars": vars(args.thread)},
+    )
+
+
+def hook_exception_handlers() -> None:
+    global _default_excepthook
+    _default_excepthook = sys.excepthook
+    sys.excepthook = internal_exception_handler
+
+    global _default_threading_excepthook
+    _default_threading_excepthook = threading.excepthook
+    threading.excepthook = internal_thread_exception_handler
+
+
+def unhook_exception_handlers() -> None:
+    global _default_excepthook
+    if _default_excepthook is not None:
+        sys.excepthook = _default_excepthook
+
+    global _default_threading_excepthook
+    if _default_threading_excepthook is not None:
+        threading.excepthook = _default_threading_excepthook

app/logging.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+import logging.config
+
+import yaml
+
+
+def configure_logging() -> None:
+    with open("logging.yaml") as f:
+        config = yaml.safe_load(f.read())
+        logging.config.dictConfig(config)

app/repositories/leaderboards.py

@@ -7,7 +7,7 @@
 
 
 class LeaderboardsRepo:
-    BASE_URI = f"http://localhost:{settings.API_PORT}"
+    BASE_URI = settings.PERFORMANCE_SERVICE_BASE_URL
 
     def __init__(self, ctx: Context) -> None:
         self.ctx = ctx

app/repositories/reworks.py

@@ -7,7 +7,7 @@
 
 
 class ReworksRepo:
-    BASE_URI = f"http://localhost:{settings.API_PORT}"
+    BASE_URI = settings.PERFORMANCE_SERVICE_BASE_URL
 
     def __init__(self, ctx: Context) -> None:
         self.ctx = ctx

app/repositories/scores.py

@@ -7,7 +7,7 @@
 
 
 class ScoresRepo:
-    BASE_URI = f"http://localhost:{settings.API_PORT}"
+    BASE_URI = settings.PERFORMANCE_SERVICE_BASE_URL
 
     def __init__(self, ctx: Context) -> None:
         self.ctx = ctx

app/repositories/sessions.py

@@ -8,7 +8,7 @@
 
 
 class SessionsRepo:
-    BASE_URI = f"http://localhost:{settings.API_PORT}"
+    BASE_URI = settings.PERFORMANCE_SERVICE_BASE_URL
 
     def __init__(self, ctx: Context) -> None:
         self.ctx = ctx

app/repositories/stats.py

@@ -7,7 +7,7 @@
 
 
 class StatsRepo:
-    BASE_URI = f"http://localhost:{settings.API_PORT}"
+    BASE_URI = settings.PERFORMANCE_SERVICE_BASE_URL
 
     def __init__(self, ctx: Context) -> None:
         self.ctx = ctx

app/repositories/users.py

@@ -7,7 +7,7 @@
 
 
 class UsersRepo:
-    BASE_URI = f"http://localhost:{settings.API_PORT}"
+    BASE_URI = settings.PERFORMANCE_SERVICE_BASE_URL
 
     def __init__(self, ctx: Context) -> None:
         self.ctx = ctx

chart/values.yaml

@@ -0,0 +1,34 @@
+apps:
+  - name: rework-frontend-api
+    environment: production
+    codebase: rework-frontend
+    replicaCount: 1
+    container:
+      image:
+        repository: osuakatsuki/rework-frontend
+        tag: latest
+      port: 80
+      readinessProbe:
+        httpGet:
+          path: /_health
+          port: 80
+        initialDelaySeconds: 10
+        periodSeconds: 10
+        timeoutSeconds: 1
+        successThreshold: 1
+        failureThreshold: 3
+      resources:
+        limits:
+          cpu: 300m
+          memory: 250Mi
+        requests:
+          cpu: 150m
+          memory: 150Mi
+      env:
+        - name: APP_COMPONENT
+          value: api
+      imagePullSecrets:
+        - name: osuakatsuki-registry-secret
+    service:
+      type: ClusterIP
+      port: 80

logging.yaml

@@ -0,0 +1,24 @@
+version: 1
+disable_existing_loggers: true
+loggers:
+  httpx:
+    level: WARNING
+    handlers: [console]
+    propagate: no
+  httpcore:
+    level: WARNING
+    handlers: [console]
+    propagate: no
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: INFO
+    formatter: json
+    stream: ext://sys.stdout
+formatters:
+  json:
+    class: pythonjsonlogger.jsonlogger.JsonFormatter
+    format: '%(asctime)s %(name)s %(levelname)s %(message)s'
+root:
+  level: INFO
+  handlers: [console]