From 3b0fe85c0d1be5cdd25c74b8420da825a1c21837 Mon Sep 17 00:00:00 2001 From: Josh Smith Date: Fri, 25 Oct 2024 00:58:50 -0400 Subject: [PATCH] Automated flow for GDPR/CCPA deletion requests (#7) * Draft: Work towards improved GDPR/CCPA compliance * ac assoc repos * todo * txn and logs * 204 resp * lastfm flags * redis conn * dep * reformat some code * add routing * wipe notes and add dt's * peppy:ban pubsub * error resp * fixes & wipe more user data * type fixes * note * reduce map * typo * take over prod --- .env.example | 6 + .github/workflows/production-deploy.yml | 1 + app/api/authorization.py | 3 +- app/api/internal/v1/__init__.py | 4 + app/api/internal/v1/users.py | 42 ++++++ app/api/public/__init__.py | 2 +- app/api/public/authentication.py | 5 +- app/api/public/overall_stats.py | 6 +- app/api/public/user_stats.py | 11 +- app/api/public/users.py | 10 +- app/common_types.py | 3 +- app/init_api.py | 20 ++- app/repositories/clans.py | 75 +++++++++++ app/repositories/lastfm_flags.py | 94 +++++++++++++ app/repositories/password_recovery.py | 47 +++++++ app/repositories/patcher_detections.py | 21 +++ app/repositories/user_hwid_associations.py | 52 ++++++++ app/repositories/user_ip_associations.py | 46 +++++++ app/repositories/user_stats.py | 5 +- app/repositories/users.py | 94 ++++++++++++- app/settings.py | 6 + app/state.py | 2 + app/usecases/user_stats.py | 5 +- app/usecases/users.py | 148 +++++++++++++++++++++ requirements.txt | 1 + 25 files changed, 683 insertions(+), 26 deletions(-) create mode 100644 app/api/internal/v1/users.py create mode 100644 app/repositories/clans.py create mode 100644 app/repositories/lastfm_flags.py create mode 100644 app/repositories/password_recovery.py create mode 100644 app/repositories/patcher_detections.py create mode 100644 app/repositories/user_hwid_associations.py create mode 100644 app/repositories/user_ip_associations.py diff --git a/.env.example b/.env.example index 903d94c..de4d8e6 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,12 @@ DB_HOST=localhost DB_PORT=3306 DB_NAME=akatsuki +REDIS_HOST=localhost +REDIS_PORT=6379 +REDIS_USER=default +REDIS_PASS= +REDIS_DB=0 + AWS_S3_ENDPOINT_URL= AWS_S3_REGION_NAME= AWS_S3_BUCKET_NAME= diff --git a/.github/workflows/production-deploy.yml b/.github/workflows/production-deploy.yml index c31da5e..f14ac08 100644 --- a/.github/workflows/production-deploy.yml +++ b/.github/workflows/production-deploy.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - gdpr-ccpa-compliance concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/app/api/authorization.py b/app/api/authorization.py index a88cb51..786593b 100644 --- a/app/api/authorization.py +++ b/app/api/authorization.py @@ -1,5 +1,6 @@ from app import security -from app.errors import Error, ErrorCode +from app.errors import Error +from app.errors import ErrorCode from app.repositories import access_tokens diff --git a/app/api/internal/v1/__init__.py b/app/api/internal/v1/__init__.py index f142283..5925119 100644 --- a/app/api/internal/v1/__init__.py +++ b/app/api/internal/v1/__init__.py @@ -1,3 +1,7 @@ from fastapi import APIRouter +from app.api.internal.v1 import users + v1_router = APIRouter() + +v1_router.include_router(users.router) diff --git a/app/api/internal/v1/users.py b/app/api/internal/v1/users.py new file mode 100644 index 0000000..69a461b --- /dev/null +++ b/app/api/internal/v1/users.py @@ -0,0 +1,42 @@ +import logging + +from fastapi import APIRouter +from fastapi import Response +from fastapi import status + +from app.api.responses import JSONResponse +from app.errors import Error +from app.errors import ErrorCode +from app.usecases import users + +router = APIRouter(tags=["(Internal) Users API"]) + + +def map_error_code_to_http_status_code(error_code: ErrorCode) -> int: + status_code = _error_code_to_http_status_code_map.get(error_code) + if status_code is None: + logging.warning( + "No HTTP status code mapping found for error code: %s", + error_code, + extra={"error_code": error_code}, + ) + return 500 + return status_code + + +_error_code_to_http_status_code_map: dict[ErrorCode, int] = { + ErrorCode.INTERNAL_SERVER_ERROR: 500, +} + + +@router.delete("/api/v1/users/{user_id}") +async def delete_user(user_id: int) -> Response: + response = await users.delete_one_by_user_id(user_id) + if isinstance(response, Error): + return JSONResponse( + content=response.model_dump(), + status_code=map_error_code_to_http_status_code(response.error_code), + ) + + # TODO: check if response body is b"none" + return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/app/api/public/__init__.py b/app/api/public/__init__.py index 3c397a4..fc772b2 100644 --- a/app/api/public/__init__.py +++ b/app/api/public/__init__.py @@ -2,8 +2,8 @@ from . import authentication from . import overall_stats -from . import users from . import user_stats +from . import users public_router = APIRouter() diff --git a/app/api/public/authentication.py b/app/api/public/authentication.py index 59b9257..ff6630f 100644 --- a/app/api/public/authentication.py +++ b/app/api/public/authentication.py @@ -1,4 +1,5 @@ -from fastapi import APIRouter, Cookie +from fastapi import APIRouter +from fastapi import Cookie from fastapi import Header from fastapi import Response from pydantic import BaseModel @@ -78,7 +79,7 @@ async def logout( return JSONResponse( content=trusted_access_token.model_dump(), status_code=map_error_code_to_http_status_code( - trusted_access_token.error_code + trusted_access_token.error_code, ), ) diff --git a/app/api/public/overall_stats.py b/app/api/public/overall_stats.py index 07a3585..837af62 100644 --- a/app/api/public/overall_stats.py +++ b/app/api/public/overall_stats.py @@ -1,7 +1,9 @@ -from fastapi import APIRouter, Response +from fastapi import APIRouter +from fastapi import Response from fastapi.responses import JSONResponse -from app.usecases import user_stats, users +from app.usecases import user_stats +from app.usecases import users router = APIRouter(tags=["(Public) Overall Stats API"]) diff --git a/app/api/public/user_stats.py b/app/api/public/user_stats.py index a044c15..c52fa10 100644 --- a/app/api/public/user_stats.py +++ b/app/api/public/user_stats.py @@ -1,13 +1,13 @@ from fastapi import APIRouter -from fastapi import Response from fastapi import Query +from fastapi import Response from app.api.responses import JSONResponse -from app.errors import Error -from app.errors import ErrorCode +from app.common_types import AkatsukiMode from app.common_types import GameMode from app.common_types import RelaxMode -from app.common_types import AkatsukiMode +from app.errors import Error +from app.errors import ErrorCode from app.usecases import user_stats router = APIRouter(tags=["(Public) User Stats API"]) @@ -35,7 +35,8 @@ async def get_user_stats( akatsuki_mode = AkatsukiMode.from_game_mode_and_relax_mode(game_mode, relax_mode) response = await user_stats.fetch_one_by_user_id_and_akatsuki_mode( - user_id, akatsuki_mode + user_id, + akatsuki_mode, ) if isinstance(response, Error): return JSONResponse( diff --git a/app/api/public/users.py b/app/api/public/users.py index 7b3358c..c030d82 100644 --- a/app/api/public/users.py +++ b/app/api/public/users.py @@ -1,5 +1,7 @@ import logging -from fastapi import APIRouter, Cookie + +from fastapi import APIRouter +from fastapi import Cookie from fastapi import Response from pydantic import BaseModel @@ -67,7 +69,7 @@ async def update_username( return JSONResponse( content=trusted_access_token.model_dump(), status_code=map_error_code_to_http_status_code( - trusted_access_token.error_code + trusted_access_token.error_code, ), ) @@ -100,7 +102,7 @@ async def update_password( return JSONResponse( content=trusted_access_token.model_dump(), status_code=map_error_code_to_http_status_code( - trusted_access_token.error_code + trusted_access_token.error_code, ), ) @@ -137,7 +139,7 @@ async def update_email_address( return JSONResponse( content=trusted_access_token.model_dump(), status_code=map_error_code_to_http_status_code( - trusted_access_token.error_code + trusted_access_token.error_code, ), ) diff --git a/app/common_types.py b/app/common_types.py index 94dc5fd..377132b 100644 --- a/app/common_types.py +++ b/app/common_types.py @@ -1,5 +1,6 @@ -from enum import IntFlag from enum import IntEnum +from enum import IntFlag + class UserPrivileges(IntFlag): USER_PUBLIC = 1 << 0 diff --git a/app/init_api.py b/app/init_api.py index c434d0c..f7c5b34 100644 --- a/app/init_api.py +++ b/app/init_api.py @@ -6,22 +6,25 @@ from databases import Database from fastapi import FastAPI from fastapi import Request -from fastapi.exceptions import RequestValidationError from fastapi import Response -from starlette.middleware.base import RequestResponseEndpoint from fastapi.exception_handlers import request_validation_exception_handler +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from redis.asyncio import Redis +from starlette.middleware.base import RequestResponseEndpoint + from app import logger from app import settings from app import state from app.adapters import mysql from app.api import api_router -from fastapi.responses import JSONResponse @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: logger.configure_logging() await state.database.connect() + await state.redis.initialize() # type: ignore[unused-awaitable] aws_session = aiobotocore.session.get_session() s3_client = aws_session.create_client( @@ -35,6 +38,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: yield await state.s3_client.__aexit__(None, None, None) + await state.redis.aclose() await state.database.disconnect() @@ -57,7 +61,8 @@ async def http_middleware( @app.exception_handler(RequestValidationError) async def validation_exception_handler( - request: Request, exc: RequestValidationError + request: Request, + exc: RequestValidationError, ) -> JSONResponse: logging.warning( "Request validation failed", @@ -70,6 +75,13 @@ async def validation_exception_handler( def init_db(app: FastAPI) -> FastAPI: + state.redis = Redis( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + db=settings.REDIS_DB, + username=settings.REDIS_USER, + password=settings.REDIS_PASS, + ) state.database = Database( url=mysql.create_dsn( driver="aiomysql", diff --git a/app/repositories/clans.py b/app/repositories/clans.py new file mode 100644 index 0000000..cb6d4a4 --- /dev/null +++ b/app/repositories/clans.py @@ -0,0 +1,75 @@ +from enum import IntEnum + +from pydantic import BaseModel + +import app.state + + +class ClanStatus(IntEnum): + CLOSED = 0 + OPEN_FOR_ALL = 1 + INVITE_ONLY = 2 + REQUEST_TO_JOIN = 3 + + +class Clan(BaseModel): + id: int + name: str + tag: str + description: str + icon: str + background: str + owner: int + invite: str + status: ClanStatus + + +READ_PARAMS = """\ + id, name, tag, description, icon, background, owner, invite, status +""" + + +async def fetch_one_by_clan_id(clan_id: int, /) -> Clan | None: + query = f"""\ + SELECT {READ_PARAMS} + FROM clans + WHERE id = :clan_id + """ + params = {"clan_id": clan_id} + + clan = await app.state.database.fetch_one(query, params) + if clan is None: + return None + + return Clan( + id=clan["id"], + name=clan["name"], + tag=clan["tag"], + description=clan["description"], + icon=clan["icon"], + background=clan["background"], + owner=clan["owner"], + invite=clan["invite"], + status=ClanStatus(clan["status"]), + ) + + +async def update_owner(clan_id: int, new_owner: int) -> None: + query = """\ + UPDATE clans + SET owner = :new_owner + WHERE id = :clan_id + """ + params = {"new_owner": new_owner, "clan_id": clan_id} + + await app.state.database.execute(query, params) + + +async def delete_one_by_clan_id(clan_id: int, /) -> None: + query = """\ + DELETE FROM clans + WHERE id = :clan_id + """ + params = {"clan_id": clan_id} + + await app.state.database.execute(query, params) diff --git a/app/repositories/lastfm_flags.py b/app/repositories/lastfm_flags.py new file mode 100644 index 0000000..c93b101 --- /dev/null +++ b/app/repositories/lastfm_flags.py @@ -0,0 +1,94 @@ +from enum import IntFlag +from typing import Any + +from pydantic import BaseModel + +import app.state + + +class LastfmFlagType(IntFlag): + """Bitwise enum flags for osu's LastFM anticheat flags (aka `BadFlags`).""" + + # 2016 Anticheat (from source) + TIMEWARP = ( + 1 << 1 + ) # Saw this one get triggered during intense lag. Compares song speed to time elapsed. + INCORRECT_MOD_VALUE = 1 << 2 # Cheat attempted to alter the mod values incorrectly + MULTIPLE_OSU_CLIENTS = 1 << 3 + CHECKSUM_FAIL = 1 << 4 # Cheats that modify memory to unrealistic values. + FLASHLIGHT_CHECKSUM_FAIL = 1 << 5 + + # These 2 are server side + OSU_EXE_CHECKSUM_FAIL = 1 << 6 + MISSING_PROCESS = 1 << 7 + + FLASHLIGHT_REMOVER = 1 << 8 # Checks actual pixels on the screen + AUTOSPIN_HACK = 1 << 9 # Unused in 2016 src + WINDOW_OVERLAY = 1 << 10 # There is a transparent window overlaying osu (cheat uis) + FAST_PRESS = 1 << 11 # Mania only. dont understand it fully. + + # These check if there is something altering the cursor pos/kb being received + # through comparing the raw input. + MOUSE_DISCREPENCY = 1 << 12 + KB_DISCREPENCY = 1 << 13 + + # These are taken from `gulag` https://github.com/cmyui/gulag/blob/master/constants/clientflags.py + # They relate to the new 2019 lastfm extension introducing measures against AQN and HQOsu. + LF_FLAG_PRESENT = 1 << 14 + OSU_DEBUGGED = 1 << 15 # A console attached to the osu process is running. + EXTRA_THREADS = ( + 1 << 16 + ) # Osu cheats usually create a new thread to run it. This aims to detect them. + + # HQOsu specific ones. + HQOSU_ASSEMBLY = 1 << 17 + HQOSU_FILE = 1 << 18 + HQ_RELIFE = 1 << 19 # Detects registry edits left by Relife + + # (Outdated) AQN detection methods + AQN_SQL2LIB = 1 << 20 + AQN_LIBEAY32 = 1 << 21 + AQN_MENU_SOUND = 1 << 22 + + +class LastfmFlag(BaseModel): + id: int + user_id: int + timestamp: int + flag_enum: LastfmFlagType + flag_text: str + + +READ_PARAMS = """\ + id, user_id, timestamp, flag_enum, flag_text +""" + + +async def delete_many_by_user_id(user_id: int, /) -> list[LastfmFlag]: + query = f"""\ + SELECT {READ_PARAMS} + FROM lastfm_flags + WHERE user_id = :user_id + """ + params: dict[str, Any] = {"user_id": user_id} + recs = await app.state.database.fetch_all(query, params) + if not recs: + return [] + + query = """\ + DELETE FROM lastfm_flags + WHERE user_id = :user_id + """ + params = {"user_id": user_id} + await app.state.database.execute(query, params) + + return [ + LastfmFlag( + id=rec["id"], + user_id=rec["user_id"], + timestamp=rec["timestamp"], + flag_enum=LastfmFlagType(rec["flag_enum"]), + flag_text=rec["flag_text"], + ) + for rec in recs + ] diff --git a/app/repositories/password_recovery.py b/app/repositories/password_recovery.py new file mode 100644 index 0000000..d257533 --- /dev/null +++ b/app/repositories/password_recovery.py @@ -0,0 +1,47 @@ +from datetime import datetime +from enum import IntEnum + +from pydantic import BaseModel + +import app.state + + +class PasswordRecovery(BaseModel): + id: int + k: str # key + u: str # username + t: datetime + + +READ_PARAMS = """\ + id, k, u, t +""" + + +async def delete_many_by_username(username: str, /) -> list[PasswordRecovery]: + query = f"""\ + SELECT {READ_PARAMS} + FROM password_recovery + WHERE u = :username + """ + params = {"username": username} + recs = await app.state.database.fetch_all(query, params) + if not recs: + return [] + + query = """\ + DELETE FROM password_recovery + WHERE u = :username + """ + params = {"username": username} + await app.state.database.execute(query, params) + + return [ + PasswordRecovery( + id=rec["id"], + k=rec["k"], + u=rec["u"], + t=rec["t"], + ) + for rec in recs + ] diff --git a/app/repositories/patcher_detections.py b/app/repositories/patcher_detections.py new file mode 100644 index 0000000..b0dad35 --- /dev/null +++ b/app/repositories/patcher_detections.py @@ -0,0 +1,21 @@ +from datetime import datetime + +from pydantic import BaseModel + + +class PatcherDetection(BaseModel): + id: str + method_name: str + method_assembly_hash: str + method_assembly_instructions: str + created_at: datetime + updated_at: datetime + patcher_token_log_id: str + + +READ_PARAMS = """\ + id, method_name, method_assembly_hash, method_assembly_instructions, created_at, updated_at, patcher_token_log_id +""" + + +# TODO: figure out a way to delete records per-user diff --git a/app/repositories/user_hwid_associations.py b/app/repositories/user_hwid_associations.py new file mode 100644 index 0000000..58e0b02 --- /dev/null +++ b/app/repositories/user_hwid_associations.py @@ -0,0 +1,52 @@ +from typing import Any + +from pydantic import BaseModel + +import app.state + + +class UserHwidAssociation(BaseModel): + id: int + userid: int + mac: str + unique_id: str + disk_id: str + occurencies: int + activated: bool + + +READ_PARAMS = """\ + id, userid, mac, unique_id, disk_id, occurencies, activated +""" + + +async def delete_many_by_user_id(user_id: int, /) -> list[UserHwidAssociation]: + query = f"""\ + SELECT {READ_PARAMS} + FROM hw_user + WHERE userid = :user_id + """ + params: dict[str, Any] = {"user_id": user_id} + recs = await app.state.database.fetch_all(query, params) + if not recs: + return [] + + query = """\ + DELETE FROM hw_user + WHERE userid = :user_id + """ + params = {"user_id": user_id} + await app.state.database.execute(query, params) + + return [ + UserHwidAssociation( + id=rec["id"], + userid=rec["userid"], + mac=rec["mac"], + unique_id=rec["unique_id"], + disk_id=rec["disk_id"], + occurencies=rec["occurencies"], + activated=rec["activated"], + ) + for rec in recs + ] diff --git a/app/repositories/user_ip_associations.py b/app/repositories/user_ip_associations.py new file mode 100644 index 0000000..398ea24 --- /dev/null +++ b/app/repositories/user_ip_associations.py @@ -0,0 +1,46 @@ +from typing import Any + +from pydantic import BaseModel + +import app.state + + +class UserIpAssociation(BaseModel): + id: int + userid: int + ip: str + occurencies: int + + +READ_PARAMS = """\ + id, userid, ip, occurencies +""" + + +async def delete_many_by_user_id(user_id: int, /) -> list[UserIpAssociation]: + query = f"""\ + SELECT {READ_PARAMS} + FROM ip_user + WHERE userid = :user_id + """ + params: dict[str, Any] = {"user_id": user_id} + recs = await app.state.database.fetch_all(query, params) + if not recs: + return [] + + query = """\ + DELETE FROM ip_user + WHERE userid = :user_id + """ + params = {"user_id": user_id} + await app.state.database.execute(query, params) + + return [ + UserIpAssociation( + id=rec["id"], + userid=rec["userid"], + ip=rec["ip"], + occurencies=rec["occurencies"], + ) + for rec in recs + ] diff --git a/app/repositories/user_stats.py b/app/repositories/user_stats.py index 3ae2e3d..cdb1cf6 100644 --- a/app/repositories/user_stats.py +++ b/app/repositories/user_stats.py @@ -1,8 +1,7 @@ from pydantic import BaseModel -from app.common_types import AkatsukiMode - import app.state +from app.common_types import AkatsukiMode class UserStats(BaseModel): @@ -82,4 +81,4 @@ async def fetch_global_total_pp_earned() -> int: val = await app.state.database.fetch_val(query) if val is None: return 0 - return val + return int(val) diff --git a/app/repositories/users.py b/app/repositories/users.py index dcb2f75..81a6443 100644 --- a/app/repositories/users.py +++ b/app/repositories/users.py @@ -1,8 +1,10 @@ +import secrets from datetime import datetime from pydantic import BaseModel import app.state +from app import security from app.common_types import GameMode from app.common_types import UserPlayStyle from app.common_types import UserPrivileges @@ -169,4 +171,94 @@ async def fetch_total_registered_user_count() -> int: val = await app.state.database.fetch_val(query) if val is None: return 0 - return val + return int(val) + + +async def anonymize_one_by_user_id(user_id: int, /) -> None: + dt = datetime.now().isoformat() + new_hashed_password = security.hash_osu_password(secrets.token_hex(16)) + await app.state.database.execute( + """\ + UPDATE users + SET username = :username, + username_safe = :username_safe, + username_aka = :username_aka, + password_md5 = :password_md5, + email = :email, + userpage_content = :userpage_content, + silence_end = :silence_end, + donor_expire = :donor_expire, + latest_activity = :latest_activity, + register_datetime = :register_datetime, + ban_datetime = :ban_datetime, + silence_reason = :silence_reason, + freeze_reason = :freeze_reason, + can_custom_badge = :can_custom_badge, + show_custom_badge = :show_custom_badge, + custom_badge_icon = :custom_badge_icon, + custom_badge_name = :custom_badge_name, + notes = :notes, + country = :country, + privileges = :privileges, + clan_id = :clan_id + WHERE id = :user_id + """, + { + "user_id": user_id, + "username": f"deleted_user_{user_id}", + "username_safe": f"deleted_user_{user_id}", + "username_aka": f"deleted_user_{user_id}", + "password_md5": new_hashed_password, + "email": f"delete_user_{user_id}@example.com", + "userpage_content": f"[{dt}] This user has been deleted.", + "silence_end": 0, + "donor_expire": 0, + "latest_activity": 0, + "register_datetime": 0, + "ban_datetime": 0, + "silence_reason": "", + "freeze_reason": "", + "can_custom_badge": False, + "show_custom_badge": False, + "custom_badge_icon": "", + "custom_badge_name": "", + "notes": f"[{dt}] This user has been deleted.", + "country": "XX", + "privileges": UserPrivileges(0), + "clan_id": 0, + }, + ) + + +async def fetch_many_by_clan_id(clan_id: int, /) -> list[User]: + query = f"""\ + SELECT {READ_PARAMS} + FROM users + WHERE clan_id = :clan_id + """ + params = {"clan_id": clan_id} + + users = await app.state.database.fetch_all(query, params) + return [ + User( + id=user["id"], + username=user["username"], + username_aka=user["username_aka"], + created_at=datetime.fromtimestamp(user["register_datetime"]), + latest_activity=datetime.fromtimestamp(user["latest_activity"]), + userpage_content=user["userpage_content"], + country=user["country"], + privileges=UserPrivileges(user["privileges"]), + hashed_password=user["password_md5"], + clan_id=user["clan_id"], + play_style=UserPlayStyle(user["play_style"]), + favourite_mode=GameMode(user["favourite_mode"]), + custom_badge_icon=user["custom_badge_icon"], + custom_badge_name=user["custom_badge_name"], + can_custom_badge=user["can_custom_badge"], + show_custom_badge=user["show_custom_badge"], + silence_reason=user["silence_reason"], + silence_end=user["silence_end"], + ) + for user in users + ] diff --git a/app/settings.py b/app/settings.py index 7fc2ee2..085e658 100644 --- a/app/settings.py +++ b/app/settings.py @@ -21,6 +21,12 @@ def read_bool(s: str) -> bool: DB_PORT = int(os.environ["DB_PORT"]) DB_NAME = os.environ["DB_NAME"] +REDIS_HOST = os.environ["REDIS_HOST"] +REDIS_PORT = int(os.environ["REDIS_PORT"]) +REDIS_DB = int(os.environ["REDIS_DB"]) +REDIS_USER = os.environ["REDIS_USER"] +REDIS_PASS = os.environ["REDIS_PASS"] + AWS_S3_ENDPOINT_URL = os.environ["AWS_S3_ENDPOINT_URL"] AWS_S3_REGION_NAME = os.environ["AWS_S3_REGION_NAME"] AWS_S3_BUCKET_NAME = os.environ["AWS_S3_BUCKET_NAME"] diff --git a/app/state.py b/app/state.py index 96b127f..4ec66ec 100644 --- a/app/state.py +++ b/app/state.py @@ -2,7 +2,9 @@ if TYPE_CHECKING: from databases import Database + from redis.asyncio import Redis from types_aiobotocore_s3.client import S3Client database: "Database" +redis: "Redis" s3_client: "S3Client" diff --git a/app/usecases/user_stats.py b/app/usecases/user_stats.py index 63b2983..0bd4400 100644 --- a/app/usecases/user_stats.py +++ b/app/usecases/user_stats.py @@ -1,12 +1,13 @@ from app.common_types import AkatsukiMode -from app.models.user_stats import UserStats from app.errors import Error from app.errors import ErrorCode +from app.models.user_stats import UserStats from app.repositories import user_stats async def fetch_one_by_user_id_and_akatsuki_mode( - user_id: int, mode: AkatsukiMode + user_id: int, + mode: AkatsukiMode, ) -> UserStats | Error: stats = await user_stats.fetch_one_by_user_id_and_akatsuki_mode(user_id, mode) if stats is None: diff --git a/app/usecases/users.py b/app/usecases/users.py index e9d6d5b..1a4d46b 100644 --- a/app/usecases/users.py +++ b/app/usecases/users.py @@ -1,3 +1,6 @@ +import logging + +import app.state from app import security from app.common_types import UserPrivileges from app.errors import Error @@ -6,7 +9,12 @@ from app.models.users import CustomBadge from app.models.users import TournamentBadge from app.models.users import User +from app.repositories import clans +from app.repositories import lastfm_flags +from app.repositories import password_recovery from app.repositories import user_badges +from app.repositories import user_hwid_associations +from app.repositories import user_ip_associations from app.repositories import user_relationships from app.repositories import user_tournament_badges from app.repositories import users @@ -192,3 +200,143 @@ async def update_email_address( async def fetch_total_registered_user_count() -> int: return await users.fetch_total_registered_user_count() + + +async def delete_one_by_user_id(user_id: int, /) -> None | Error: + """\ + An anonymization process for user deletion, mainly implemented + for the purpose of complying with GDPR, CCPA and other regulations. + """ + # sql tables associated with users: + # - [anonymize] users + # - [leave as-is] users_stats + # - [leave as-is] rx_stats + # - [leave as-is] ap_stats + # - [TODO/AC] ip_user + # - [TODO/AC] hw_user + # - [leave as-is] user_badges + # - [leave as-is] user_tourmnt_badges + # - [leave as-is] user_achievements + # - [transfer perms if owner & kick] clans + # - [leave as-is] identity_tokens + # - [leave as-is] irc_tokens + # - [TODO/AC] lastfm_flags + # - [leave as-is] beatmaps_rating + # - [leave as-is] clan_requests (empty?) + # - [leave as-is] comments + # - [leave as-is] matches + # - [leave as-is] match_events + # - [leave as-is] match_games + # - [leave as-is] match_game_scores + # - [TODO/financial] notifications + # - [delete; key'd by username??] password_recovery + # - [TODO/AC] patcher_detections + # - [TODO/AC] patcher_token_logs + # - [TODO] profile_backgrounds (and filesystem data) + # - [TODO] rap_logs + # - [leave as-is] remember + # - [leave as-is] reports + # - [leave as-is] rework_queue + # - [leave as-is] rework_scores + # - [leave as-is] rework_stats + # - [leave as-is] scheduled_bans + # - [leave as-is] scores + # - [leave as-is] scores_ap + # - [leave as-is] scores_relax + # - [leave as-is] scores_first + # - [TODO/AC] score_submission_logs + # - [leave as-is] tokens + # - [leave as-is] user_relationships + # - [leave as-is] user_beatmaps + # - [leave as-is] user_favourites + # - [leave as-is] user_profile_history + # - [leave as-is] user_speedruns + # - [leave as-is] user_tokens + + # misc. + # - [anonymize] replay data for all scores + # - [TODO] youtube uploads + + # PII to focus on: + # - username / username aka + # - email + # - clan association + # - country + # - (potetnailly) user notes + # - (potentially) userpage content + + transaction = await app.state.database.transaction() + try: + user = await users.fetch_one_by_user_id(user_id) + if user is None: + return Error( + error_code=ErrorCode.NOT_FOUND, + user_feedback="User not found.", + ) + + if user.clan_id: + clan = await clans.fetch_one_by_clan_id(user.clan_id) + if clan is not None: + if user.id == clan.owner: + # transfer clan ownership to another member, if available + other_clan_members = sorted( + [ + u + for u in await users.fetch_many_by_clan_id(user.clan_id) + if u.id != user.id + ], + # XXX: heuristic; clan join date would be better + # but it is not something we currently store + key=lambda u: (u.privileges, u.latest_activity), + ) + if other_clan_members: + new_owner = other_clan_members[0] + await clans.update_owner(user.clan_id, new_owner.id) + else: + # no other members in the clan; just delete it + await clans.delete_one_by_clan_id(user.clan_id) + + await password_recovery.delete_many_by_username(user.username) + + # TODO: consider what ac data should be anonymized instead of wiped + await user_ip_associations.delete_many_by_user_id(user_id) + await user_hwid_associations.delete_many_by_user_id(user_id) + await lastfm_flags.delete_many_by_user_id(user_id) + # TODO: patcher_detections & patcher_token_logs + + # TODO: wipe or anonymize all replay data. + # probably a good idea to call scores-service + + # TODO: wipe all static content (screenshots, profile bgs, etc.) + # TODO: potentially wipe youtube uploads + + # last step of the process; remove all associated pii + # TODO: split this to make it more clear what's being done + # at the usecase layer + await users.anonymize_one_by_user_id(user_id) + + # TODO: (technically required) anonymize data in data backups + + # inform other systems of the user's deletion (or "ban") + await app.state.redis.publish("peppy:ban", str(user_id)) + + # TODO: make sure they're removed from leaderboards + except Exception: + logging.exception( + "Failed to process GDPR/CCPA user deletion request", + extra={"user_id": user_id}, + ) + await transaction.rollback() + return Error( + error_code=ErrorCode.INTERNAL_SERVER_ERROR, + user_feedback="Failed to process user deletion request.", + ) + else: + logging.info( + "Successfully processed GDPR/CCPA user deletion request", + # NOTE: intentionally not logging any pii + extra={"user_id": user_id}, + ) + await transaction.commit() + + return None diff --git a/requirements.txt b/requirements.txt index 302220a..0ac1cf3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ httpx python-dotenv python-json-logger pyyaml +redis uvicorn