From 2cd353b4365e00049b51511db78567c1e82d1a79 Mon Sep 17 00:00:00 2001 From: Aaron Luna Date: Fri, 26 Jan 2024 12:06:59 -0800 Subject: [PATCH 01/11] fix: :bug: mypy: fix normalization of character filter flag names --- app/schemas/enums/char_filter_flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/schemas/enums/char_filter_flags.py b/app/schemas/enums/char_filter_flags.py index ab30a3d..a7c1090 100644 --- a/app/schemas/enums/char_filter_flags.py +++ b/app/schemas/enums/char_filter_flags.py @@ -55,7 +55,7 @@ def __repr__(self) -> str: @property def normalized(self) -> str: - return normalize_string_lm3(self.name) + return normalize_string_lm3(str(self.name)) @property def flag_name(self) -> str: From b36d043ce48c8f34ba49057f6f05e954d4cd4a5e Mon Sep 17 00:00:00 2001 From: Aaron Luna Date: Fri, 26 Jan 2024 12:07:46 -0800 Subject: [PATCH 02/11] refactor: :recycle: refactor imports and update function names in session.py --- app/db/session.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/db/session.py b/app/db/session.py index 4cd255a..7a78ad6 100644 --- a/app/db/session.py +++ b/app/db/session.py @@ -3,9 +3,10 @@ from sqlalchemy.engine import Engine from sqlmodel import Session -import app.db.procs.filter_characters as proc_filter_all_characters -import app.db.procs.get_char_details as proc_get_character_properties -import app.db.procs.get_unicode_versions as proc_get_all_unicode_versions +import app.db.procs.filter_characters as proc_filter +import app.db.procs.get_char_details as proc_char +import app.db.procs.get_unicode_versions as proc_ver +from app.config.api_settings import get_settings from app.db.engine import engine from app.schemas.enums import CharPropertyGroup @@ -22,14 +23,15 @@ class DBSession: def __init__(self, session: Session, engine: Engine): self.session = session self.engine = engine + self.api_settings = get_settings() def all_unicode_versions(self): - return proc_get_all_unicode_versions.get_all_unicode_versions(self.session) + return proc_ver.get_all_unicode_versions(self.session) def get_character_properties( self, codepoint: int, show_props: list[CharPropertyGroup] | None, verbose: bool ) -> dict[str, Any]: - return proc_get_character_properties.get_character_properties(self.engine, codepoint, show_props, verbose) + return proc_char.get_character_properties(self.engine, codepoint, show_props, verbose) def filter_all_characters(self, filter_params: "FilterParameters") -> list[int]: - return proc_filter_all_characters.filter_all_characters(self.session, filter_params) + return proc_filter.filter_all_characters(self.session, filter_params) From 9f36fdbe191b39d06a2ebf0f1a7ede6093df6c40 Mon Sep 17 00:00:00 2001 From: Aaron Luna Date: Fri, 26 Jan 2024 12:08:48 -0800 Subject: [PATCH 03/11] refactor: :recycle: refactor download_xml_unicode_database function --- app/data/scripts/update_all_data.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/data/scripts/update_all_data.py b/app/data/scripts/update_all_data.py index 854039a..04f9900 100644 --- a/app/data/scripts/update_all_data.py +++ b/app/data/scripts/update_all_data.py @@ -51,14 +51,14 @@ def update_all_data() -> Result[None]: def get_xml_unicode_database(config: UnicodeApiSettings) -> Result[Path]: spinner = Spinner() - get_xml_result = download_xml_unicode_database(config) - if get_xml_result.failure or not get_xml_result.value: + result = download_xml_unicode_database(config) + if result.failure or not result.value: spinner.start("") spinner.failed("Download failed! Please check the internet connection.") - return get_xml_result + return result spinner.start("") spinner.successful(f"Successfully downloaded Unicode XML Database v{config.UNICODE_VERSION}!") - xml_file = get_xml_result.value + xml_file = result.value return Result.Ok(xml_file) From 070ac9392765bdfe20dbccbfd2a1a61a94c60cf2 Mon Sep 17 00:00:00 2001 From: Aaron Luna Date: Fri, 26 Jan 2024 12:10:11 -0800 Subject: [PATCH 04/11] fix: :bug: mypy: refactor UnicodeModel type in script_types.py --- app/data/scripts/script_types.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/data/scripts/script_types.py b/app/data/scripts/script_types.py index 3205e27..04a3c06 100644 --- a/app/data/scripts/script_types.py +++ b/app/data/scripts/script_types.py @@ -3,6 +3,4 @@ CharDetailsDict = dict[str, bool | int | str] BlockOrPlaneDetailsDict = dict[str, int | str] AllParsedUnicodeData = tuple[list[BlockOrPlaneDetailsDict], list[BlockOrPlaneDetailsDict], list[CharDetailsDict]] -UnicodeModel = ( - type[db.UnicodePlane] | type[db.UnicodeBlock] | type[db.UnicodeCharacter] | type[db.UnicodeCharacterUnihan] -) +UnicodeModel = db.UnicodePlane | db.UnicodeBlock | db.UnicodeCharacter | db.UnicodeCharacterUnihan From 774d050ae85cbcd9826fc2a5497e41bb8d1dde0c Mon Sep 17 00:00:00 2001 From: Aaron Luna Date: Fri, 26 Jan 2024 12:11:05 -0800 Subject: [PATCH 05/11] refactor: :recycle: refactor get_unicode_version_release_date function --- app/core/util.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/core/util.py b/app/core/util.py index 948501c..a1588fb 100644 --- a/app/core/util.py +++ b/app/core/util.py @@ -7,9 +7,11 @@ def get_unicode_version_release_date(version: str) -> str: - if release_date := UNICODE_VERSION_RELEASE_DATES.get(version, None): - return release_date.strftime(DATE_MONTH_NAME) - return "" + return ( + release_date.strftime(DATE_MONTH_NAME) + if (release_date := UNICODE_VERSION_RELEASE_DATES.get(version, None)) + else "" + ) def make_tzaware(dt: datetime, use_tz: tzinfo | None = None, localize: bool = True) -> datetime: @@ -57,9 +59,9 @@ def format_timedelta_str(td: timedelta, precise: bool = True) -> str: return duration -def get_time_until_timestamp(ts: float, precise: bool = True) -> str: - return get_duration_between_timestamps(datetime.now().timestamp(), ts, precise) +def get_time_until_timestamp(ts: float) -> timedelta: + return get_duration_between_timestamps(datetime.now().timestamp(), ts) -def get_duration_between_timestamps(ts1: float, ts2: float, precise: bool = True) -> str: +def get_duration_between_timestamps(ts1: float, ts2: float) -> timedelta: return dtaware_fromtimestamp(ts2) - dtaware_fromtimestamp(ts1) From 3d80809998411c87166360827ae7341e2b0fe683 Mon Sep 17 00:00:00 2001 From: Aaron Luna Date: Fri, 26 Jan 2024 13:00:34 -0800 Subject: [PATCH 06/11] fix: :bug: mypy: create TypedDict for instantiating api settings dataclass --- app/config/api_settings.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/app/config/api_settings.py b/app/config/api_settings.py index 041b020..fb1269b 100644 --- a/app/config/api_settings.py +++ b/app/config/api_settings.py @@ -4,11 +4,29 @@ from dataclasses import dataclass, field from datetime import timedelta from pathlib import Path +from typing import TypedDict import app.db.models as db from app.config.dotenv_file import read_dotenv_file from app.data.constants import UNICODE_PLANES_DEFAULT, UNICODE_VERSION_RELEASE_DATES + +class ApiSettingsDict(TypedDict): + ENV: str + UNICODE_VERSION: str + PROJECT_NAME: str + API_VERSION: str + REDIS_PW: str + REDIS_HOST: str + REDIS_PORT: int + REDIS_DB: int + RATE_LIMIT_PER_PERIOD: int + RATE_LIMIT_PERIOD_SECONDS: timedelta + RATE_LIMIT_BURST: int + SERVER_NAME: str + SERVER_HOST: str + + UNICODE_ORG_ROOT = "https://www.unicode.org/Public" UNICODE_XML_FOLDER = "ucdxml" HTTP_BUCKET_URL = "https://unicode-api.us-southeast-1.linodeobjects.com" @@ -48,7 +66,6 @@ class UnicodeApiSettings: RATE_LIMIT_BURST: int SERVER_NAME: str SERVER_HOST: str - CACHE_HEADER: str API_ROOT: str = field(init=False, default="") ROOT_FOLDER: Path = field(init=False) APP_FOLDER: Path = field(init=False) @@ -85,7 +102,7 @@ def __post_init__(self) -> None: json_folder = version_folder.joinpath("json") csv_folder = version_folder.joinpath("csv") - self.API_ROOT = DEV_API_ROOT if "PROD" not in self.ENV else PROD_API_ROOT + self.API_ROOT = DEV_API_ROOT if self.is_dev else PROD_API_ROOT self.ROOT_FOLDER = ROOT_FOLDER self.APP_FOLDER = ROOT_FOLDER.joinpath("app") self.DATA_FOLDER = data_folder @@ -182,7 +199,7 @@ def init_data_folders(self) -> None: # pragma: no cover def get_api_settings() -> UnicodeApiSettings: # pragma: no cover env_vars = read_dotenv_file(DOTENV_FILE) - settings = { + settings: ApiSettingsDict = { "ENV": env_vars.get("ENV", "DEV"), "UNICODE_VERSION": env_vars.get("UNICODE_VERSION", get_latest_unicode_version()), "PROJECT_NAME": "Unicode API", @@ -196,13 +213,12 @@ def get_api_settings() -> UnicodeApiSettings: # pragma: no cover "RATE_LIMIT_BURST": int(env_vars.get("RATE_LIMIT_BURST", "10")), "SERVER_NAME": "unicode-api.aaronluna.dev", "SERVER_HOST": PROD_API_ROOT, - "CACHE_HEADER": "X-UnicodeAPI-Cache", } return UnicodeApiSettings(**settings) def get_test_settings() -> UnicodeApiSettings: - settings = { + settings: ApiSettingsDict = { "ENV": "TEST", "UNICODE_VERSION": "15.0.0", "PROJECT_NAME": "Test Unicode API", @@ -216,14 +232,12 @@ def get_test_settings() -> UnicodeApiSettings: "RATE_LIMIT_BURST": 1, "SERVER_NAME": "", "SERVER_HOST": "", - "CACHE_HEADER": "", } return UnicodeApiSettings(**settings) def get_settings() -> UnicodeApiSettings: - env = os.environ.get("ENV", "DEV") - settings = get_test_settings() if "TEST" in env else get_api_settings() + settings = get_test_settings() if "TEST" in os.environ.get("ENV", "DEV") else get_api_settings() logger = logging.getLogger("app.api") logger.debug(settings.api_settings_report) logger.debug(settings.rate_limit_settings_report) From d11c1f4f23f1f5bc2a0a182ee8735cbad35fbd67 Mon Sep 17 00:00:00 2001 From: Aaron Luna Date: Fri, 26 Jan 2024 13:01:42 -0800 Subject: [PATCH 07/11] refactor: :recycle: update api version in character endpoints --- app/api/api_v1/endpoints/characters.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/api/api_v1/endpoints/characters.py b/app/api/api_v1/endpoints/characters.py index 930bb3e..05d40af 100644 --- a/app/api/api_v1/endpoints/characters.py +++ b/app/api/api_v1/endpoints/characters.py @@ -12,7 +12,6 @@ from app.api.api_v1.dependencies.filter_param_matcher import filter_param_matcher from app.api.api_v1.endpoints.util import get_character_details from app.api.api_v1.pagination import paginate_search_results -from app.config.api_settings import get_settings from app.data.cache import cached_data from app.data.encoding import get_codepoint_string from app.db.session import DBSession, get_session @@ -38,7 +37,7 @@ def list_all_unicode_characters( ): (start, stop) = get_char_list_endpoints(list_params, block) return { - "url": f"{get_settings().API_VERSION}/characters", + "url": f"{db_ctx.api_settings.API_VERSION}/characters", "has_more": stop <= block.finish, "data": [get_character_details(db_ctx, codepoint, []) for codepoint in range(start, stop)], } @@ -53,7 +52,7 @@ def search_unicode_characters_by_name( db_ctx: Annotated[DBSession, Depends(get_session)], search_parameters: Annotated[CharacterSearchParameters, Depends()], ): - response_data = {"url": f"{get_settings().API_VERSION}/characters/search", "query": search_parameters.name} + response_data = {"url": f"{db_ctx.api_settings.API_VERSION}/characters/search", "query": search_parameters.name} search_results = cached_data.search_characters_by_name(search_parameters.name, search_parameters.min_score) return get_paginated_character_list( db_ctx, @@ -80,7 +79,7 @@ def filter_unicode_characters( detail="No filter settings were specified in the request.", ) response_data = { - "url": f"{get_settings().API_VERSION}/characters/filter", + "url": f"{db_ctx.api_settings.API_VERSION}/characters/filter", "filter_settings": filter_parameters.settings, } codepoints = db_ctx.filter_all_characters(filter_parameters) From 4a7e5f89a3d74d76c7147008d54ed3eca72cc42f Mon Sep 17 00:00:00 2001 From: Aaron Luna Date: Fri, 26 Jan 2024 13:02:28 -0800 Subject: [PATCH 08/11] chore: :coffin: remove unused TEST_HEADER environment variable --- Dockerfile | 2 -- docker-bake.hcl | 1 - 2 files changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ae23766..654c28f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,6 @@ ENV REDIS_PW=${REDIS_PW} ENV RATE_LIMIT_PER_PERIOD=${RATE_LIMIT_PER_PERIOD} ENV RATE_LIMIT_PERIOD_SECONDS=${RATE_LIMIT_PERIOD_SECONDS} ENV RATE_LIMIT_BURST=${RATE_LIMIT_BURST} -ENV TEST_HEADER=${TEST_HEADER} WORKDIR /code @@ -36,7 +35,6 @@ RUN echo "REDIS_PW=$REDIS_PW" >> /code/.env RUN echo "RATE_LIMIT_PER_PERIOD=$RATE_LIMIT_PER_PERIOD" >> /code/.env RUN echo "RATE_LIMIT_PERIOD_SECONDS=$RATE_LIMIT_PERIOD_SECONDS" >> /code/.env RUN echo "RATE_LIMIT_BURST=$RATE_LIMIT_BURST" >> /code/.env -RUN echo "TEST_HEADER=$TEST_HEADER" >> /code/.env RUN pip install -U pip setuptools wheel COPY ./requirements.txt /code/requirements.txt diff --git a/docker-bake.hcl b/docker-bake.hcl index 7c6ef30..8cbccf5 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -23,6 +23,5 @@ target "unicode-api" { RATE_LIMIT_PER_PERIOD="50" RATE_LIMIT_PERIOD_SECONDS="60" RATE_LIMIT_BURST="10" - TEST_HEADER="X-UnicodeAPI-Test" } } From d811aa6d86b0fa729775a6c9adcbf334ef9a3d77 Mon Sep 17 00:00:00 2001 From: Aaron Luna Date: Fri, 26 Jan 2024 13:03:06 -0800 Subject: [PATCH 09/11] chore: :arrow_up: update black version to 24.1.0 --- requirements-lock.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-lock.txt b/requirements-lock.txt index f854a13..c8d2ed1 100644 --- a/requirements-lock.txt +++ b/requirements-lock.txt @@ -7,7 +7,7 @@ asttokens==2.4.1 async-timeout==4.0.3 attrs==23.2.0 backcall==0.2.0 -black==23.12.1 +black==24.1.0 cachetools==5.3.2 certifi==2023.11.17 chardet==5.2.0 From 5f46e123922e36f32ad908d217f67dafd495f41b Mon Sep 17 00:00:00 2001 From: Aaron Luna Date: Fri, 26 Jan 2024 23:50:55 -0800 Subject: [PATCH 10/11] fix: :bug: fix logic for applying rate limit conditionally --- app/core/rate_limit.py | 26 +++++++++---------- .../test_rate_limiting/test_rate_limiting.py | 17 ------------ 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/app/core/rate_limit.py b/app/core/rate_limit.py index a3cc34d..56edb53 100644 --- a/app/core/rate_limit.py +++ b/app/core/rate_limit.py @@ -81,7 +81,7 @@ def is_exceeded(self, request: Request) -> Result[None]: Adapted for Python from this article: https://vikas-kumar.medium.com/rate-limiting-techniques-245c3a5e9cad """ - if self.rate_limit_not_required(request): + if not self.apply_rate_limit_to_request(request): return Result.Ok() client_ip = request.client.host if request.client else "localhost" arrived_at = self.redis.time() @@ -100,10 +100,10 @@ def is_exceeded(self, request: Request) -> Result[None]: except LockError: # pragma: no cover return self.lock_error(client_ip) - def rate_limit_not_required(self, request: Request): - if self.settings.is_prod or self.settings.is_dev: # pragma: no cover - return request_origin_is_internal(request) and requested_route_is_not_rate_limited(request) - return not rate_limit_feature_is_under_test(request) + def apply_rate_limit_to_request(self, request: Request): + if self.settings.is_test: + return enable_rate_limit_feature_for_test(request) + return request_origin_is_external(request) and requested_route_is_rate_limited(request) def get_allowed_at(self, tat: float) -> float: return (dtaware_fromtimestamp(tat) - self.delay_tolerance_ms).timestamp() @@ -135,7 +135,7 @@ def lock_error(self, client) -> Result[None]: # pragma: no cover return Result.Fail(error) -def rate_limit_feature_is_under_test(request: Request) -> bool: +def enable_rate_limit_feature_for_test(request: Request) -> bool: if "x-verify-rate-limiting" in request.headers: return request.headers["x-verify-rate-limiting"] == "true" if "access-control-request-headers" in request.headers: # pragma: no cover @@ -145,16 +145,16 @@ def rate_limit_feature_is_under_test(request: Request) -> bool: return False # pragma: no cover -def request_origin_is_internal(request: Request) -> bool: # pragma: no cover - if "localhost" in request.client.host: - return True +def request_origin_is_external(request: Request) -> bool: + if request.client.host in ["localhost", "127.0.0.1", "testserver"]: + return False if "sec-fetch-site" in request.headers: - return request.headers["sec-fetch-site"] == "same-site" - return False + return request.headers["sec-fetch-site"] != "same-site" + return True -def requested_route_is_not_rate_limited(request: Request): # pragma: no cover - return not RATE_LIMIT_ROUTE_REGEX.search(request.url.path) +def requested_route_is_rate_limited(request: Request): + return RATE_LIMIT_ROUTE_REGEX.search(request.url.path) def get_time_portion(ts: float) -> str: diff --git a/app/tests/test_rate_limiting/test_rate_limiting.py b/app/tests/test_rate_limiting/test_rate_limiting.py index 59c0a43..2ef1ddb 100644 --- a/app/tests/test_rate_limiting/test_rate_limiting.py +++ b/app/tests/test_rate_limiting/test_rate_limiting.py @@ -1,23 +1,6 @@ -import os - -import pytest -from fastapi.testclient import TestClient - from app.tests.test_rate_limiting.data import PLANE_0 -@pytest.fixture -def client(): - os.environ["ENV"] = "TEST" - from app.main import app - - with TestClient(app) as client: - headers = {} - headers["x-verify-rate-limiting"] = "true" - client.headers = headers - yield client - - def test_rate_limiting(client): response = client.get("/v1/planes/0") assert response.status_code == 200 From a1f56958762d53d0c8ac285d589b5cc0b94bb9ad Mon Sep 17 00:00:00 2001 From: Aaron Luna Date: Fri, 26 Jan 2024 23:51:49 -0800 Subject: [PATCH 11/11] chore: :loud_sound: update logging format to include [unicode-api] prefix --- app/core/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/core/logging.py b/app/core/logging.py index 52c46e2..54f93df 100644 --- a/app/core/logging.py +++ b/app/core/logging.py @@ -13,7 +13,7 @@ "formatters": { "default": { "()": "app.core.logging.DefaultFormatter", - "fmt": "%(levelprefix)s %(message)s", + "fmt": "%(levelprefix)s [unicode-api] %(message)s", "use_colors": True, }, },