From 609eb45d7a13b40f5604c5ecc8825183ee85d312 Mon Sep 17 00:00:00 2001 From: Emmanuel Evbuomwan Date: Fri, 22 Nov 2024 12:28:28 +0100 Subject: [PATCH] feat: standalone module - we create a new module schema_registry - we move all FastAPI related code into this new folder - going forward, move out all SR related things into this new folder --- .coveragerc | 2 +- container/compose.yml | 17 +- container/karapace.env | 46 +++++ container/start.sh | 18 +- mypy.ini | 2 +- pyproject.toml | 4 + requirements/requirements-dev.txt | 144 +++++++++++++-- requirements/requirements-typing.txt | 141 ++++++++++++-- requirements/requirements.txt | 138 +++++++++++++- src/karapace/auth/auth.py | 4 +- .../dependencies/controller_dependency.py | 2 +- src/karapace/karapace_all.py | 172 ++++-------------- src/karapace/logging.py | 45 +++++ src/karapace/routers/__init__.py | 0 src/schema_registry/__init__.py | 4 + src/schema_registry/__main__.py | 65 +++++++ src/schema_registry/container.py | 4 + src/schema_registry/http_handlers/__init__.py | 34 ++++ src/schema_registry/middlewares/__init__.py | 33 ++++ src/schema_registry/routers/__init__.py | 23 +++ .../routers/compatibility_router.py | 4 +- .../routers/config_router.py | 4 +- .../routers/errors.py | 14 ++ .../routers/health_router.py | 0 .../routers/mode_router.py | 2 +- .../routers/requests.py | 2 +- .../routers/root_router.py | 0 .../routers/schemas_router.py | 2 +- .../routers/subjects_router.py | 4 +- .../schema_registry_apis.py | 57 +----- tests/integration/test_schema.py | 2 +- tests/unit/test_schema_registry_api.py | 4 +- 32 files changed, 722 insertions(+), 271 deletions(-) create mode 100644 container/karapace.env create mode 100644 src/karapace/logging.py delete mode 100644 src/karapace/routers/__init__.py create mode 100644 src/schema_registry/__init__.py create mode 100644 src/schema_registry/__main__.py create mode 100644 src/schema_registry/container.py create mode 100644 src/schema_registry/http_handlers/__init__.py create mode 100644 src/schema_registry/middlewares/__init__.py create mode 100644 src/schema_registry/routers/__init__.py rename src/{karapace => schema_registry}/routers/compatibility_router.py (88%) rename src/{karapace => schema_registry}/routers/config_router.py (95%) rename src/{karapace => schema_registry}/routers/errors.py (75%) rename src/{karapace => schema_registry}/routers/health_router.py (100%) rename src/{karapace => schema_registry}/routers/mode_router.py (95%) rename src/{karapace => schema_registry}/routers/requests.py (97%) rename src/{karapace => schema_registry}/routers/root_router.py (100%) rename src/{karapace => schema_registry}/routers/schemas_router.py (96%) rename src/{karapace => schema_registry}/routers/subjects_router.py (96%) rename src/{karapace => schema_registry}/schema_registry_apis.py (96%) diff --git a/.coveragerc b/.coveragerc index 2a6a5d055..8232ff3b7 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,4 +1,4 @@ [run] branch = true relative_files = true -source = src/karapace +source = src diff --git a/container/compose.yml b/container/compose.yml index fa2c53265..f17c21eb3 100644 --- a/container/compose.yml +++ b/container/compose.yml @@ -67,21 +67,10 @@ services: - kafka ports: - "8081:8081" + volumes: + - ./karapace.env:/opt/karapace/karapace.env environment: - KARAPACE_ADVERTISED_HOSTNAME: karapace-registry - KARAPACE_BOOTSTRAP_URI: kafka:29092 - KARAPACE_PORT: 8081 - KARAPACE_HOST: 0.0.0.0 - KARAPACE_CLIENT_ID: karapace - KARAPACE_GROUP_ID: karapace-registry - KARAPACE_MASTER_ELIGIBILITY: "true" - KARAPACE_TOPIC_NAME: _schemas - KARAPACE_LOG_LEVEL: WARNING - KARAPACE_COMPATIBILITY: FULL - KARAPACE_STATSD_HOST: statsd-exporter - KARAPACE_STATSD_PORT: 8125 - KARAPACE_KAFKA_SCHEMA_READER_STRICT_MODE: false - KARAPACE_KAFKA_RETRIABLE_ERRORS_SILENCED: true + KARAPACE_DOTENV: /opt/karapace/karapace.env karapace-rest: image: ghcr.io/aiven-open/karapace:develop diff --git a/container/karapace.env b/container/karapace.env new file mode 100644 index 000000000..6cee31a2e --- /dev/null +++ b/container/karapace.env @@ -0,0 +1,46 @@ +KARAPACE_DOTENV=/opt/karapace/karapace.env +ACCESS_LOGS_DEBUG=False +ADVERTISED_HOSTNAME=karapace-schema-registry +ADVERTISED_PORT=8081 +ADVERTISED_PROTOCOL=http +BOOTSTRAP_URI=kafka:29092 +CLIENT_ID=karapace-schema-registry +COMPATIBILITY=BACKWARD +CONNECTIONS_MAX_IDLE_MS=15000 +CONSUMER_ENABLE_AUTO_COMMIT=True +CONSUMER_REQUEST_TIMEOUT_MS=11000 +CONSUMER_REQUEST_MAX_BYTES=67108864 +CONSUMER_IDLE_DISCONNECT_TIMEOUT=0 +FETCH_MIN_BYTES=1 +GROUP_ID=karapace-schema-registry +HOST=0.0.0.0 +PORT=8081 +REGISTRY_HOST=karapace-registry +REGISTRY_PORT=8081 +REST_AUTHORIZATION=False +LOG_HANDLER=stdout +LOG_LEVEL=DEBUG +LOG_FORMAT=%(asctime)s [%(threadName)s] %(filename)s:%(funcName)s:%(lineno)d %(message)s +MASTER_ELIGIBILITY=True +REPLICATION_FACTOR=1 +SECURITY_PROTOCOL=PLAINTEXT +SSL_CHECK_HOSTNAME=True +TOPIC_NAME=_schemas +METADATA_MAX_AGE_MS=60000 +ADMIN_METADATA_MAX_AGE=5 +PRODUCER_ACKS=1 +PRODUCER_COUNT=5 +PRODUCER_LINGER_MS=100 +PRODUCER_MAX_REQUEST_SIZE=1048576 +SESSION_TIMEOUT_MS=10000 +KARAPACE_REST=False +KARAPACE_REGISTRY=True +NAME_STRATEGY=topic_name +NAME_STRATEGY_VALIDATION=True +MASTER_ELECTION_STRATEGY=lowest +PROTOBUF_RUNTIME_DIRECTORY=runtime +STATSD_HOST=statsd-exporter +STATSD_PORT=8125 +KAFKA_SCHEMA_READER_STRICT_MODE=False +KAFKA_RETRIABLE_ERRORS_SILENCED=True +USE_PROTOBUF_FORMATTER=False diff --git a/container/start.sh b/container/start.sh index 95ac86aa2..f65dcfb31 100755 --- a/container/start.sh +++ b/container/start.sh @@ -28,24 +28,8 @@ rest) exec python3 -m karapace.karapace_all /opt/karapace/rest.config.json ;; registry) - # Reexport variables for compatibility - [[ -n ${KARAPACE_REGISTRY_ADVERTISED_HOSTNAME+isset} ]] && export KARAPACE_ADVERTISED_HOSTNAME="${KARAPACE_REGISTRY_ADVERTISED_HOSTNAME}" - [[ -n ${KARAPACE_REGISTRY_BOOTSTRAP_URI+isset} ]] && export KARAPACE_BOOTSTRAP_URI="${KARAPACE_REGISTRY_BOOTSTRAP_URI}" - [[ -n ${KARAPACE_REGISTRY_HOST+isset} ]] && export KARAPACE_HOST="${KARAPACE_REGISTRY_HOST}" - [[ -n ${KARAPACE_REGISTRY_PORT+isset} ]] && export KARAPACE_PORT="${KARAPACE_REGISTRY_PORT}" - [[ -n ${KARAPACE_REGISTRY_CLIENT_ID+isset} ]] && export KARAPACE_CLIENT_ID="${KARAPACE_REGISTRY_CLIENT_ID}" - [[ -n ${KARAPACE_REGISTRY_GROUP_ID+isset} ]] && export KARAPACE_GROUP_ID="${KARAPACE_REGISTRY_GROUP_ID}" - # Map misspelled environment variables to correct spelling for backwards compatibility. - [[ -n ${KARAPACE_REGISTRY_MASTER_ELIGIBITY+isset} ]] && export KARAPACE_MASTER_ELIGIBILITY="${KARAPACE_REGISTRY_MASTER_ELIGIBITY}" - [[ -n ${KARAPACE_REGISTRY_MASTER_ELIGIBILITY+isset} ]] && export KARAPACE_MASTER_ELIGIBILITY="${KARAPACE_REGISTRY_MASTER_ELIGIBILITY}" - [[ -n ${KARAPACE_REGISTRY_TOPIC_NAME+isset} ]] && export KARAPACE_TOPIC_NAME="${KARAPACE_REGISTRY_TOPIC_NAME}" - [[ -n ${KARAPACE_REGISTRY_COMPATIBILITY+isset} ]] && export KARAPACE_COMPATIBILITY="${KARAPACE_REGISTRY_COMPATIBILITY}" - [[ -n ${KARAPACE_REGISTRY_LOG_LEVEL+isset} ]] && export KARAPACE_LOG_LEVEL="${KARAPACE_REGISTRY_LOG_LEVEL}" - export KARAPACE_REGISTRY=1 - echo "{}" >/opt/karapace/registry.config.json - echo "Starting Karapace Schema Registry" - exec python3 -m karapace.karapace_all /opt/karapace/registry.config.json + exec python3 -m schema_registry ;; *) echo "usage: start-karapace.sh " diff --git a/mypy.ini b/mypy.ini index c4ef8efd1..30d56b0bc 100644 --- a/mypy.ini +++ b/mypy.ini @@ -15,7 +15,7 @@ warn_no_return = True warn_unreachable = True strict_equality = True -[mypy-karapace.schema_registry_apis] +[mypy-schema_registry.schema_registry_apis] ignore_errors = True [mypy-karapace.compatibility.jsonschema.checks] diff --git a/pyproject.toml b/pyproject.toml index f1f9016cb..c5bad6d99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,10 @@ dependencies = [ "zstandard", "prometheus-client == 0.20.0", "yarl == 1.12.1", + "opentelemetry-api == 1.28.2", + "opentelemetry-sdk == 1.28.2", + "opentelemetry-instrumentation-fastapi == 0.49b2", + "dependency-injector == 4.43.0", # Patched dependencies # diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 9848f80e0..c749d0098 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -8,15 +8,20 @@ accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -38,20 +43,35 @@ cachetools==5.3.3 certifi==2024.8.30 # via # geventhttpclient + # httpcore + # httpx # requests # sentry-sdk charset-normalizer==3.4.0 # via requests click==8.1.7 - # via flask + # via + # flask + # typer + # uvicorn configargparse==1.7 # via locust confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) -coverage[toml]==7.6.4 +coverage[toml]==7.6.7 # via pytest-cov cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via # anyio @@ -61,9 +81,13 @@ execnet==2.1.1 # via pytest-xdist fancycompleter==0.9.1 # via pdbpp +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.5 + # via fastapi filelock==3.16.1 # via karapace (/karapace/pyproject.toml) -flask==3.0.3 +flask==3.1.0 # via # flask-cors # flask-login @@ -84,15 +108,29 @@ geventhttpclient==2.3.1 # via locust greenlet==3.1.1 # via gevent -hypothesis==6.118.8 +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.27.2 + # via fastapi +hypothesis==6.119.4 # via karapace (/karapace/pyproject.toml) idna==3.10 # via # anyio + # email-validator + # httpx # requests # yarl importlib-metadata==8.5.0 - # via flask + # via + # flask + # opentelemetry-api iniconfig==2.0.0 # via pytest isodate==0.7.2 @@ -100,12 +138,14 @@ isodate==0.7.2 itsdangerous==2.2.0 # via flask jinja2==3.1.4 - # via flask + # via + # fastapi + # flask jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 # via jsonschema -locust==2.32.2 +locust==2.32.3 # via karapace (/karapace/pyproject.toml) lz4==4.3.3 # via karapace (/karapace/pyproject.toml) @@ -125,9 +165,38 @@ multidict==6.1.0 # yarl networkx==3.2.1 # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 # via # aiokafka + # opentelemetry-instrumentation # pytest pdbpp==0.10.3 # via karapace (/karapace/pyproject.toml) @@ -142,11 +211,15 @@ psutil==6.1.0 # karapace (/karapace/pyproject.toml) # locust # pytest-xdist +pydantic==1.10.17 + # via + # fastapi + # karapace (/karapace/pyproject.toml) pygments==2.18.0 # via # pdbpp # rich -pyjwt==2.9.0 +pyjwt==2.10.0 # via karapace (/karapace/pyproject.toml) pyrepl==0.9.0 # via fancycompleter @@ -167,8 +240,14 @@ pytest-xdist[psutil]==3.6.1 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via uvicorn +python-multipart==0.0.17 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn pyzmq==26.2.0 # via locust referencing==0.35.1 @@ -180,19 +259,29 @@ requests==2.32.3 # karapace (/karapace/pyproject.toml) # locust rich==13.7.1 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # typer rpds-py==0.21.0 # via # jsonschema # referencing -sentry-sdk==2.18.0 +sentry-sdk==2.19.0 # via karapace (/karapace/pyproject.toml) +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 - # via anyio + # via + # anyio + # httpx sortedcontainers==2.4.0 # via hypothesis +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) tomli==2.1.0 @@ -200,12 +289,21 @@ tomli==2.1.0 # coverage # locust # pytest +typer==0.13.1 + # via fastapi-cli typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # locust # multidict + # opentelemetry-sdk + # pydantic + # starlette + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) urllib3==2.2.3 @@ -213,8 +311,18 @@ urllib3==2.2.3 # geventhttpclient # requests # sentry-sdk +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn werkzeug==3.1.3 # via # flask @@ -222,6 +330,10 @@ werkzeug==3.1.3 # locust wmctrl==0.5 # via pdbpp +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 diff --git a/requirements/requirements-typing.txt b/requirements/requirements-typing.txt index aef63ee86..b705e7cc8 100644 --- a/requirements/requirements-typing.txt +++ b/requirements/requirements-typing.txt @@ -8,15 +8,20 @@ accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -30,23 +35,60 @@ avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c cachetools==5.3.3 # via karapace (/karapace/pyproject.toml) certifi==2024.8.30 - # via sentry-sdk + # via + # httpcore + # httpx + # sentry-sdk +click==8.1.7 + # via + # typer + # uvicorn confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via anyio +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.5 + # via fastapi frozenlist==1.5.0 # via # aiohttp # aiosignal +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.27.2 + # via fastapi idna==3.10 # via # anyio + # email-validator + # httpx # yarl +importlib-metadata==8.5.0 + # via opentelemetry-api isodate==0.7.2 # via karapace (/karapace/pyproject.toml) +jinja2==3.1.4 + # via fastapi jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 @@ -55,6 +97,8 @@ lz4==4.3.3 # via karapace (/karapace/pyproject.toml) markdown-it-py==3.0.0 # via rich +markupsafe==3.0.2 + # via jinja2 mdurl==0.1.2 # via markdown-it-py multidict==6.1.0 @@ -67,41 +111,93 @@ mypy-extensions==1.0.0 # via mypy networkx==3.2.1 # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 - # via aiokafka + # via + # aiokafka + # opentelemetry-instrumentation prometheus-client==0.20.0 # via karapace (/karapace/pyproject.toml) protobuf==3.20.3 # via karapace (/karapace/pyproject.toml) +pydantic==1.10.17 + # via + # fastapi + # karapace (/karapace/pyproject.toml) pygments==2.18.0 # via rich -pyjwt==2.9.0 +pyjwt==2.10.0 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via uvicorn +python-multipart==0.0.17 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn referencing==0.35.1 # via # jsonschema # jsonschema-specifications # types-jsonschema rich==13.7.1 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # typer rpds-py==0.21.0 # via # jsonschema # referencing -sentry-sdk==2.18.0 +sentry-sdk==2.19.0 # via karapace (/karapace/pyproject.toml) +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 - # via anyio + # via + # anyio + # httpx +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) tomli==2.1.0 # via mypy +typer==0.13.1 + # via fastapi-cli types-cachetools==5.5.0.20240820 # via karapace (/karapace/pyproject.toml) types-jsonschema==4.23.0.20240813 @@ -111,20 +207,43 @@ types-protobuf==3.20.4.6 typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # multidict # mypy + # opentelemetry-sdk + # pydantic + # starlette + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) urllib3==2.2.3 # via sentry-sdk +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 # via # aiohttp # karapace (/karapace/pyproject.toml) +zipp==3.21.0 + # via importlib-metadata zstandard==0.23.0 # via karapace (/karapace/pyproject.toml) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 5bb9cf22e..52ed6bde3 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -8,15 +8,20 @@ accept-types==0.4.1 # via karapace (/karapace/pyproject.toml) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.10.11 # via karapace (/karapace/pyproject.toml) aiokafka==0.10.0 # via karapace (/karapace/pyproject.toml) aiosignal==1.3.1 # via aiohttp anyio==4.6.2.post1 - # via watchfiles -async-timeout==4.0.3 + # via + # httpx + # starlette + # watchfiles +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +async-timeout==5.0.1 # via # aiohttp # aiokafka @@ -29,22 +34,60 @@ avro @ https://github.com/aiven/avro/archive/5a82d57f2a650fd87c819a30e433f1abb2c # via karapace (/karapace/pyproject.toml) cachetools==5.3.3 # via karapace (/karapace/pyproject.toml) +certifi==2024.8.30 + # via + # httpcore + # httpx +click==8.1.7 + # via + # typer + # uvicorn confluent-kafka==2.4.0 # via karapace (/karapace/pyproject.toml) cramjam==2.9.0 # via python-snappy +dependency-injector==4.43.0 + # via karapace (/karapace/pyproject.toml) +deprecated==1.2.15 + # via + # opentelemetry-api + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via fastapi exceptiongroup==1.2.2 # via anyio +fastapi[standard]==0.115.5 + # via karapace (/karapace/pyproject.toml) +fastapi-cli[standard]==0.0.5 + # via fastapi frozenlist==1.5.0 # via # aiohttp # aiosignal +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httptools==0.6.4 + # via uvicorn +httpx==0.27.2 + # via fastapi idna==3.10 # via # anyio + # email-validator + # httpx # yarl +importlib-metadata==8.5.0 + # via opentelemetry-api isodate==0.7.2 # via karapace (/karapace/pyproject.toml) +jinja2==3.1.4 + # via fastapi jsonschema==4.23.0 # via karapace (/karapace/pyproject.toml) jsonschema-specifications==2024.10.1 @@ -53,6 +96,8 @@ lz4==4.3.3 # via karapace (/karapace/pyproject.toml) markdown-it-py==3.0.0 # via rich +markupsafe==3.0.2 + # via jinja2 mdurl==0.1.2 # via markdown-it-py multidict==6.1.0 @@ -61,50 +106,125 @@ multidict==6.1.0 # yarl networkx==3.2.1 # via karapace (/karapace/pyproject.toml) +opentelemetry-api==1.28.2 + # via + # karapace (/karapace/pyproject.toml) + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-instrumentation==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asgi==0.49b2 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.49b2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-sdk==1.28.2 + # via karapace (/karapace/pyproject.toml) +opentelemetry-semantic-conventions==0.49b2 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.49b2 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi packaging==24.2 - # via aiokafka + # via + # aiokafka + # opentelemetry-instrumentation prometheus-client==0.20.0 # via karapace (/karapace/pyproject.toml) protobuf==3.20.3 # via karapace (/karapace/pyproject.toml) +pydantic==1.10.17 + # via + # fastapi + # karapace (/karapace/pyproject.toml) pygments==2.18.0 # via rich -pyjwt==2.9.0 +pyjwt==2.10.0 # via karapace (/karapace/pyproject.toml) python-dateutil==2.9.0.post0 # via karapace (/karapace/pyproject.toml) +python-dotenv==1.0.1 + # via uvicorn +python-multipart==0.0.17 + # via fastapi python-snappy==0.7.3 # via karapace (/karapace/pyproject.toml) +pyyaml==6.0.2 + # via uvicorn referencing==0.35.1 # via # jsonschema # jsonschema-specifications rich==13.7.1 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # typer rpds-py==0.21.0 # via # jsonschema # referencing +shellingham==1.5.4 + # via typer six==1.16.0 - # via python-dateutil + # via + # dependency-injector + # python-dateutil sniffio==1.3.1 - # via anyio + # via + # anyio + # httpx +starlette==0.41.3 + # via fastapi tenacity==9.0.0 # via karapace (/karapace/pyproject.toml) +typer==0.13.1 + # via fastapi-cli typing-extensions==4.12.2 # via # anyio + # asgiref + # fastapi # karapace (/karapace/pyproject.toml) # multidict + # opentelemetry-sdk + # pydantic + # starlette + # typer + # uvicorn ujson==5.10.0 # via karapace (/karapace/pyproject.toml) +uvicorn[standard]==0.32.1 + # via + # fastapi + # fastapi-cli +uvloop==0.21.0 + # via uvicorn watchfiles==0.24.0 - # via karapace (/karapace/pyproject.toml) + # via + # karapace (/karapace/pyproject.toml) + # uvicorn +websockets==14.1 + # via uvicorn +wrapt==1.17.0 + # via + # deprecated + # opentelemetry-instrumentation xxhash==3.5.0 # via karapace (/karapace/pyproject.toml) yarl==1.12.1 # via # aiohttp # karapace (/karapace/pyproject.toml) +zipp==3.21.0 + # via importlib-metadata zstandard==0.23.0 # via karapace (/karapace/pyproject.toml) diff --git a/src/karapace/auth/auth.py b/src/karapace/auth/auth.py index 5546a43a8..ccaab30e6 100644 --- a/src/karapace/auth/auth.py +++ b/src/karapace/auth/auth.py @@ -11,8 +11,8 @@ from karapace.config import InvalidConfiguration from karapace.statsd import StatsClient from karapace.utils import json_decode, json_encode -from typing import override, Protocol -from typing_extensions import TypedDict +from typing import Protocol +from typing_extensions import override, TypedDict from watchfiles import awatch, Change import argparse diff --git a/src/karapace/dependencies/controller_dependency.py b/src/karapace/dependencies/controller_dependency.py index e056b52c2..5e0f21e6b 100644 --- a/src/karapace/dependencies/controller_dependency.py +++ b/src/karapace/dependencies/controller_dependency.py @@ -8,7 +8,7 @@ from karapace.dependencies.config_dependency import ConfigDep from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep from karapace.dependencies.stats_dependeny import StatsDep -from karapace.schema_registry_apis import KarapaceSchemaRegistryController +from schema_registry.schema_registry_apis import KarapaceSchemaRegistryController from typing import Annotated diff --git a/src/karapace/karapace_all.py b/src/karapace/karapace_all.py index 7ae1a5f9c..c176bc337 100644 --- a/src/karapace/karapace_all.py +++ b/src/karapace/karapace_all.py @@ -4,160 +4,59 @@ """ from __future__ import annotations -from collections.abc import AsyncGenerator -from contextlib import asynccontextmanager -from fastapi import FastAPI, HTTPException, Request, status -from fastapi.exceptions import RequestValidationError -from fastapi.responses import JSONResponse -from http import HTTPStatus +from contextlib import closing from karapace import version as karapace_version -from karapace.auth.auth import AuthenticatorAndAuthorizer -from karapace.auth.dependencies import AuthorizationDependencyManager -from karapace.config import Config -from karapace.content_type import check_schema_headers -from karapace.dependencies.config_dependency import ConfigDependencyManager -from karapace.dependencies.schema_registry_dependency import SchemaRegistryDependencyManager -from karapace.dependencies.stats_dependeny import StatsDependencyManager +from karapace.config import read_config from karapace.instrumentation.prometheus import PrometheusInstrumentation -from karapace.routers.errors import KarapaceValidationError -from karapace.schema_registry import KarapaceSchemaRegistry -from starlette.exceptions import HTTPException as StarletteHTTPException -from starlette.requests import Request as StarletteHTTPRequest -from typing import Final +from karapace.kafka_rest_apis import KafkaRest +from karapace.logging import configure_logging +from karapace.rapu import RestApp +from schema_registry.schema_registry_apis import KarapaceSchemaRegistryController +import argparse import logging import sys -import uvicorn -# from karapace.kafka_rest_apis import KafkaRest +class KarapaceAll(KafkaRest, KarapaceSchemaRegistryController): + pass -def _configure_logging(*, config: Config) -> None: - log_handler = config.log_handler - root_handler: logging.Handler | None = None - if "systemd" == log_handler: - from systemd import journal +def main() -> int: + parser = argparse.ArgumentParser(prog="karapace", description="Karapace: Your Kafka essentials in one tool") + parser.add_argument("--version", action="version", help="show program version", version=karapace_version.__version__) + parser.add_argument("config_file", help="configuration file path", type=argparse.FileType()) + arg = parser.parse_args() - root_handler = journal.JournalHandler(SYSLOG_IDENTIFIER="karapace") - elif "stdout" == log_handler or log_handler is None: - root_handler = logging.StreamHandler(stream=sys.stdout) - else: - logging.basicConfig(level=config.log_level, format=config.log_format) - logging.getLogger().setLevel(config.log_level) - logging.warning("Log handler %s not recognized, root handler not set.", log_handler) - - if root_handler is not None: - root_handler.setFormatter(logging.Formatter(config.log_format)) - root_handler.setLevel(config.log_level) - root_handler.set_name(name="karapace") - logging.root.addHandler(root_handler) - - logging.root.setLevel(config.log_level) - logging.getLogger("uvicorn.error").setLevel(config.log_level) + with closing(arg.config_file): + config = read_config(arg.config_file) + configure_logging(config=config) -@asynccontextmanager -async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: - schema_registry: KarapaceSchemaRegistry | None = None - authorizer: AuthenticatorAndAuthorizer | None = None - try: - schema_registry = await SchemaRegistryDependencyManager.get_schema_registry() - await schema_registry.start() - await schema_registry.get_master() - authorizer = AuthorizationDependencyManager.get_authorizer() - if authorizer is not None: - await authorizer.start(StatsDependencyManager.get_stats()) - yield - finally: - if schema_registry: - await schema_registry.close() - if authorizer: - await authorizer.close() - + app: RestApp + if config["karapace_rest"] and config["karapace_registry"]: + info_str = "both services" + app = KarapaceAll(config=config) + elif config["karapace_rest"]: + info_str = "karapace rest" + app = KafkaRest(config=config) + elif config["karapace_registry"]: + info_str = "karapace schema registry" + app = KarapaceSchemaRegistryController(config=config) + else: + print("Both rest and registry options are disabled, exiting") + return 1 -def create_karapace_application(*, config: Config) -> FastAPI: - # TODO: this lifespan is SR related lifespan - app = FastAPI(lifespan=lifespan) - _configure_logging(config=config) + info_str_separator = "=" * 100 + logging.log(logging.INFO, "\n%s\nStarting %s\n%s", info_str_separator, info_str, info_str_separator) config_without_secrets = {} - for key, value in config.dict().items(): + for key, value in config.items(): if "password" in key: value = "****" - elif "keyfile" in key: - value = "****" config_without_secrets[key] = value logging.log(logging.DEBUG, "Config %r", config_without_secrets) - logging.log(logging.INFO, "Karapace version %s", karapace_version) - - @app.exception_handler(StarletteHTTPException) - async def http_exception_handler(_: StarletteHTTPRequest, exc: StarletteHTTPException): - return JSONResponse(status_code=exc.status_code, content=exc.detail) - - @app.exception_handler(RequestValidationError) - async def validation_exception_handler(_: StarletteHTTPRequest, exc: RequestValidationError): - error_code = HTTPStatus.UNPROCESSABLE_ENTITY.value - if isinstance(exc, KarapaceValidationError): - error_code = exc.error_code - message = exc.body - else: - message = exc.errors() - return JSONResponse( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - content={ - "error_code": error_code, - "message": message, - }, - ) - @app.middleware("http") - async def set_content_types(request: Request, call_next): - try: - response_content_type = check_schema_headers(request) - except HTTPException as exc: - return JSONResponse( - status_code=exc.status_code, - headers=exc.headers, - content=exc.detail, - ) - - # Schema registry supports application/octet-stream, assumption is JSON object body. - # Force internally to use application/json in this case for compatibility. - if request.headers.get("Content-Type") == "application/octet-stream": - new_headers = request.headers.mutablecopy() - new_headers["Content-Type"] = "application/json" - request._headers = new_headers - request.scope.update(headers=request.headers.raw) - - response = await call_next(request) - response.headers["Content-Type"] = response_content_type - return response - - if config.karapace_registry: - from karapace.routers.compatibility_router import compatibility_router - from karapace.routers.config_router import config_router - from karapace.routers.health_router import health_router - from karapace.routers.mode_router import mode_router - from karapace.routers.root_router import root_router - from karapace.routers.schemas_router import schemas_router - from karapace.routers.subjects_router import subjects_router - - app.include_router(compatibility_router) - app.include_router(config_router) - app.include_router(health_router) - app.include_router(mode_router) - app.include_router(root_router) - app.include_router(schemas_router) - app.include_router(subjects_router) - if config.karapace_rest: - # add rest router. - pass - - return app - - -def __old_main() -> int: try: PrometheusInstrumentation.setup_metrics(app=app) app.run() # `close` will be called by the callback `close_by_app` set by `KarapaceBase` @@ -167,8 +66,5 @@ def __old_main() -> int: return 0 -CONFIG: Final = ConfigDependencyManager.get_config() - if __name__ == "__main__": - app = create_karapace_application(config=CONFIG) - uvicorn.run(app, host=CONFIG.host, port=CONFIG.port, log_level=CONFIG.log_level.lower()) + sys.exit(main()) diff --git a/src/karapace/logging.py b/src/karapace/logging.py new file mode 100644 index 000000000..ad6656bc3 --- /dev/null +++ b/src/karapace/logging.py @@ -0,0 +1,45 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from karapace.config import Config + +import logging +import sys + + +def configure_logging(*, config: Config) -> None: + log_handler = config.log_handler + + root_handler: logging.Handler | None = None + if "systemd" == log_handler: + from systemd import journal + + root_handler = journal.JournalHandler(SYSLOG_IDENTIFIER="karapace") + elif "stdout" == log_handler or log_handler is None: + root_handler = logging.StreamHandler(stream=sys.stdout) + else: + logging.basicConfig(level=config.log_level, format=config.log_format) + logging.getLogger().setLevel(config.log_level) + logging.warning("Log handler %s not recognized, root handler not set.", log_handler) + + if root_handler is not None: + root_handler.setFormatter(logging.Formatter(config.log_format)) + root_handler.setLevel(config.log_level) + root_handler.set_name(name="karapace") + logging.root.addHandler(root_handler) + + logging.root.setLevel(config.log_level) + logging.getLogger("uvicorn.error").setLevel(config.log_level) + + +def log_config_without_secrets(config: Config) -> None: + config_without_secrets = {} + for key, value in config.dict().items(): + if "password" in key: + value = "****" + elif "keyfile" in key: + value = "****" + config_without_secrets[key] = value + logging.log(logging.DEBUG, "Config %r", config_without_secrets) diff --git a/src/karapace/routers/__init__.py b/src/karapace/routers/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/schema_registry/__init__.py b/src/schema_registry/__init__.py new file mode 100644 index 000000000..f53be7121 --- /dev/null +++ b/src/schema_registry/__init__.py @@ -0,0 +1,4 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" diff --git a/src/schema_registry/__main__.py b/src/schema_registry/__main__.py new file mode 100644 index 000000000..89408e4c6 --- /dev/null +++ b/src/schema_registry/__main__.py @@ -0,0 +1,65 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from fastapi import FastAPI +from karapace import version as karapace_version +from karapace.auth.auth import AuthenticatorAndAuthorizer +from karapace.auth.dependencies import AuthorizationDependencyManager +from karapace.config import Config +from karapace.dependencies.config_dependency import ConfigDependencyManager +from karapace.dependencies.schema_registry_dependency import SchemaRegistryDependencyManager +from karapace.dependencies.stats_dependeny import StatsDependencyManager +from karapace.logging import configure_logging, log_config_without_secrets +from karapace.schema_registry import KarapaceSchemaRegistry +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from schema_registry.http_handlers import setup_exception_handlers +from schema_registry.middlewares import setup_middlewares +from schema_registry.routers import setup_routers +from typing import Final + +import logging +import uvicorn + + +@asynccontextmanager +async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: + schema_registry: KarapaceSchemaRegistry | None = None + authorizer: AuthenticatorAndAuthorizer | None = None + try: + schema_registry = await SchemaRegistryDependencyManager.get_schema_registry() + await schema_registry.start() + await schema_registry.get_master() + authorizer = AuthorizationDependencyManager.get_authorizer() + if authorizer is not None: + await authorizer.start(StatsDependencyManager.get_stats()) + yield + finally: + if schema_registry: + await schema_registry.close() + if authorizer: + await authorizer.close() + + +def create_karapace_application(*, config: Config) -> FastAPI: + configure_logging(config=config) + log_config_without_secrets(config=config) + logging.info("Starting Karapace Schema Registry (%s)", karapace_version.__version__) + + app = FastAPI(lifespan=lifespan) + setup_routers(app=app) + setup_exception_handlers(app=app) + setup_middlewares(app=app) + + FastAPIInstrumentor.instrument_app(app) + + return app + + +CONFIG: Final = ConfigDependencyManager.get_config() + +if __name__ == "__main__": + app = create_karapace_application(config=CONFIG) + uvicorn.run(app, host=CONFIG.host, port=CONFIG.port, log_level=CONFIG.log_level.lower()) diff --git a/src/schema_registry/container.py b/src/schema_registry/container.py new file mode 100644 index 000000000..f53be7121 --- /dev/null +++ b/src/schema_registry/container.py @@ -0,0 +1,4 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" diff --git a/src/schema_registry/http_handlers/__init__.py b/src/schema_registry/http_handlers/__init__.py new file mode 100644 index 000000000..93bc853cc --- /dev/null +++ b/src/schema_registry/http_handlers/__init__.py @@ -0,0 +1,34 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from fastapi import FastAPI, status +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from http import HTTPStatus +from schema_registry.routers.errors import KarapaceValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.requests import Request as StarletteHTTPRequest + + +def setup_exception_handlers(app: FastAPI) -> None: + @app.exception_handler(StarletteHTTPException) + async def http_exception_handler(_: StarletteHTTPRequest, exc: StarletteHTTPException): + return JSONResponse(status_code=exc.status_code, content=exc.detail) + + @app.exception_handler(RequestValidationError) + async def validation_exception_handler(_: StarletteHTTPRequest, exc: RequestValidationError): + error_code = HTTPStatus.UNPROCESSABLE_ENTITY.value + if isinstance(exc, KarapaceValidationError): + error_code = exc.error_code + message = exc.body + else: + message = exc.errors() + return JSONResponse( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + content={ + "error_code": error_code, + "message": message, + }, + ) diff --git a/src/schema_registry/middlewares/__init__.py b/src/schema_registry/middlewares/__init__.py new file mode 100644 index 000000000..b5fb2e125 --- /dev/null +++ b/src/schema_registry/middlewares/__init__.py @@ -0,0 +1,33 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse +from karapace.content_type import check_schema_headers + + +def setup_middlewares(app: FastAPI) -> None: + @app.middleware("http") + async def set_content_types(request: Request, call_next): + try: + response_content_type = check_schema_headers(request) + except HTTPException as exc: + return JSONResponse( + status_code=exc.status_code, + headers=exc.headers, + content=exc.detail, + ) + + # Schema registry supports application/octet-stream, assumption is JSON object body. + # Force internally to use application/json in this case for compatibility. + if request.headers.get("Content-Type") == "application/octet-stream": + new_headers = request.headers.mutablecopy() + new_headers["Content-Type"] = "application/json" + request._headers = new_headers + request.scope.update(headers=request.headers.raw) + + response = await call_next(request) + response.headers["Content-Type"] = response_content_type + return response diff --git a/src/schema_registry/routers/__init__.py b/src/schema_registry/routers/__init__.py new file mode 100644 index 000000000..e077a1551 --- /dev/null +++ b/src/schema_registry/routers/__init__.py @@ -0,0 +1,23 @@ +""" +Copyright (c) 2024 Aiven Ltd +See LICENSE for details +""" + +from fastapi import FastAPI +from schema_registry.routers.compatibility_router import compatibility_router +from schema_registry.routers.config_router import config_router +from schema_registry.routers.health_router import health_router +from schema_registry.routers.mode_router import mode_router +from schema_registry.routers.root_router import root_router +from schema_registry.routers.schemas_router import schemas_router +from schema_registry.routers.subjects_router import subjects_router + + +def setup_routers(app: FastAPI) -> None: + app.include_router(compatibility_router) + app.include_router(config_router) + app.include_router(health_router) + app.include_router(mode_router) + app.include_router(root_router) + app.include_router(schemas_router) + app.include_router(subjects_router) diff --git a/src/karapace/routers/compatibility_router.py b/src/schema_registry/routers/compatibility_router.py similarity index 88% rename from src/karapace/routers/compatibility_router.py rename to src/schema_registry/routers/compatibility_router.py index 0db406d2a..25c759553 100644 --- a/src/karapace/routers/compatibility_router.py +++ b/src/schema_registry/routers/compatibility_router.py @@ -7,9 +7,9 @@ from karapace.auth.auth import Operation from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.routers.errors import unauthorized -from karapace.routers.requests import CompatibilityCheckResponse, SchemaRequest from karapace.typing import Subject +from schema_registry.routers.errors import unauthorized +from schema_registry.routers.requests import CompatibilityCheckResponse, SchemaRequest compatibility_router = APIRouter( prefix="/compatibility", diff --git a/src/karapace/routers/config_router.py b/src/schema_registry/routers/config_router.py similarity index 95% rename from src/karapace/routers/config_router.py rename to src/schema_registry/routers/config_router.py index a83f24f60..b5b77b4bb 100644 --- a/src/karapace/routers/config_router.py +++ b/src/schema_registry/routers/config_router.py @@ -9,9 +9,9 @@ from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep from karapace.dependencies.forward_client_dependency import ForwardClientDep from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep -from karapace.routers.errors import no_primary_url_error, unauthorized -from karapace.routers.requests import CompatibilityLevelResponse, CompatibilityRequest, CompatibilityResponse from karapace.typing import Subject +from schema_registry.routers.errors import no_primary_url_error, unauthorized +from schema_registry.routers.requests import CompatibilityLevelResponse, CompatibilityRequest, CompatibilityResponse config_router = APIRouter( prefix="/config", diff --git a/src/karapace/routers/errors.py b/src/schema_registry/routers/errors.py similarity index 75% rename from src/karapace/routers/errors.py rename to src/schema_registry/routers/errors.py index a16c9797a..18c80299d 100644 --- a/src/karapace/routers/errors.py +++ b/src/schema_registry/routers/errors.py @@ -33,6 +33,20 @@ class SchemaErrorCodes(Enum): NO_MASTER_ERROR = 50003 +@unique +class SchemaErrorMessages(Enum): + SUBJECT_NOT_FOUND_FMT = "Subject '{subject}' not found." + INVALID_COMPATIBILITY_LEVEL = ( + "Invalid compatibility level. Valid values are none, backward, " + "forward, full, backward_transitive, forward_transitive, and " + "full_transitive" + ) + SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED_FMT = ( + "Subject '{subject}' does not have subject-level compatibility configured" + ) + REFERENCES_SUPPORT_NOT_IMPLEMENTED = "Schema references are not supported for '{schema_type}' schema type" + + class KarapaceValidationError(RequestValidationError): def __init__(self, error_code: int, error: str): super().__init__(errors=[], body=error) diff --git a/src/karapace/routers/health_router.py b/src/schema_registry/routers/health_router.py similarity index 100% rename from src/karapace/routers/health_router.py rename to src/schema_registry/routers/health_router.py diff --git a/src/karapace/routers/mode_router.py b/src/schema_registry/routers/mode_router.py similarity index 95% rename from src/karapace/routers/mode_router.py rename to src/schema_registry/routers/mode_router.py index d8c98363a..0789d2baf 100644 --- a/src/karapace/routers/mode_router.py +++ b/src/schema_registry/routers/mode_router.py @@ -7,8 +7,8 @@ from karapace.auth.auth import Operation from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.routers.errors import unauthorized from karapace.typing import Subject +from schema_registry.routers.errors import unauthorized mode_router = APIRouter( prefix="/mode", diff --git a/src/karapace/routers/requests.py b/src/schema_registry/routers/requests.py similarity index 97% rename from src/karapace/routers/requests.py rename to src/schema_registry/routers/requests.py index 8400f629d..fb4b51511 100644 --- a/src/karapace/routers/requests.py +++ b/src/schema_registry/routers/requests.py @@ -3,10 +3,10 @@ See LICENSE for details """ -from karapace.routers.errors import KarapaceValidationError from karapace.schema_type import SchemaType from karapace.typing import Subject from pydantic import BaseModel, Field, validator +from schema_registry.routers.errors import KarapaceValidationError from typing import Any diff --git a/src/karapace/routers/root_router.py b/src/schema_registry/routers/root_router.py similarity index 100% rename from src/karapace/routers/root_router.py rename to src/schema_registry/routers/root_router.py diff --git a/src/karapace/routers/schemas_router.py b/src/schema_registry/routers/schemas_router.py similarity index 96% rename from src/karapace/routers/schemas_router.py rename to src/schema_registry/routers/schemas_router.py index c06cd4a48..58f5bd7da 100644 --- a/src/karapace/routers/schemas_router.py +++ b/src/schema_registry/routers/schemas_router.py @@ -6,7 +6,7 @@ from fastapi import APIRouter from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep, CurrentUserDep from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep -from karapace.routers.requests import SchemaListingItem, SchemasResponse, SubjectVersion +from schema_registry.routers.requests import SchemaListingItem, SchemasResponse, SubjectVersion schemas_router = APIRouter( prefix="/schemas", diff --git a/src/karapace/routers/subjects_router.py b/src/schema_registry/routers/subjects_router.py similarity index 96% rename from src/karapace/routers/subjects_router.py rename to src/schema_registry/routers/subjects_router.py index 9bde67743..0189b398d 100644 --- a/src/karapace/routers/subjects_router.py +++ b/src/schema_registry/routers/subjects_router.py @@ -9,9 +9,9 @@ from karapace.dependencies.controller_dependency import KarapaceSchemaRegistryControllerDep from karapace.dependencies.forward_client_dependency import ForwardClientDep from karapace.dependencies.schema_registry_dependency import SchemaRegistryDep -from karapace.routers.errors import no_primary_url_error, unauthorized -from karapace.routers.requests import SchemaIdResponse, SchemaRequest, SchemaResponse, SubjectSchemaVersionResponse from karapace.typing import Subject +from schema_registry.routers.errors import no_primary_url_error, unauthorized +from schema_registry.routers.requests import SchemaIdResponse, SchemaRequest, SchemaResponse, SubjectSchemaVersionResponse import logging diff --git a/src/karapace/schema_registry_apis.py b/src/schema_registry/schema_registry_apis.py similarity index 96% rename from src/karapace/schema_registry_apis.py rename to src/schema_registry/schema_registry_apis.py index 56e79243c..a11e29be2 100644 --- a/src/karapace/schema_registry_apis.py +++ b/src/schema_registry/schema_registry_apis.py @@ -5,7 +5,6 @@ from __future__ import annotations from avro.errors import SchemaParseException -from enum import Enum, unique from fastapi import HTTPException, Request, Response, status from karapace.auth.auth import Operation, User from karapace.auth.dependencies import AuthenticatorAndAuthorizerDep @@ -31,8 +30,14 @@ ) from karapace.forward_client import ForwardClient from karapace.protobuf.exception import ProtobufUnresolvedDependencyException -from karapace.routers.errors import no_primary_url_error -from karapace.routers.requests import ( +from karapace.schema_models import ParsedTypedSchema, SchemaType, SchemaVersion, TypedSchema, ValidatedTypedSchema, Versioner +from karapace.schema_references import LatestVersionReference, Reference +from karapace.schema_registry import KarapaceSchemaRegistry +from karapace.statsd import StatsClient +from karapace.typing import JsonData, JsonObject, SchemaId, Subject, Version +from karapace.utils import JSONDecodeError +from schema_registry.routers.errors import no_primary_url_error, SchemaErrorCodes, SchemaErrorMessages +from schema_registry.routers.requests import ( CompatibilityCheckResponse, CompatibilityLevelResponse, CompatibilityRequest, @@ -46,12 +51,6 @@ SubjectSchemaVersionResponse, SubjectVersion, ) -from karapace.schema_models import ParsedTypedSchema, SchemaType, SchemaVersion, TypedSchema, ValidatedTypedSchema, Versioner -from karapace.schema_references import LatestVersionReference, Reference -from karapace.schema_registry import KarapaceSchemaRegistry -from karapace.statsd import StatsClient -from karapace.typing import JsonData, JsonObject, SchemaId, Subject, Version -from karapace.utils import JSONDecodeError from typing import Any, cast import json @@ -61,46 +60,6 @@ LOG = logging.getLogger(__name__) -# TODO Remove, already in router/errors -@unique -class SchemaErrorCodes(Enum): - HTTP_BAD_REQUEST = status.HTTP_400_BAD_REQUEST - HTTP_NOT_FOUND = status.HTTP_404_NOT_FOUND - HTTP_CONFLICT = status.HTTP_409_CONFLICT - HTTP_UNPROCESSABLE_ENTITY = status.HTTP_422_UNPROCESSABLE_ENTITY - HTTP_INTERNAL_SERVER_ERROR = status.HTTP_500_INTERNAL_SERVER_ERROR - SUBJECT_NOT_FOUND = 40401 - VERSION_NOT_FOUND = 40402 - SCHEMA_NOT_FOUND = 40403 - SUBJECT_SOFT_DELETED = 40404 - SUBJECT_NOT_SOFT_DELETED = 40405 - SCHEMAVERSION_SOFT_DELETED = 40406 - SCHEMAVERSION_NOT_SOFT_DELETED = 40407 - SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED_ERROR_CODE = 40408 - INVALID_VERSION_ID = 42202 - INVALID_COMPATIBILITY_LEVEL = 42203 - INVALID_SCHEMA = 42201 - INVALID_SUBJECT = 42208 - SCHEMA_TOO_LARGE_ERROR_CODE = 42209 - REFERENCES_SUPPORT_NOT_IMPLEMENTED = 44302 - REFERENCE_EXISTS = 42206 - NO_MASTER_ERROR = 50003 - - -@unique -class SchemaErrorMessages(Enum): - SUBJECT_NOT_FOUND_FMT = "Subject '{subject}' not found." - INVALID_COMPATIBILITY_LEVEL = ( - "Invalid compatibility level. Valid values are none, backward, " - "forward, full, backward_transitive, forward_transitive, and " - "full_transitive" - ) - SUBJECT_LEVEL_COMPATIBILITY_NOT_CONFIGURED_FMT = ( - "Subject '{subject}' does not have subject-level compatibility configured" - ) - REFERENCES_SUPPORT_NOT_IMPLEMENTED = "Schema references are not supported for '{schema_type}' schema type" - - class KarapaceSchemaRegistryController: def __init__(self, config: Config, schema_registry: KarapaceSchemaRegistry, stats: StatsClient) -> None: # super().__init__(config=config, not_ready_handler=self._forward_if_not_ready_to_serve) diff --git a/tests/integration/test_schema.py b/tests/integration/test_schema.py index 9a9eb31e7..d4076f98e 100644 --- a/tests/integration/test_schema.py +++ b/tests/integration/test_schema.py @@ -9,9 +9,9 @@ from karapace.client import Client from karapace.kafka.producer import KafkaProducer from karapace.rapu import is_success -from karapace.schema_registry_apis import SchemaErrorMessages from karapace.schema_type import SchemaType from karapace.utils import json_encode +from schema_registry.schema_registry_apis import SchemaErrorMessages from tests.base_testcase import BaseTestCase from tests.integration.utils.cluster import RegistryDescription from tests.integration.utils.kafka_server import KafkaServers diff --git a/tests/unit/test_schema_registry_api.py b/tests/unit/test_schema_registry_api.py index 7fcecd47e..a9dc897e2 100644 --- a/tests/unit/test_schema_registry_api.py +++ b/tests/unit/test_schema_registry_api.py @@ -7,7 +7,7 @@ from karapace.rapu import HTTPResponse from karapace.schema_reader import KafkaSchemaReader from karapace.schema_registry import KarapaceSchemaRegistry -from karapace.schema_registry_apis import KarapaceSchemaRegistryController +from schema_registry.schema_registry_apis import KarapaceSchemaRegistryController from unittest.mock import ANY, AsyncMock, Mock, patch, PropertyMock import asyncio @@ -31,7 +31,7 @@ async def test_validate_schema_request_body() -> None: async def test_forward_when_not_ready() -> None: - with patch("karapace.schema_registry_apis.KarapaceSchemaRegistry") as schema_registry_class: + with patch("schema_registry.schema_registry_apis.KarapaceSchemaRegistry") as schema_registry_class: schema_reader_mock = Mock(spec=KafkaSchemaReader) ready_property_mock = PropertyMock(return_value=False) schema_registry = AsyncMock(spec=KarapaceSchemaRegistry)