feat: wire more dependencies and modules

we drop the dependency managers and use the DI containers and providers

.dockerignore

@@ -10,7 +10,6 @@
 !LICENSE
 !pyproject.toml
 !setup.py
-!container/start.sh
 !container/healthcheck.py
 
 # Ignore some files in source directories.

.github/workflows/tests.yml

@@ -14,6 +14,7 @@ env:
   FORCE_COLOR: 1
   PIP_PROGRESS_BAR: off
   PYTHONUNBUFFERED: 1
+  KARAPACE_DOTENV: ${{ github.workspace }}/karapace.config.env
 
 jobs:
   tests:
@@ -44,11 +45,11 @@ jobs:
     - run: make unit-tests
       env:
         COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=karapace --cov-append --numprocesses 4"
+        PYTEST_ARGS: "--cov=src --cov-append --numprocesses 4"
     - run: make integration-tests
       env:
         COVERAGE_FILE: ".coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=karapace --cov-append --random-order --numprocesses 4"
+        PYTEST_ARGS: "--cov=src --cov-append --random-order --numprocesses 4"
 
     - name: Archive logs
       uses: actions/upload-artifact@v4

bin/smoke-test-registry.sh

@@ -6,6 +6,7 @@ for ((i = 0; i <= retries; i++)); do
     response=$(
         curl --silent --verbose --fail --request POST \
             --header 'Content-Type: application/vnd.schemaregistry.v1+json' \
+            --header 'Authorization: Basic YWRtaW46YWRtaW4=' \
             --data '{"schema": "{\"type\": \"record\", \"name\": \"Obj\", \"fields\":[{\"name\": \"age\", \"type\": \"int\"}]}"}' \
             http://localhost:8081/subjects/test-key/versions
     )

container/Dockerfile

@@ -55,10 +55,6 @@ RUN apt-get update \
 COPY --from=builder /venv /venv
 ENV PATH="/venv/bin:$PATH"
 
-COPY ./container/start.sh /opt/karapace
-RUN chmod 500 /opt/karapace/start.sh \
- && chown karapace:karapace /opt/karapace/start.sh
-
 COPY ./container/healthcheck.py /opt/karapace
 
 WORKDIR /opt/karapace

container/compose.yml

@@ -4,7 +4,7 @@ services:
   zookeeper:
     image: confluentinc/cp-zookeeper:latest
     ports:
-      - "2181:2181"
+      - 2181:2181
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
@@ -14,16 +14,16 @@ services:
     depends_on:
       - zookeeper
     ports:
-      - "9101:9101"  # JMX
-      - "9092:9092"  # Kafka
+      - 9101:9101  # JMX
+      - 9092:9092  # Kafka
     environment:
       # Listeners:
       # PLAINTEXT_HOST -> Expose kafka to the host network
       # PLAINTEXT -> Used by kafka for inter broker communication / containers
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
-      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://karapace-registry:8081
+      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://karapace-schema-registry:8081
       # Metrics:
       KAFKA_JMX_PORT: 9101
       KAFKA_JMX_HOSTNAME: localhost
@@ -54,51 +54,44 @@ services:
       KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 6000
       KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
 
-  karapace-registry:
+  karapace-schema-registry:
     image: ghcr.io/aiven-open/karapace:develop
     build:
       context: ..
       dockerfile: container/Dockerfile
     entrypoint:
-      - /bin/bash
-      - /opt/karapace/start.sh
-      - registry
+      - python3
+      - -m
+      - schema_registry
     depends_on:
       - kafka
     ports:
-      - "8081:8081"
+      - 8081:8081
     volumes:
-    - ./karapace.env:/opt/karapace/karapace.env
+    - ./karapace.registry.env:/opt/karapace/karapace.env
     environment:
       KARAPACE_DOTENV: /opt/karapace/karapace.env
+      KARAPACE_PORT: 8081
 
-  karapace-rest:
+  karapace-rest-proxy:
     image: ghcr.io/aiven-open/karapace:develop
     build:
       context: ..
       dockerfile: container/Dockerfile
     entrypoint:
-      - /bin/bash
-      - /opt/karapace/start.sh
-      - rest
+      - python3
+      - -m
+      - karapace.karapace_all
     depends_on:
       - kafka
-      - karapace-registry
+      - karapace-schema-registry
     ports:
-      - "8082:8082"
+      - 8082:8082
+    volumes:
+    - ./karapace.rest.env:/opt/karapace/karapace.env
     environment:
+      KARAPACE_DOTENV: /opt/karapace/karapace.env
       KARAPACE_PORT: 8082
-      KARAPACE_HOST: 0.0.0.0
-      KARAPACE_ADVERTISED_HOSTNAME: karapace-rest
-      KARAPACE_BOOTSTRAP_URI: kafka:29092
-      KARAPACE_REGISTRY_HOST: karapace-registry
-      KARAPACE_REGISTRY_PORT: 8081
-      KARAPACE_ADMIN_METADATA_MAX_AGE: 0
-      KARAPACE_LOG_LEVEL: WARNING
-      KARAPACE_STATSD_HOST: statsd-exporter
-      KARAPACE_STATSD_PORT: 8125
-      KARAPACE_KAFKA_SCHEMA_READER_STRICT_MODE: false
-      KARAPACE_KAFKA_RETRIABLE_ERRORS_SILENCED: true
 
   prometheus:
     image: prom/prometheus

container/karapace.env → container/karapace.registry.env

@@ -15,7 +15,7 @@ FETCH_MIN_BYTES=1
 GROUP_ID=karapace-schema-registry
 HOST=0.0.0.0
 PORT=8081
-REGISTRY_HOST=karapace-registry
+REGISTRY_HOST=karapace-schema-registry
 REGISTRY_PORT=8081
 REST_AUTHORIZATION=False
 LOG_HANDLER=stdout
@@ -35,6 +35,7 @@ PRODUCER_MAX_REQUEST_SIZE=1048576
 SESSION_TIMEOUT_MS=10000
 KARAPACE_REST=False
 KARAPACE_REGISTRY=True
+KARAPACE_PORT=8081
 NAME_STRATEGY=topic_name
 NAME_STRATEGY_VALIDATION=True
 MASTER_ELECTION_STRATEGY=lowest

container/karapace.rest.env

@@ -0,0 +1,51 @@
+KARAPACE_DOTENV=/opt/karapace/karapace.env
+ACCESS_LOGS_DEBUG=False
+# ACCESS_LOG_CLASS=karapace.utils.DebugAccessLogger
+ACCESS_LOG_CLASS=aiohttp.web_log.AccessLogger
+ADVERTISED_HOSTNAME=karapace-rest-proxy
+ADVERTISED_PORT=8082
+ADVERTISED_PROTOCOL=http
+BOOTSTRAP_URI=kafka:29092
+CLIENT_ID=karapace-rest-proxy
+COMPATIBILITY=BACKWARD
+CONNECTIONS_MAX_IDLE_MS=15000
+CONSUMER_ENABLE_AUTO_COMMIT=True
+CONSUMER_REQUEST_TIMEOUT_MS=11000
+CONSUMER_REQUEST_MAX_BYTES=67108864
+CONSUMER_IDLE_DISCONNECT_TIMEOUT=0
+FETCH_MIN_BYTES=1
+GROUP_ID=karapace-rest-proxy
+HOST=0.0.0.0
+PORT=8082
+REGISTRY_HOST=karapace-schema-registry
+REGISTRY_PORT=8081
+REST_AUTHORIZATION=False
+LOG_HANDLER=stdout
+LOG_LEVEL=DEBUG
+LOG_FORMAT=%(asctime)s [%(threadName)s] %(filename)s:%(funcName)s:%(lineno)d %(message)s
+MASTER_ELIGIBILITY=True
+REPLICATION_FACTOR=1
+SECURITY_PROTOCOL=PLAINTEXT
+SSL_CHECK_HOSTNAME=True
+TOPIC_NAME=_schemas
+METADATA_MAX_AGE_MS=60000
+ADMIN_METADATA_MAX_AGE=5
+PRODUCER_ACKS=1
+PRODUCER_COUNT=5
+PRODUCER_LINGER_MS=100
+PRODUCER_MAX_REQUEST_SIZE=1048576
+SESSION_TIMEOUT_MS=10000
+KARAPACE_REST=True
+KARAPACE_REGISTRY=False
+KARAPACE_PORT=8082
+NAME_STRATEGY=topic_name
+NAME_STRATEGY_VALIDATION=True
+MASTER_ELECTION_STRATEGY=lowest
+PROTOBUF_RUNTIME_DIRECTORY=runtime
+STATSD_HOST=statsd-exporter
+STATSD_PORT=8125
+KAFKA_SCHEMA_READER_STRICT_MODE=False
+KAFKA_RETRIABLE_ERRORS_SILENCED=True
+USE_PROTOBUF_FORMATTER=False
+HTTP_REQUEST_MAX_SIZE=1048576
+TAGS='{ "app": "karapace-rest-proxy" }'

pyproject.toml

@@ -113,7 +113,7 @@ include-package-data = true
 where = ["src"]
 
 [tool.setuptools.package-data]
-schema_registry = ["*.yaml"]
+karapace = ["*.yaml"]
 
 [tool.setuptools_scm]
 version_file = "src/karapace/version.py"

requirements/requirements-dev.txt

@@ -58,7 +58,7 @@ configargparse==1.7
     # via locust
 confluent-kafka==2.4.0
     # via karapace (/karapace/pyproject.toml)
-coverage[toml]==7.6.7
+coverage[toml]==7.6.8
     # via pytest-cov
 cramjam==2.9.0
     # via python-snappy
@@ -104,7 +104,7 @@ gevent==24.11.1
     # via
     #   geventhttpclient
     #   locust
-geventhttpclient==2.3.1
+geventhttpclient==2.3.3
     # via locust
 greenlet==3.1.1
     # via gevent

src/karapace/auth/auth.py → src/karapace/auth.py

@@ -8,7 +8,7 @@
 from dataclasses import dataclass, field
 from enum import Enum, unique
 from hmac import compare_digest
-from karapace.config import InvalidConfiguration
+from karapace.config import Config, InvalidConfiguration
 from karapace.statsd import StatsClient
 from karapace.utils import json_decode, json_encode
 from typing import Protocol
@@ -205,14 +205,12 @@ def check_authorization_any(self, user: User | None, operation: Operation, resou
 
 
 class HTTPAuthorizer(ACLAuthorizer, AuthenticatorAndAuthorizer):
-    def __init__(self, filename: str) -> None:
+    def __init__(self, config: Config) -> None:
         super().__init__()
-        self._auth_filename: str = filename
+        self._auth_filename: str = config.registry_authfile
         self._auth_mtime: float = -1
         self._refresh_auth_task: asyncio.Task | None = None
         self._refresh_auth_awatch_stop_event = asyncio.Event()
-        # Once first, can raise if file not valid
-        self._load_authfile()
 
     @property
     def authfile_last_modified(self) -> float:
@@ -221,6 +219,7 @@ def authfile_last_modified(self) -> float:
     @override
     async def start(self, stats: StatsClient) -> None:
         """Start authfile refresher task"""
+        self._load_authfile()
 
         async def _refresh_authfile() -> None:
             """Reload authfile, but keep old auth data if loading fails"""
@@ -294,6 +293,14 @@ def authenticate(self, *, username: str, password: str) -> User:
         return user
 
 
+def get_authorizer(
+    config: Config,
+    http_authorizer: HTTPAuthorizer,
+    no_auth_authorizer: NoAuthAndAuthz,
+) -> AuthenticatorAndAuthorizer:
+    return http_authorizer if config.registry_authfile else no_auth_authorizer
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(prog="karapace_mkpasswd", description="Karapace password hasher")
     parser.add_argument("-u", "--user", help="Username", type=str)