From 38170e0904627b2cbb8ea622c85fa9628fab183d Mon Sep 17 00:00:00 2001 From: Elia Migliore Date: Tue, 8 Oct 2024 17:29:10 +0200 Subject: [PATCH] feat: add rest endpoint to display current config --- src/karapace/config.py | 81 +++++++++++++++++++++++++++++++++- src/karapace/karapace.py | 10 ++++- tests/integration/test_rest.py | 31 +++++++++++++ 3 files changed, 119 insertions(+), 3 deletions(-) diff --git a/src/karapace/config.py b/src/karapace/config.py index 3761b0072..85613c7a6 100644 --- a/src/karapace/config.py +++ b/src/karapace/config.py @@ -8,10 +8,10 @@ from collections.abc import Mapping from karapace.constants import DEFAULT_AIOHTTP_CLIENT_MAX_SIZE, DEFAULT_PRODUCER_MAX_REQUEST, DEFAULT_SCHEMA_TOPIC -from karapace.typing import ElectionStrategy, NameStrategy +from karapace.typing import ElectionStrategy, JsonObject, NameStrategy from karapace.utils import json_decode, json_encode, JSONDecodeError from pathlib import Path -from typing import IO +from typing import Final, IO from typing_extensions import NotRequired, TypedDict import logging @@ -19,6 +19,17 @@ import socket import ssl +SECRET_FIELDS: Final = ( + "bootstrap_uri", + "sasl_bootstrap_uri", + "registry_password", + "ssl_password", + "sasl_plain_password", + "sasl_oauth_token", +) + +IGNORED_FIELDS: Final = ("sentry", "tags") + class Config(TypedDict): access_logs_debug: bool @@ -89,6 +100,72 @@ class Config(TypedDict): tags: NotRequired[Mapping[str, object]] +def service_without_secret(config: Config) -> JsonObject: + plaintext_config: JsonObject = { + "access_logs_debug": config["access_logs_debug"], + "access_log_class": config["access_log_class"], + "advertised_hostname": config["advertised_hostname"], + "advertised_port": config["advertised_port"], + "advertised_protocol": config["advertised_protocol"], + "client_id": config["client_id"], + "compatibility": config["compatibility"], + "connections_max_idle_ms": config["connections_max_idle_ms"], + "consumer_enable_auto_commit": config["consumer_enable_auto_commit"], + "consumer_request_timeout_ms": config["consumer_request_timeout_ms"], + "consumer_request_max_bytes": config["consumer_request_max_bytes"], + "consumer_idle_disconnect_timeout": config["consumer_idle_disconnect_timeout"], + "fetch_min_bytes": config["fetch_min_bytes"], + "group_id": config["group_id"], + "host": config["host"], + "port": config["port"], + # the file itself shouldn't be a problem, just the content should be protected + "server_tls_certfile": config["server_tls_certfile"], + "server_tls_keyfile": config["server_tls_keyfile"], + "registry_host": config["registry_host"], + "registry_port": config["registry_port"], + "registry_user": config["registry_user"], + "registry_ca": config["registry_ca"], + "registry_authfile": config["registry_authfile"], + "rest_authorization": config["rest_authorization"], + "rest_base_uri": config["rest_base_uri"], + "log_level": config["log_level"], + "log_format": config["log_format"], + "master_eligibility": config["master_eligibility"], + "replication_factor": config["replication_factor"], + "security_protocol": config["security_protocol"], + "ssl_cafile": config["ssl_cafile"], + "ssl_certfile": config["ssl_certfile"], + "ssl_keyfile": config["ssl_keyfile"], + "ssl_check_hostname": config["ssl_check_hostname"], + "ssl_crlfile": config["ssl_crlfile"], + "sasl_mechanism": config["sasl_mechanism"], + "sasl_plain_username": config["sasl_plain_username"], + "topic_name": config["topic_name"], + "metadata_max_age_ms": config["metadata_max_age_ms"], + "admin_metadata_max_age": config["admin_metadata_max_age"], + "producer_acks": config["producer_acks"], + "producer_compression_type": config["producer_compression_type"], + "producer_count": config["producer_count"], + "producer_linger_ms": config["producer_linger_ms"], + "producer_max_request_size": config["producer_max_request_size"], + "session_timeout_ms": config["session_timeout_ms"], + "karapace_rest": config["karapace_rest"], + "karapace_registry": config["karapace_registry"], + "name_strategy": config["name_strategy"], + "name_strategy_validation": config["name_strategy_validation"], + "master_election_strategy": config["master_election_strategy"], + "protobuf_runtime_directory": config["protobuf_runtime_directory"], + "statsd_host": config["statsd_host"], + "statsd_port": config["statsd_port"], + "kafka_schema_reader_strict_mode": config["kafka_schema_reader_strict_mode"], + "kafka_retriable_errors_silenced": config["kafka_retriable_errors_silenced"], + "use_protobuf_formatter": config["use_protobuf_formatter"], + } + for key in SECRET_FIELDS: + plaintext_config[key] = "REDACTED" + return plaintext_config + + class ConfigDefaults(Config, total=False): ... diff --git a/src/karapace/karapace.py b/src/karapace/karapace.py index 75cd96da4..d02b210d7 100644 --- a/src/karapace/karapace.py +++ b/src/karapace/karapace.py @@ -11,7 +11,7 @@ from collections.abc import Awaitable from functools import partial from http import HTTPStatus -from karapace.config import Config +from karapace.config import Config, service_without_secret from karapace.rapu import HTTPRequest, HTTPResponse, RestApp from karapace.typing import JsonObject from karapace.utils import json_encode @@ -33,6 +33,7 @@ def __init__(self, config: Config, not_ready_handler: Callable[[HTTPRequest], No self.health_hooks: list[HealthHook] = [] # Do not use rapu's etag, readiness and other wrapping self.app.router.add_route("GET", "/_health", self.health) + self.app.router.add_route("GET", "/_config", self.config_endpoint) self.kafka_timeout = 10 self.route("/", callback=self.root_get, method="GET") @@ -91,6 +92,13 @@ def service_unavailable(message: str, sub_code: int, content_type: str) -> NoRet async def root_get(self) -> NoReturn: self.r({}, "application/json") + async def config_endpoint(self, _request: Request) -> aiohttp.web.Response: + return aiohttp.web.Response( + body=json_encode(service_without_secret(self.config), binary=True, compact=True), + status=HTTPStatus.OK.value, + headers={"Content-Type": "application/json"}, + ) + async def health(self, _request: Request) -> aiohttp.web.Response: resp: JsonObject = { "process_uptime_sec": int(time.monotonic() - self._process_start_time), diff --git a/tests/integration/test_rest.py b/tests/integration/test_rest.py index ee504366b..fc6c3b21e 100644 --- a/tests/integration/test_rest.py +++ b/tests/integration/test_rest.py @@ -7,6 +7,7 @@ from collections.abc import Mapping from dataclasses import dataclass from karapace.client import Client +from karapace.config import Config, IGNORED_FIELDS, SECRET_FIELDS from karapace.kafka.admin import KafkaAdminClient from karapace.kafka.producer import KafkaProducer from karapace.kafka_rest_apis import KafkaRest, SUBJECT_VALID_POSTFIX @@ -58,6 +59,36 @@ async def test_health_endpoint(rest_async_client: Client) -> None: assert response["karapace_version"] == __version__ +def test_secret_fields_are_config_fields() -> None: + config_keys = list(Config.__annotations__) + for key in SECRET_FIELDS: + assert key in config_keys + + +def test_ignored_fields_are_config_fields() -> None: + config_keys = list(Config.__annotations__) + for key in IGNORED_FIELDS: + assert key in config_keys + + +async def test_config_endpoint(rest_async_client: Client) -> None: + res = await rest_async_client.get("/_config") + assert res.status_code == 200 + response = res.json() + required_keys = list(Config.__annotations__) + + for key in required_keys: + if key not in IGNORED_FIELDS: + assert key in response, "Need to explicitly decide if config needs to be shown as plaintext or hidden" + + for key in response: + if key not in IGNORED_FIELDS: + assert key in required_keys, "All keys should be a config or explicitly ignored" + + for key in SECRET_FIELDS: + assert response[key] == "REDACTED" + + async def test_request_body_too_large(rest_async_client: KafkaAdminClient, admin_client: Client) -> None: tn = new_topic(admin_client) await wait_for_topics(rest_async_client, topic_names=[tn], timeout=NEW_TOPIC_TIMEOUT, sleep=1)