From f0397462591769a7a07610789a892e4f3c5fd3ce Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Fri, 11 Oct 2024 19:15:11 +0000 Subject: [PATCH] Split clickhouse admin keeper and replica servers Also, split the clients and remove the dependency of clickhouse-admin-api from nexus. --- Cargo.lock | 28 +- Cargo.toml | 9 +- clickhouse-admin/api/src/lib.rs | 63 +- ...se-admin.rs => clickhouse-admin-keeper.rs} | 9 +- .../src/bin/clickhouse-admin-server.rs | 75 ++ clickhouse-admin/src/clickward.rs | 5 +- clickhouse-admin/src/http_entrypoints.rs | 28 +- clickhouse-admin/src/lib.rs | 51 +- clickhouse-admin/tests/integration_test.rs | 5 +- clickhouse-admin/types/src/config.rs | 16 +- clickhouse-admin/types/src/lib.rs | 43 +- .../Cargo.toml | 3 +- .../src/lib.rs | 7 +- .../clickhouse-admin-server-client/Cargo.toml | 18 + .../clickhouse-admin-server-client/src/lib.rs | 26 + dev-tools/openapi-manager/src/spec.rs | 19 +- nexus/Cargo.toml | 3 +- nexus/inventory/Cargo.toml | 3 +- nexus/inventory/src/collector.rs | 6 +- nexus/reconfigurator/execution/Cargo.toml | 4 +- .../execution/src/clickhouse.rs | 73 +- .../background/tasks/inventory_collection.rs | 2 +- openapi/clickhouse-admin-keeper.json | 898 ++++++++++++++++++ openapi/clickhouse-admin-server.json | 423 +++++++++ 24 files changed, 1675 insertions(+), 142 deletions(-) rename clickhouse-admin/src/bin/{clickhouse-admin.rs => clickhouse-admin-keeper.rs} (90%) create mode 100644 clickhouse-admin/src/bin/clickhouse-admin-server.rs rename clients/{clickhouse-admin-client => clickhouse-admin-keeper-client}/Cargo.toml (85%) rename clients/{clickhouse-admin-client => clickhouse-admin-keeper-client}/src/lib.rs (76%) create mode 100644 clients/clickhouse-admin-server-client/Cargo.toml create mode 100644 clients/clickhouse-admin-server-client/src/lib.rs create mode 100644 openapi/clickhouse-admin-keeper.json create mode 100644 openapi/clickhouse-admin-server.json diff --git a/Cargo.lock b/Cargo.lock index dd2f30c2a0..0ddb16472b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1295,11 +1295,25 @@ dependencies = [ ] [[package]] -name = "clickhouse-admin-client" +name = "clickhouse-admin-keeper-client" +version = "0.1.0" +dependencies = [ + "chrono", + "clickhouse-admin-types", + "omicron-uuid-kinds", + "omicron-workspace-hack", + "progenitor", + "reqwest 0.12.7", + "schemars", + "serde", + "slog", +] + +[[package]] +name = "clickhouse-admin-server-client" version = "0.1.0" dependencies = [ "chrono", - "clickhouse-admin-api", "clickhouse-admin-types", "omicron-uuid-kinds", "omicron-workspace-hack", @@ -5603,7 +5617,8 @@ dependencies = [ "anyhow", "base64 0.22.1", "chrono", - "clickhouse-admin-client", + "clickhouse-admin-keeper-client", + "clickhouse-admin-server-client", "clickhouse-admin-types", "expectorate", "futures", @@ -5689,8 +5704,8 @@ dependencies = [ "async-bb8-diesel", "camino", "chrono", - "clickhouse-admin-api", - "clickhouse-admin-client", + "clickhouse-admin-keeper-client", + "clickhouse-admin-server-client", "clickhouse-admin-types", "cockroach-admin-client", "diesel", @@ -6543,7 +6558,8 @@ dependencies = [ "cancel-safe-futures", "chrono", "clap", - "clickhouse-admin-client", + "clickhouse-admin-keeper-client", + "clickhouse-admin-server-client", "cockroach-admin-client", "criterion", "crucible-agent-client", diff --git a/Cargo.toml b/Cargo.toml index ea0a46d49c..50660bf9bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,8 @@ members = [ "clickhouse-admin", "clickhouse-admin/api", "clients/bootstrap-agent-client", - "clients/clickhouse-admin-client", + "clients/clickhouse-admin-keeper-client", + "clients/clickhouse-admin-server-client", "clients/cockroach-admin-client", "clients/ddm-admin-client", "clients/dns-service-client", @@ -128,7 +129,8 @@ default-members = [ "clickhouse-admin/api", "clickhouse-admin/types", "clients/bootstrap-agent-client", - "clients/clickhouse-admin-client", + "clients/clickhouse-admin-keeper-client", + "clients/clickhouse-admin-server-client", "clients/cockroach-admin-client", "clients/ddm-admin-client", "clients/dns-service-client", @@ -313,7 +315,8 @@ chrono = { version = "0.4", features = [ "serde" ] } ciborium = "0.2.2" clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] } clickhouse-admin-api = { path = "clickhouse-admin/api" } -clickhouse-admin-client = { path = "clients/clickhouse-admin-client" } +clickhouse-admin-keeper-client = { path = "clients/clickhouse-admin-keeper-client" } +clickhouse-admin-server-client = { path = "clients/clickhouse-admin-server-client" } clickhouse-admin-types = { path = "clickhouse-admin/types" } clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "ceec762e6a87d2a22bf56792a3025e145caa095e" } cockroach-admin-api = { path = "cockroach-admin/api" } diff --git a/clickhouse-admin/api/src/lib.rs b/clickhouse-admin/api/src/lib.rs index bdd29f52b8..eb49c3632e 100644 --- a/clickhouse-admin/api/src/lib.rs +++ b/clickhouse-admin/api/src/lib.rs @@ -2,56 +2,29 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, KeeperConf, KeeperSettings, Lgif, - RaftConfig, ServerSettings, + ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig, + KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig, + ServerConfigurableSettings, }; use dropshot::{ HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody, }; -use omicron_common::api::external::Generation; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, Deserialize, JsonSchema)] -pub struct ServerConfigurableSettings { - /// A unique identifier for the configuration generation. - pub generation: Generation, - /// Configurable settings for a ClickHouse replica server node. - pub settings: ServerSettings, -} - -#[derive(Debug, Serialize, Deserialize, JsonSchema)] -pub struct KeeperConfigurableSettings { - /// A unique identifier for the configuration generation. - pub generation: Generation, - /// Configurable settings for a ClickHouse keeper node. - pub settings: KeeperSettings, -} #[dropshot::api_description] -pub trait ClickhouseAdminApi { +pub trait ClickhouseAdminKeeperApi { type Context; - /// Generate a ClickHouse configuration file for a server node on a specified - /// directory and enable the SMF service. - #[endpoint { - method = PUT, - path = "/server/config-and-enable", - }] - async fn generate_server_config_and_enable( - rqctx: RequestContext, - body: TypedBody, - ) -> Result, HttpError>; - /// Generate a ClickHouse configuration file for a keeper node on a specified - /// directory and enable the SMF service. + /// directory and enable the SMF service if not currently enabled. + /// + /// Note that we cannot start the keeper service until there is an initial + /// configuration set via this endpoint. #[endpoint { method = PUT, - path = "/keeper/config-and-enable", + path = "/config", }] - async fn generate_keeper_config_and_enable( + async fn generate_config( rqctx: RequestContext, body: TypedBody, ) -> Result, HttpError>; @@ -95,3 +68,19 @@ pub trait ClickhouseAdminApi { rqctx: RequestContext, ) -> Result, HttpError>; } + +#[dropshot::api_description] +pub trait ClickhouseAdminServerApi { + type Context; + + /// Generate a ClickHouse configuration file for a server node on a specified + /// directory and enable the SMF service. + #[endpoint { + method = PUT, + path = "/config" + }] + async fn generate_config( + rqctx: RequestContext, + body: TypedBody, + ) -> Result, HttpError>; +} diff --git a/clickhouse-admin/src/bin/clickhouse-admin.rs b/clickhouse-admin/src/bin/clickhouse-admin-keeper.rs similarity index 90% rename from clickhouse-admin/src/bin/clickhouse-admin.rs rename to clickhouse-admin/src/bin/clickhouse-admin-keeper.rs index 3391a3459a..4ec998920b 100644 --- a/clickhouse-admin/src/bin/clickhouse-admin.rs +++ b/clickhouse-admin/src/bin/clickhouse-admin-keeper.rs @@ -2,7 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Executable program to run the Omicron ClickHouse admin interface +//! Executable program to run the Omicron ClickHouse admin interface for +//! clickhouse keepers. use anyhow::anyhow; use camino::Utf8PathBuf; @@ -14,8 +15,8 @@ use std::net::{SocketAddr, SocketAddrV6}; #[derive(Debug, Parser)] #[clap( - name = "clickhouse-admin", - about = "Omicron ClickHouse cluster admin server" + name = "clickhouse-admin-keeper", + about = "Omicron ClickHouse cluster admin server for keepers" )] enum Args { /// Start the ClickHouse admin server @@ -57,7 +58,7 @@ async fn main_impl() -> Result<(), CmdError> { let clickhouse_cli = ClickhouseCli::new(binary_path, listen_address); - let server = omicron_clickhouse_admin::start_server( + let server = omicron_clickhouse_admin::start_keeper_admin_server( clickward, clickhouse_cli, config, diff --git a/clickhouse-admin/src/bin/clickhouse-admin-server.rs b/clickhouse-admin/src/bin/clickhouse-admin-server.rs new file mode 100644 index 0000000000..1258f75805 --- /dev/null +++ b/clickhouse-admin/src/bin/clickhouse-admin-server.rs @@ -0,0 +1,75 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Executable program to run the Omicron ClickHouse admin interface for +//! clickhouse servers. + +use anyhow::anyhow; +use camino::Utf8PathBuf; +use clap::Parser; +use omicron_clickhouse_admin::{ClickhouseCli, Clickward, Config}; +use omicron_common::cmd::fatal; +use omicron_common::cmd::CmdError; +use std::net::{SocketAddr, SocketAddrV6}; + +#[derive(Debug, Parser)] +#[clap( + name = "clickhouse-admin-server", + about = "Omicron ClickHouse cluster admin server for replica servers" +)] +enum Args { + /// Start the ClickHouse admin server + Run { + /// Address on which this server should run + #[clap(long, short = 'a', action)] + http_address: SocketAddrV6, + + /// Path to the server configuration file + #[clap(long, short, action)] + config: Utf8PathBuf, + + /// Address on which the clickhouse server or keeper is listening on + #[clap(long, short = 'l', action)] + listen_address: SocketAddrV6, + + /// Path to the clickhouse binary + #[clap(long, short, action)] + binary_path: Utf8PathBuf, + }, +} + +#[tokio::main] +async fn main() { + if let Err(err) = main_impl().await { + fatal(err); + } +} + +async fn main_impl() -> Result<(), CmdError> { + let args = Args::parse(); + + match args { + Args::Run { http_address, config, listen_address, binary_path } => { + let mut config = Config::from_file(&config) + .map_err(|err| CmdError::Failure(anyhow!(err)))?; + config.dropshot.bind_address = SocketAddr::V6(http_address); + let clickward = Clickward::new(); + let clickhouse_cli = + ClickhouseCli::new(binary_path, listen_address); + + let server = omicron_clickhouse_admin::start_server_admin_server( + clickward, + clickhouse_cli, + config, + ) + .await + .map_err(|err| CmdError::Failure(anyhow!(err)))?; + server.await.map_err(|err| { + CmdError::Failure(anyhow!( + "server failed after starting: {err}" + )) + }) + } + } +} diff --git a/clickhouse-admin/src/clickward.rs b/clickhouse-admin/src/clickward.rs index 5202b9b090..ca5d3df3de 100644 --- a/clickhouse-admin/src/clickward.rs +++ b/clickhouse-admin/src/clickward.rs @@ -2,8 +2,9 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; -use clickhouse_admin_types::{KeeperSettings, ServerSettings}; +use clickhouse_admin_types::{ + KeeperConfig, KeeperSettings, ReplicaConfig, ServerSettings, +}; use dropshot::HttpError; use slog_error_chain::{InlineErrorChain, SlogInlineError}; diff --git a/clickhouse-admin/src/http_entrypoints.rs b/clickhouse-admin/src/http_entrypoints.rs index 72cd105862..49138b9cc3 100644 --- a/clickhouse-admin/src/http_entrypoints.rs +++ b/clickhouse-admin/src/http_entrypoints.rs @@ -4,9 +4,10 @@ use crate::context::ServerContext; use clickhouse_admin_api::*; -use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, KeeperConf, Lgif, RaftConfig, + ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig, + KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig, + ServerConfigurableSettings, }; use dropshot::{ HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody, @@ -16,17 +17,22 @@ use std::sync::Arc; type ClickhouseApiDescription = dropshot::ApiDescription>; -pub fn api() -> ClickhouseApiDescription { - clickhouse_admin_api_mod::api_description::() +pub fn clickhouse_admin_server_api() -> ClickhouseApiDescription { + clickhouse_admin_server_api_mod::api_description::() .expect("registered entrypoints") } -enum ClickhouseAdminImpl {} +pub fn clickhouse_admin_keeper_api() -> ClickhouseApiDescription { + clickhouse_admin_keeper_api_mod::api_description::() + .expect("registered entrypoints") +} -impl ClickhouseAdminApi for ClickhouseAdminImpl { +enum ClickhouseAdminServerImpl {} + +impl ClickhouseAdminServerApi for ClickhouseAdminServerImpl { type Context = Arc; - async fn generate_server_config_and_enable( + async fn generate_config( rqctx: RequestContext, body: TypedBody, ) -> Result, HttpError> { @@ -41,8 +47,14 @@ impl ClickhouseAdminApi for ClickhouseAdminImpl { Ok(HttpResponseCreated(output)) } +} + +enum ClickhouseAdminKeeperImpl {} + +impl ClickhouseAdminKeeperApi for ClickhouseAdminKeeperImpl { + type Context = Arc; - async fn generate_keeper_config_and_enable( + async fn generate_config( rqctx: RequestContext, body: TypedBody, ) -> Result, HttpError> { diff --git a/clickhouse-admin/src/lib.rs b/clickhouse-admin/src/lib.rs index 511a32dd50..1697d24adc 100644 --- a/clickhouse-admin/src/lib.rs +++ b/clickhouse-admin/src/lib.rs @@ -33,8 +33,9 @@ pub enum StartError { pub type Server = dropshot::HttpServer>; -/// Start the dropshot server -pub async fn start_server( +/// Start the dropshot server for `clickhouse-admin-server` which +/// manages clickhouse replica servers. +pub async fn start_server_admin_server( clickward: Clickward, clickhouse_cli: ClickhouseCli, server_config: Config, @@ -42,7 +43,7 @@ pub async fn start_server( let (drain, registration) = slog_dtrace::with_drain( server_config .log - .to_logger("clickhouse-admin") + .to_logger("clickhouse-admin-server") .map_err(StartError::InitializeLogger)?, ); let log = slog::Logger::root(drain.fuse(), slog::o!(FileKv)); @@ -65,7 +66,49 @@ pub async fn start_server( ); let http_server_starter = dropshot::HttpServerStarter::new( &server_config.dropshot, - http_entrypoints::api(), + http_entrypoints::clickhouse_admin_server_api(), + Arc::new(context), + &log.new(slog::o!("component" => "dropshot")), + ) + .map_err(StartError::InitializeHttpServer)?; + + Ok(http_server_starter.start()) +} + +/// Start the dropshot server for `clickhouse-admin-server` which +/// manages clickhouse replica servers. +pub async fn start_keeper_admin_server( + clickward: Clickward, + clickhouse_cli: ClickhouseCli, + server_config: Config, +) -> Result { + let (drain, registration) = slog_dtrace::with_drain( + server_config + .log + .to_logger("clickhouse-admin-keeper") + .map_err(StartError::InitializeLogger)?, + ); + let log = slog::Logger::root(drain.fuse(), slog::o!(FileKv)); + match registration { + ProbeRegistration::Success => { + debug!(log, "registered DTrace probes"); + } + ProbeRegistration::Failed(err) => { + let err = StartError::RegisterDtraceProbes(err); + error!(log, "failed to register DTrace probes"; &err); + return Err(err); + } + } + + let context = ServerContext::new( + clickward, + clickhouse_cli + .with_log(log.new(slog::o!("component" => "ClickhouseCli"))), + log.new(slog::o!("component" => "ServerContext")), + ); + let http_server_starter = dropshot::HttpServerStarter::new( + &server_config.dropshot, + http_entrypoints::clickhouse_admin_keeper_api(), Arc::new(context), &log.new(slog::o!("component" => "dropshot")), ) diff --git a/clickhouse-admin/tests/integration_test.rs b/clickhouse-admin/tests/integration_test.rs index fd706722a3..eb26bec668 100644 --- a/clickhouse-admin/tests/integration_test.rs +++ b/clickhouse-admin/tests/integration_test.rs @@ -4,10 +4,9 @@ use anyhow::Context; use camino::Utf8PathBuf; -use clickhouse_admin_types::config::ClickhouseHost; use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, KeeperId, KeeperServerInfo, - KeeperServerType, RaftConfig, + ClickhouseHost, ClickhouseKeeperClusterMembership, KeeperId, + KeeperServerInfo, KeeperServerType, RaftConfig, }; use clickward::{BasePorts, Deployment, DeploymentConfig}; use dropshot::test_util::log_prefix_for_test; diff --git a/clickhouse-admin/types/src/config.rs b/clickhouse-admin/types/src/config.rs index 3cbf4c7fe2..20aa3d71f8 100644 --- a/clickhouse-admin/types/src/config.rs +++ b/clickhouse-admin/types/src/config.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::{KeeperId, ServerId, OXIMETER_CLUSTER}; +use crate::{path_schema, KeeperId, ServerId, OXIMETER_CLUSTER}; use anyhow::{bail, Error}; use camino::Utf8PathBuf; use omicron_common::address::{ @@ -10,23 +10,11 @@ use omicron_common::address::{ CLICKHOUSE_KEEPER_RAFT_PORT, CLICKHOUSE_KEEPER_TCP_PORT, CLICKHOUSE_TCP_PORT, }; -use schemars::{ - gen::SchemaGenerator, - schema::{Schema, SchemaObject}, - JsonSchema, -}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::net::{Ipv4Addr, Ipv6Addr}; use std::{fmt::Display, str::FromStr}; -// Used for schemars to be able to be used with camino: -// See https://github.com/camino-rs/camino/issues/91#issuecomment-2027908513 -pub fn path_schema(gen: &mut SchemaGenerator) -> Schema { - let mut schema: SchemaObject = ::json_schema(gen).into(); - schema.format = Some("Utf8PathBuf".to_owned()); - schema.into() -} - /// Configuration for a ClickHouse replica server #[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] pub struct ReplicaConfig { diff --git a/clickhouse-admin/types/src/lib.rs b/clickhouse-admin/types/src/lib.rs index 83d56be1a1..021b9db356 100644 --- a/clickhouse-admin/types/src/lib.rs +++ b/clickhouse-admin/types/src/lib.rs @@ -7,7 +7,12 @@ use atomicwrites::AtomicFile; use camino::Utf8PathBuf; use derive_more::{Add, AddAssign, Display, From}; use itertools::Itertools; -use schemars::JsonSchema; +use omicron_common::api::external::Generation; +use schemars::{ + gen::SchemaGenerator, + schema::{Schema, SchemaObject}, + JsonSchema, +}; use serde::{Deserialize, Serialize}; use slog::{info, Logger}; use std::collections::BTreeSet; @@ -16,8 +21,20 @@ use std::io::{ErrorKind, Write}; use std::net::Ipv6Addr; use std::str::FromStr; -pub mod config; -use config::*; +mod config; +pub use config::{ + ClickhouseHost, KeeperConfig, KeeperConfigsForReplica, KeeperNodeConfig, + LogConfig, LogLevel, Macros, NodeType, RaftServerConfig, + RaftServerSettings, RaftServers, ReplicaConfig, ServerNodeConfig, +}; + +// Used for schemars to be able to be used with camino: +// See https://github.com/camino-rs/camino/issues/91#issuecomment-2027908513 +pub fn path_schema(gen: &mut SchemaGenerator) -> Schema { + let mut schema: SchemaObject = ::json_schema(gen).into(); + schema.format = Some("Utf8PathBuf".to_owned()); + schema.into() +} pub const OXIMETER_CLUSTER: &str = "oximeter_cluster"; @@ -59,6 +76,26 @@ pub struct KeeperId(pub u64); )] pub struct ServerId(pub u64); +/// The top most type for configuring clickhouse-servers via +/// clickhouse-admin-server-api +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct ServerConfigurableSettings { + /// A unique identifier for the configuration generation. + pub generation: Generation, + /// Configurable settings for a ClickHouse replica server node. + pub settings: ServerSettings, +} + +/// The top most type for configuring clickhouse-servers via +/// clickhouse-admin-keeper-api +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct KeeperConfigurableSettings { + /// A unique identifier for the configuration generation. + pub generation: Generation, + /// Configurable settings for a ClickHouse keeper node. + pub settings: KeeperSettings, +} + /// Configurable settings for a ClickHouse replica server node. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "snake_case")] diff --git a/clients/clickhouse-admin-client/Cargo.toml b/clients/clickhouse-admin-keeper-client/Cargo.toml similarity index 85% rename from clients/clickhouse-admin-client/Cargo.toml rename to clients/clickhouse-admin-keeper-client/Cargo.toml index c1e05c4f71..1b8839ae64 100644 --- a/clients/clickhouse-admin-client/Cargo.toml +++ b/clients/clickhouse-admin-keeper-client/Cargo.toml @@ -1,12 +1,11 @@ [package] -name = "clickhouse-admin-client" +name = "clickhouse-admin-keeper-client" version = "0.1.0" edition = "2021" [dependencies] chrono.workspace = true clickhouse-admin-types.workspace = true -clickhouse-admin-api.workspace = true omicron-uuid-kinds.workspace = true progenitor.workspace = true reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] } diff --git a/clients/clickhouse-admin-client/src/lib.rs b/clients/clickhouse-admin-keeper-client/src/lib.rs similarity index 76% rename from clients/clickhouse-admin-client/src/lib.rs rename to clients/clickhouse-admin-keeper-client/src/lib.rs index 067c3e6939..0bdc3a1350 100644 --- a/clients/clickhouse-admin-client/src/lib.rs +++ b/clients/clickhouse-admin-keeper-client/src/lib.rs @@ -2,11 +2,11 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Interface for making API requests to a clickhouse-admin server running in +//! Interface for making API requests to a clickhouse-admin-{keeper|server} server running in //! an omicron zone. progenitor::generate_api!( - spec = "../../openapi/clickhouse-admin.json", + spec = "../../openapi/clickhouse-admin-keeper.json", inner_type = slog::Logger, pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { slog::debug!(log, "client request"; @@ -21,8 +21,7 @@ progenitor::generate_api!( derives = [schemars::JsonSchema], replace = { TypedUuidForOmicronZoneKind = omicron_uuid_kinds::OmicronZoneUuid, - KeeperConfigurableSettings = clickhouse_admin_api::KeeperConfigurableSettings, - ServerConfigurableSettings = clickhouse_admin_api::ServerConfigurableSettings, + KeeperConfigurableSettings = clickhouse_admin_types::KeeperConfigurableSettings, ClickhouseKeeperClusterMembership = clickhouse_admin_types::ClickhouseKeeperClusterMembership, KeeperId = clickhouse_admin_types::KeeperId } diff --git a/clients/clickhouse-admin-server-client/Cargo.toml b/clients/clickhouse-admin-server-client/Cargo.toml new file mode 100644 index 0000000000..61142f8a7b --- /dev/null +++ b/clients/clickhouse-admin-server-client/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "clickhouse-admin-server-client" +version = "0.1.0" +edition = "2021" + +[dependencies] +chrono.workspace = true +clickhouse-admin-types.workspace = true +omicron-uuid-kinds.workspace = true +progenitor.workspace = true +reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] } +schemars.workspace = true +serde.workspace = true +slog.workspace = true +omicron-workspace-hack.workspace = true + +[lints] +workspace = true diff --git a/clients/clickhouse-admin-server-client/src/lib.rs b/clients/clickhouse-admin-server-client/src/lib.rs new file mode 100644 index 0000000000..f5b41ffeb5 --- /dev/null +++ b/clients/clickhouse-admin-server-client/src/lib.rs @@ -0,0 +1,26 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Interface for making API requests to a clickhouse-admin-{keeper|server} server running in +//! an omicron zone. + +progenitor::generate_api!( + spec = "../../openapi/clickhouse-admin-server.json", + inner_type = slog::Logger, + pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + }), + post_hook = (|log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + }), + derives = [schemars::JsonSchema], + replace = { + TypedUuidForOmicronZoneKind = omicron_uuid_kinds::OmicronZoneUuid, + ServerConfigurableSettings = clickhouse_admin_types::ServerConfigurableSettings, + } +); diff --git a/dev-tools/openapi-manager/src/spec.rs b/dev-tools/openapi-manager/src/spec.rs index 7d734218fc..dd01100346 100644 --- a/dev-tools/openapi-manager/src/spec.rs +++ b/dev-tools/openapi-manager/src/spec.rs @@ -27,14 +27,25 @@ pub fn all_apis() -> Vec { extra_validation: None, }, ApiSpec { - title: "ClickHouse Cluster Admin API", + title: "ClickHouse Cluster Admin Keeper API", version: "0.0.1", description: "API for interacting with the Oxide \ - control plane's ClickHouse cluster", + control plane's ClickHouse cluster keepers", boundary: ApiBoundary::Internal, api_description: - clickhouse_admin_api::clickhouse_admin_api_mod::stub_api_description, - filename: "clickhouse-admin.json", + clickhouse_admin_api::clickhouse_admin_keeper_api_mod::stub_api_description, + filename: "clickhouse-admin-keeper.json", + extra_validation: None, + }, + ApiSpec { + title: "ClickHouse Cluster Admin Keeper API", + version: "0.0.1", + description: "API for interacting with the Oxide \ + control plane's ClickHouse cluster replica servers", + boundary: ApiBoundary::Internal, + api_description: + clickhouse_admin_api::clickhouse_admin_server_api_mod::stub_api_description, + filename: "clickhouse-admin-server.json", extra_validation: None, }, ApiSpec { diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 8e54baf2d3..d430009360 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -22,7 +22,8 @@ camino.workspace = true camino-tempfile.workspace = true clap.workspace = true chrono.workspace = true -clickhouse-admin-client.workspace = true +clickhouse-admin-keeper-client.workspace = true +clickhouse-admin-server-client.workspace = true cockroach-admin-client.workspace = true crucible-agent-client.workspace = true crucible-pantry-client.workspace = true diff --git a/nexus/inventory/Cargo.toml b/nexus/inventory/Cargo.toml index f664523950..b5ca91283e 100644 --- a/nexus/inventory/Cargo.toml +++ b/nexus/inventory/Cargo.toml @@ -11,7 +11,8 @@ workspace = true anyhow.workspace = true base64.workspace = true chrono.workspace = true -clickhouse-admin-client.workspace = true +clickhouse-admin-keeper-client.workspace = true +clickhouse-admin-server-client.workspace = true clickhouse-admin-types.workspace = true futures.workspace = true gateway-client.workspace = true diff --git a/nexus/inventory/src/collector.rs b/nexus/inventory/src/collector.rs index dda000cd8c..eebc692019 100644 --- a/nexus/inventory/src/collector.rs +++ b/nexus/inventory/src/collector.rs @@ -28,7 +28,7 @@ const SLED_AGENT_TIMEOUT: Duration = Duration::from_secs(60); pub struct Collector<'a> { log: slog::Logger, mgs_clients: Vec, - keeper_admin_clients: Vec, + keeper_admin_clients: Vec, sled_agent_lister: &'a (dyn SledAgentEnumerator + Send + Sync), in_progress: CollectionBuilder, } @@ -37,7 +37,7 @@ impl<'a> Collector<'a> { pub fn new( creator: &str, mgs_clients: Vec, - keeper_admin_clients: Vec, + keeper_admin_clients: Vec, sled_agent_lister: &'a (dyn SledAgentEnumerator + Send + Sync), log: slog::Logger, ) -> Self { @@ -384,7 +384,7 @@ impl<'a> Collector<'a> { /// Collect inventory about one keeper from one `ClickhouseAdminKeeper` async fn collect_one_keeper( - client: &clickhouse_admin_client::Client, + client: &clickhouse_admin_keeper_client::Client, log: &slog::Logger, in_progress: &mut CollectionBuilder, ) { diff --git a/nexus/reconfigurator/execution/Cargo.toml b/nexus/reconfigurator/execution/Cargo.toml index e3ccddf846..21d861ef51 100644 --- a/nexus/reconfigurator/execution/Cargo.toml +++ b/nexus/reconfigurator/execution/Cargo.toml @@ -12,8 +12,8 @@ omicron-rpaths.workspace = true [dependencies] anyhow.workspace = true camino.workspace = true -clickhouse-admin-api.workspace = true -clickhouse-admin-client.workspace = true +clickhouse-admin-keeper-client.workspace = true +clickhouse-admin-server-client.workspace = true clickhouse-admin-types.workspace = true cockroach-admin-client.workspace = true chrono.workspace = true diff --git a/nexus/reconfigurator/execution/src/clickhouse.rs b/nexus/reconfigurator/execution/src/clickhouse.rs index 926a608e2c..e623952ed9 100644 --- a/nexus/reconfigurator/execution/src/clickhouse.rs +++ b/nexus/reconfigurator/execution/src/clickhouse.rs @@ -7,12 +7,13 @@ use anyhow::anyhow; use camino::Utf8PathBuf; -use clickhouse_admin_api::KeeperConfigurableSettings; -use clickhouse_admin_api::ServerConfigurableSettings; -use clickhouse_admin_client::Client; -use clickhouse_admin_types::config::ClickhouseHost; -use clickhouse_admin_types::config::RaftServerSettings; +use clickhouse_admin_keeper_client::Client as ClickhouseKeeperClient; +use clickhouse_admin_server_client::Client as ClickhouseServerClient; +use clickhouse_admin_types::ClickhouseHost; +use clickhouse_admin_types::KeeperConfigurableSettings; use clickhouse_admin_types::KeeperSettings; +use clickhouse_admin_types::RaftServerSettings; +use clickhouse_admin_types::ServerConfigurableSettings; use clickhouse_admin_types::ServerSettings; use futures::future::Either; use futures::stream::FuturesUnordered; @@ -91,23 +92,19 @@ pub(crate) async fn deploy_nodes( let admin_url = format!("http://{admin_addr}"); let log = log.new(slog::o!("admin_url" => admin_url.clone())); futs.push(Either::Left(async move { - let client = Client::new(&admin_url, log.clone()); - client - .generate_keeper_config_and_enable(&config) - .await - .map(|_| ()) - .map_err(|e| { - anyhow!( - concat!( - "failed to send config for clickhouse keeper ", - "with id {} to clickhouse-admin; admin_url = {}", - "error = {}" - ), - config.settings.id, - admin_url, - e - ) - }) + let client = ClickhouseKeeperClient::new(&admin_url, log.clone()); + client.generate_config(&config).await.map(|_| ()).map_err(|e| { + anyhow!( + concat!( + "failed to send config for clickhouse keeper ", + "with id {} to clickhouse-admin-keeper; admin_url = {}", + "error = {}" + ), + config.settings.id, + admin_url, + e + ) + }) })); } for config in server_configs { @@ -120,23 +117,19 @@ pub(crate) async fn deploy_nodes( let admin_url = format!("http://{admin_addr}"); let log = opctx.log.new(slog::o!("admin_url" => admin_url.clone())); futs.push(Either::Right(async move { - let client = Client::new(&admin_url, log.clone()); - client - .generate_server_config_and_enable(&config) - .await - .map(|_| ()) - .map_err(|e| { - anyhow!( - concat!( - "failed to send config for clickhouse server ", - "with id {} to clickhouse-admin; admin_url = {}", - "error = {}" - ), - config.settings.id, - admin_url, - e - ) - }) + let client = ClickhouseServerClient::new(&admin_url, log.clone()); + client.generate_config(&config).await.map(|_| ()).map_err(|e| { + anyhow!( + concat!( + "failed to send config for clickhouse server ", + "with id {} to clickhouse-admin-server; admin_url = {}", + "error = {}" + ), + config.settings.id, + admin_url, + e + ) + }) })); } @@ -285,7 +278,7 @@ fn keeper_configs( #[cfg(test)] mod test { use super::*; - use clickhouse_admin_types::config::ClickhouseHost; + use clickhouse_admin_types::ClickhouseHost; use clickhouse_admin_types::KeeperId; use clickhouse_admin_types::ServerId; use nexus_sled_agent_shared::inventory::OmicronZoneDataset; diff --git a/nexus/src/app/background/tasks/inventory_collection.rs b/nexus/src/app/background/tasks/inventory_collection.rs index e0b2f25365..c4271c58d8 100644 --- a/nexus/src/app/background/tasks/inventory_collection.rs +++ b/nexus/src/app/background/tasks/inventory_collection.rs @@ -142,7 +142,7 @@ async fn inventory_activate( let url = format!("http://{}", sockaddr); let log = opctx.log.new(o!("clickhouse_admin_keeper_url" => url.clone())); - clickhouse_admin_client::Client::new(&url, log) + clickhouse_admin_keeper_client::Client::new(&url, log) }) .collect::>(); diff --git a/openapi/clickhouse-admin-keeper.json b/openapi/clickhouse-admin-keeper.json new file mode 100644 index 0000000000..3c48a082b7 --- /dev/null +++ b/openapi/clickhouse-admin-keeper.json @@ -0,0 +1,898 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "ClickHouse Cluster Admin Keeper API", + "description": "API for interacting with the Oxide control plane's ClickHouse cluster keepers", + "contact": { + "url": "https://oxide.computer", + "email": "api@oxide.computer" + }, + "version": "0.0.1" + }, + "paths": { + "/config": { + "put": { + "summary": "Generate a ClickHouse configuration file for a keeper node on a specified", + "description": "directory and enable the SMF service if not currently enabled.\n\nNote that we cannot start the keeper service until there is an initial configuration set via this endpoint.", + "operationId": "generate_config", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/KeeperConfigurableSettings" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/KeeperConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/keeper/cluster-membership": { + "get": { + "summary": "Retrieve cluster membership information from a keeper node.", + "operationId": "keeper_cluster_membership", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClickhouseKeeperClusterMembership" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/keeper/conf": { + "get": { + "summary": "Retrieve configuration information from a keeper node.", + "operationId": "keeper_conf", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/KeeperConf" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/keeper/lgif": { + "get": { + "summary": "Retrieve a logically grouped information file from a keeper node.", + "description": "This information is used internally by ZooKeeper to manage snapshots and logs for consistency and recovery.", + "operationId": "lgif", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Lgif" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/keeper/raft-config": { + "get": { + "summary": "Retrieve information from ClickHouse virtual node /keeper/config which", + "description": "contains last committed cluster configuration.", + "operationId": "raft_config", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RaftConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "ClickhouseHost": { + "oneOf": [ + { + "type": "object", + "properties": { + "ipv6": { + "type": "string", + "format": "ipv6" + } + }, + "required": [ + "ipv6" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ipv4": { + "type": "string", + "format": "ipv4" + } + }, + "required": [ + "ipv4" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "domain_name": { + "type": "string" + } + }, + "required": [ + "domain_name" + ], + "additionalProperties": false + } + ] + }, + "ClickhouseKeeperClusterMembership": { + "description": "The configuration of the clickhouse keeper raft cluster returned from a single keeper node\n\nEach keeper is asked for its known raft configuration via `clickhouse-admin` dropshot servers running in `ClickhouseKeeper` zones. state. We include the leader committed log index known to the current keeper node (whether or not it is the leader) to determine which configuration is newest.", + "type": "object", + "properties": { + "leader_committed_log_index": { + "description": "Index of the last committed log entry from the leader's perspective", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "queried_keeper": { + "description": "Keeper ID of the keeper being queried", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperId" + } + ] + }, + "raft_config": { + "description": "Keeper IDs of all keepers in the cluster", + "type": "array", + "items": { + "$ref": "#/components/schemas/KeeperId" + }, + "uniqueItems": true + } + }, + "required": [ + "leader_committed_log_index", + "queried_keeper", + "raft_config" + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "Generation": { + "description": "Generation numbers stored in the database, used for optimistic concurrency control", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "KeeperConf": { + "description": "Keeper configuration information", + "type": "object", + "properties": { + "auto_forwarding": { + "description": "Allow to forward write requests from followers to the leader.", + "type": "boolean" + }, + "compress_logs": { + "description": "Whether to write compressed coordination logs in ZSTD format.", + "type": "boolean" + }, + "compress_snapshots_with_zstd_format": { + "description": "Whether to write compressed snapshots in ZSTD format (instead of custom LZ4).", + "type": "boolean" + }, + "configuration_change_tries_count": { + "description": "How many times we will try to apply configuration change (add/remove server) to the cluster.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "dead_session_check_period_ms": { + "description": "How often ClickHouse Keeper checks for dead sessions and removes them (ms).", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "election_timeout_lower_bound_ms": { + "description": "If the follower does not receive a heartbeat from the leader in this interval, then it can initiate leader election. Must be less than or equal to election_timeout_upper_bound_ms. Ideally they shouldn't be equal.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "election_timeout_upper_bound_ms": { + "description": "If the follower does not receive a heartbeat from the leader in this interval, then it must initiate leader election.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "enable_ipv6": { + "description": "Whether Ipv6 is enabled.", + "type": "boolean" + }, + "force_sync": { + "description": "Whether to call fsync on each change in RAFT changelog.", + "type": "boolean" + }, + "four_letter_word_allow_list": { + "description": "Allow list of 4lw commands.", + "type": "string" + }, + "fresh_log_gap": { + "description": "When the node became fresh.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "heart_beat_interval_ms": { + "description": "How often a ClickHouse Keeper leader will send heartbeats to followers (ms).", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "log_storage_disk": { + "description": "Name of disk used for logs.", + "type": "string" + }, + "log_storage_path": { + "description": "Path to coordination logs, just like ZooKeeper it is best to store logs on non-busy nodes.", + "type": "string", + "format": "Utf8PathBuf" + }, + "max_request_queue_size": { + "description": "Maximum number of requests that can be in queue for processing.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_requests_batch_bytes_size": { + "description": "Max size in bytes of batch of requests that can be sent to RAFT.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_requests_batch_size": { + "description": "Max size of batch in requests count before it will be sent to RAFT.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "max_requests_quick_batch_size": { + "description": "Max size of batch of requests to try to get before proceeding with RAFT. Keeper will not wait for requests but take only requests that are already in the queue.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "min_session_timeout_ms": { + "description": "Min timeout for client session (ms).", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "operation_timeout_ms": { + "description": "Timeout for a single client operation (ms).", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "quorum_reads": { + "description": "Whether to execute read requests as writes through whole RAFT consesus with similar speed.", + "type": "boolean" + }, + "raft_limits_reconnect_limit": { + "description": "If connection to a peer is silent longer than this limit * (heartbeat interval), we re-establish the connection.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "raft_logs_level": { + "description": "Text logging level about coordination (trace, debug, and so on).", + "allOf": [ + { + "$ref": "#/components/schemas/LogLevel" + } + ] + }, + "reserved_log_items": { + "description": "How many coordination log records to store before compaction.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "rotate_log_storage_interval": { + "description": "How many log records to store in a single file.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "server_id": { + "description": "Unique server id, each participant of the ClickHouse Keeper cluster must have a unique number (1, 2, 3, and so on).", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperId" + } + ] + }, + "session_timeout_ms": { + "description": "Max timeout for client session (ms).", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "shutdown_timeout": { + "description": "Wait to finish internal connections and shutdown (ms).", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "snapshot_distance": { + "description": "How often ClickHouse Keeper will create new snapshots (in the number of records in logs).", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "snapshot_storage_disk": { + "description": "Name of disk used for storage.", + "type": "string" + }, + "snapshot_storage_path": { + "description": "Path to coordination snapshots.", + "type": "string", + "format": "Utf8PathBuf" + }, + "snapshots_to_keep": { + "description": "How many snapshots to keep.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "stale_log_gap": { + "description": "Threshold when leader considers follower as stale and sends the snapshot to it instead of logs.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "startup_timeout": { + "description": "If the server doesn't connect to other quorum participants in the specified timeout it will terminate (ms).", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "tcp_port": { + "description": "Port for a client to connect.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "auto_forwarding", + "compress_logs", + "compress_snapshots_with_zstd_format", + "configuration_change_tries_count", + "dead_session_check_period_ms", + "election_timeout_lower_bound_ms", + "election_timeout_upper_bound_ms", + "enable_ipv6", + "force_sync", + "four_letter_word_allow_list", + "fresh_log_gap", + "heart_beat_interval_ms", + "log_storage_disk", + "log_storage_path", + "max_request_queue_size", + "max_requests_batch_bytes_size", + "max_requests_batch_size", + "max_requests_quick_batch_size", + "min_session_timeout_ms", + "operation_timeout_ms", + "quorum_reads", + "raft_limits_reconnect_limit", + "raft_logs_level", + "reserved_log_items", + "rotate_log_storage_interval", + "server_id", + "session_timeout_ms", + "shutdown_timeout", + "snapshot_distance", + "snapshot_storage_disk", + "snapshot_storage_path", + "snapshots_to_keep", + "stale_log_gap", + "startup_timeout", + "tcp_port" + ] + }, + "KeeperConfig": { + "description": "Configuration for a ClickHouse keeper", + "type": "object", + "properties": { + "coordination_settings": { + "description": "Internal coordination settings", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperCoordinationSettings" + } + ] + }, + "datastore_path": { + "description": "Directory for all files generated by ClickHouse itself", + "type": "string", + "format": "Utf8PathBuf" + }, + "listen_host": { + "description": "Address the keeper is listening on", + "type": "string", + "format": "ipv6" + }, + "log_storage_path": { + "description": "Directory for coordination logs", + "type": "string", + "format": "Utf8PathBuf" + }, + "logger": { + "description": "Logging settings", + "allOf": [ + { + "$ref": "#/components/schemas/LogConfig" + } + ] + }, + "raft_config": { + "description": "Settings for each server in the keeper cluster", + "allOf": [ + { + "$ref": "#/components/schemas/RaftServers" + } + ] + }, + "server_id": { + "description": "Unique ID for this keeper node", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperId" + } + ] + }, + "snapshot_storage_path": { + "description": "Directory for coordination snapshot storage", + "type": "string", + "format": "Utf8PathBuf" + }, + "tcp_port": { + "description": "Port for TCP connections", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "coordination_settings", + "datastore_path", + "listen_host", + "log_storage_path", + "logger", + "raft_config", + "server_id", + "snapshot_storage_path", + "tcp_port" + ] + }, + "KeeperConfigurableSettings": { + "description": "The top most type for configuring clickhouse-servers via clickhouse-admin-keeper-api", + "type": "object", + "properties": { + "generation": { + "description": "A unique identifier for the configuration generation.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "settings": { + "description": "Configurable settings for a ClickHouse keeper node.", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperSettings" + } + ] + } + }, + "required": [ + "generation", + "settings" + ] + }, + "KeeperCoordinationSettings": { + "type": "object", + "properties": { + "operation_timeout_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "raft_logs_level": { + "$ref": "#/components/schemas/LogLevel" + }, + "session_timeout_ms": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "operation_timeout_ms", + "raft_logs_level", + "session_timeout_ms" + ] + }, + "KeeperId": { + "description": "A unique ID for a ClickHouse Keeper", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "KeeperServerInfo": { + "type": "object", + "properties": { + "host": { + "description": "Host of the keeper server", + "allOf": [ + { + "$ref": "#/components/schemas/ClickhouseHost" + } + ] + }, + "priority": { + "description": "non-negative integer telling which nodes should be prioritised on leader elections. Priority of 0 means server will never be a leader.", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "raft_port": { + "description": "Keeper server raft port", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "server_id": { + "description": "Unique, immutable ID of the keeper server", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperId" + } + ] + }, + "server_type": { + "description": "A keeper server either participant or learner (learner does not participate in leader elections).", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperServerType" + } + ] + } + }, + "required": [ + "host", + "priority", + "raft_port", + "server_id", + "server_type" + ] + }, + "KeeperServerType": { + "type": "string", + "enum": [ + "participant", + "learner" + ] + }, + "KeeperSettings": { + "description": "Configurable settings for a ClickHouse keeper node.", + "type": "object", + "properties": { + "config_dir": { + "description": "Directory for the generated keeper configuration XML file", + "type": "string", + "format": "Utf8PathBuf" + }, + "datastore_path": { + "description": "Directory for all files generated by ClickHouse itself", + "type": "string", + "format": "Utf8PathBuf" + }, + "id": { + "description": "Unique ID of the keeper node", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperId" + } + ] + }, + "listen_addr": { + "description": "Address the keeper is listening on", + "type": "string", + "format": "ipv6" + }, + "raft_servers": { + "description": "ID and host of each server in the keeper cluster", + "type": "array", + "items": { + "$ref": "#/components/schemas/RaftServerSettings" + } + } + }, + "required": [ + "config_dir", + "datastore_path", + "id", + "listen_addr", + "raft_servers" + ] + }, + "Lgif": { + "description": "Logically grouped information file from a keeper node", + "type": "object", + "properties": { + "first_log_idx": { + "description": "Index of the first log entry in the current log segment", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "first_log_term": { + "description": "Term of the leader when the first log entry was created", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "last_committed_log_idx": { + "description": "Index of the last committed log entry", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "last_log_idx": { + "description": "Index of the last log entry in the current log segment", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "last_log_term": { + "description": "Term of the leader when the last log entry was created", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "last_snapshot_idx": { + "description": "Index of the most recent snapshot taken", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "leader_committed_log_idx": { + "description": "Index of the last committed log entry from the leader's perspective", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "target_committed_log_idx": { + "description": "Target index for log commitment during replication or recovery", + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "first_log_idx", + "first_log_term", + "last_committed_log_idx", + "last_log_idx", + "last_log_term", + "last_snapshot_idx", + "leader_committed_log_idx", + "target_committed_log_idx" + ] + }, + "LogConfig": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "errorlog": { + "type": "string", + "format": "Utf8PathBuf" + }, + "level": { + "$ref": "#/components/schemas/LogLevel" + }, + "log": { + "type": "string", + "format": "Utf8PathBuf" + }, + "size": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "count", + "errorlog", + "level", + "log", + "size" + ] + }, + "LogLevel": { + "type": "string", + "enum": [ + "trace", + "debug" + ] + }, + "RaftConfig": { + "description": "Keeper raft configuration information", + "type": "object", + "properties": { + "keeper_servers": { + "type": "array", + "items": { + "$ref": "#/components/schemas/KeeperServerInfo" + }, + "uniqueItems": true + } + }, + "required": [ + "keeper_servers" + ] + }, + "RaftServerConfig": { + "type": "object", + "properties": { + "hostname": { + "$ref": "#/components/schemas/ClickhouseHost" + }, + "id": { + "$ref": "#/components/schemas/KeeperId" + }, + "port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "hostname", + "id", + "port" + ] + }, + "RaftServerSettings": { + "type": "object", + "properties": { + "host": { + "$ref": "#/components/schemas/ClickhouseHost" + }, + "id": { + "$ref": "#/components/schemas/KeeperId" + } + }, + "required": [ + "host", + "id" + ] + }, + "RaftServers": { + "type": "object", + "properties": { + "servers": { + "type": "array", + "items": { + "$ref": "#/components/schemas/RaftServerConfig" + } + } + }, + "required": [ + "servers" + ] + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/openapi/clickhouse-admin-server.json b/openapi/clickhouse-admin-server.json new file mode 100644 index 0000000000..12691565dc --- /dev/null +++ b/openapi/clickhouse-admin-server.json @@ -0,0 +1,423 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "ClickHouse Cluster Admin Keeper API", + "description": "API for interacting with the Oxide control plane's ClickHouse cluster replica servers", + "contact": { + "url": "https://oxide.computer", + "email": "api@oxide.computer" + }, + "version": "0.0.1" + }, + "paths": { + "/config": { + "put": { + "summary": "Generate a ClickHouse configuration file for a server node on a specified", + "description": "directory and enable the SMF service.", + "operationId": "generate_config", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServerConfigurableSettings" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReplicaConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "ClickhouseHost": { + "oneOf": [ + { + "type": "object", + "properties": { + "ipv6": { + "type": "string", + "format": "ipv6" + } + }, + "required": [ + "ipv6" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ipv4": { + "type": "string", + "format": "ipv4" + } + }, + "required": [ + "ipv4" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "domain_name": { + "type": "string" + } + }, + "required": [ + "domain_name" + ], + "additionalProperties": false + } + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "Generation": { + "description": "Generation numbers stored in the database, used for optimistic concurrency control", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "KeeperConfigsForReplica": { + "type": "object", + "properties": { + "nodes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/KeeperNodeConfig" + } + } + }, + "required": [ + "nodes" + ] + }, + "KeeperNodeConfig": { + "type": "object", + "properties": { + "host": { + "$ref": "#/components/schemas/ClickhouseHost" + }, + "port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "host", + "port" + ] + }, + "LogConfig": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "errorlog": { + "type": "string", + "format": "Utf8PathBuf" + }, + "level": { + "$ref": "#/components/schemas/LogLevel" + }, + "log": { + "type": "string", + "format": "Utf8PathBuf" + }, + "size": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "count", + "errorlog", + "level", + "log", + "size" + ] + }, + "LogLevel": { + "type": "string", + "enum": [ + "trace", + "debug" + ] + }, + "Macros": { + "type": "object", + "properties": { + "cluster": { + "type": "string" + }, + "replica": { + "$ref": "#/components/schemas/ServerId" + }, + "shard": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "cluster", + "replica", + "shard" + ] + }, + "RemoteServers": { + "type": "object", + "properties": { + "cluster": { + "type": "string" + }, + "replicas": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ServerNodeConfig" + } + }, + "secret": { + "type": "string" + } + }, + "required": [ + "cluster", + "replicas", + "secret" + ] + }, + "ReplicaConfig": { + "description": "Configuration for a ClickHouse replica server", + "type": "object", + "properties": { + "data_path": { + "description": "Directory for all files generated by ClickHouse itself", + "type": "string", + "format": "Utf8PathBuf" + }, + "http_port": { + "description": "Port for HTTP connections", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "interserver_http_port": { + "description": "Port for interserver HTTP connections", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "keepers": { + "description": "Contains settings that allow ClickHouse servers to interact with a Keeper cluster", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperConfigsForReplica" + } + ] + }, + "listen_host": { + "description": "Address the server is listening on", + "type": "string", + "format": "ipv6" + }, + "logger": { + "description": "Logging settings", + "allOf": [ + { + "$ref": "#/components/schemas/LogConfig" + } + ] + }, + "macros": { + "description": "Parameter substitutions for replicated tables", + "allOf": [ + { + "$ref": "#/components/schemas/Macros" + } + ] + }, + "remote_servers": { + "description": "Configuration of clusters used by the Distributed table engine and bythe cluster table function", + "allOf": [ + { + "$ref": "#/components/schemas/RemoteServers" + } + ] + }, + "tcp_port": { + "description": "Port for TCP connections", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "data_path", + "http_port", + "interserver_http_port", + "keepers", + "listen_host", + "logger", + "macros", + "remote_servers", + "tcp_port" + ] + }, + "ServerConfigurableSettings": { + "description": "The top most type for configuring clickhouse-servers via clickhouse-admin-server-api", + "type": "object", + "properties": { + "generation": { + "description": "A unique identifier for the configuration generation.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "settings": { + "description": "Configurable settings for a ClickHouse replica server node.", + "allOf": [ + { + "$ref": "#/components/schemas/ServerSettings" + } + ] + } + }, + "required": [ + "generation", + "settings" + ] + }, + "ServerId": { + "description": "A unique ID for a Clickhouse Server", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "ServerNodeConfig": { + "type": "object", + "properties": { + "host": { + "$ref": "#/components/schemas/ClickhouseHost" + }, + "port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "host", + "port" + ] + }, + "ServerSettings": { + "description": "Configurable settings for a ClickHouse replica server node.", + "type": "object", + "properties": { + "config_dir": { + "description": "Directory for the generated server configuration XML file", + "type": "string", + "format": "Utf8PathBuf" + }, + "datastore_path": { + "description": "Directory for all files generated by ClickHouse itself", + "type": "string", + "format": "Utf8PathBuf" + }, + "id": { + "description": "Unique ID of the server node", + "allOf": [ + { + "$ref": "#/components/schemas/ServerId" + } + ] + }, + "keepers": { + "description": "Addresses for each of the individual nodes in the Keeper cluster", + "type": "array", + "items": { + "$ref": "#/components/schemas/ClickhouseHost" + } + }, + "listen_addr": { + "description": "Address the server is listening on", + "type": "string", + "format": "ipv6" + }, + "remote_servers": { + "description": "Addresses for each of the individual replica servers", + "type": "array", + "items": { + "$ref": "#/components/schemas/ClickhouseHost" + } + } + }, + "required": [ + "config_dir", + "datastore_path", + "id", + "keepers", + "listen_addr", + "remote_servers" + ] + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +}