diff --git a/Cargo.lock b/Cargo.lock index dd2f30c2a0..0ddb16472b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1295,11 +1295,25 @@ dependencies = [ ] [[package]] -name = "clickhouse-admin-client" +name = "clickhouse-admin-keeper-client" +version = "0.1.0" +dependencies = [ + "chrono", + "clickhouse-admin-types", + "omicron-uuid-kinds", + "omicron-workspace-hack", + "progenitor", + "reqwest 0.12.7", + "schemars", + "serde", + "slog", +] + +[[package]] +name = "clickhouse-admin-server-client" version = "0.1.0" dependencies = [ "chrono", - "clickhouse-admin-api", "clickhouse-admin-types", "omicron-uuid-kinds", "omicron-workspace-hack", @@ -5603,7 +5617,8 @@ dependencies = [ "anyhow", "base64 0.22.1", "chrono", - "clickhouse-admin-client", + "clickhouse-admin-keeper-client", + "clickhouse-admin-server-client", "clickhouse-admin-types", "expectorate", "futures", @@ -5689,8 +5704,8 @@ dependencies = [ "async-bb8-diesel", "camino", "chrono", - "clickhouse-admin-api", - "clickhouse-admin-client", + "clickhouse-admin-keeper-client", + "clickhouse-admin-server-client", "clickhouse-admin-types", "cockroach-admin-client", "diesel", @@ -6543,7 +6558,8 @@ dependencies = [ "cancel-safe-futures", "chrono", "clap", - "clickhouse-admin-client", + "clickhouse-admin-keeper-client", + "clickhouse-admin-server-client", "cockroach-admin-client", "criterion", "crucible-agent-client", diff --git a/Cargo.toml b/Cargo.toml index ea0a46d49c..50660bf9bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,8 @@ members = [ "clickhouse-admin", "clickhouse-admin/api", "clients/bootstrap-agent-client", - "clients/clickhouse-admin-client", + "clients/clickhouse-admin-keeper-client", + "clients/clickhouse-admin-server-client", "clients/cockroach-admin-client", "clients/ddm-admin-client", "clients/dns-service-client", @@ -128,7 +129,8 @@ default-members = [ "clickhouse-admin/api", "clickhouse-admin/types", "clients/bootstrap-agent-client", - "clients/clickhouse-admin-client", + "clients/clickhouse-admin-keeper-client", + "clients/clickhouse-admin-server-client", "clients/cockroach-admin-client", "clients/ddm-admin-client", "clients/dns-service-client", @@ -313,7 +315,8 @@ chrono = { version = "0.4", features = [ "serde" ] } ciborium = "0.2.2" clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] } clickhouse-admin-api = { path = "clickhouse-admin/api" } -clickhouse-admin-client = { path = "clients/clickhouse-admin-client" } +clickhouse-admin-keeper-client = { path = "clients/clickhouse-admin-keeper-client" } +clickhouse-admin-server-client = { path = "clients/clickhouse-admin-server-client" } clickhouse-admin-types = { path = "clickhouse-admin/types" } clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "ceec762e6a87d2a22bf56792a3025e145caa095e" } cockroach-admin-api = { path = "cockroach-admin/api" } diff --git a/clickhouse-admin/api/src/lib.rs b/clickhouse-admin/api/src/lib.rs index bdd29f52b8..19cd2b3e8e 100644 --- a/clickhouse-admin/api/src/lib.rs +++ b/clickhouse-admin/api/src/lib.rs @@ -2,56 +2,40 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, KeeperConf, KeeperSettings, Lgif, - RaftConfig, ServerSettings, + ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig, + KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig, + ServerConfigurableSettings, }; use dropshot::{ HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody, }; -use omicron_common::api::external::Generation; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, Deserialize, JsonSchema)] -pub struct ServerConfigurableSettings { - /// A unique identifier for the configuration generation. - pub generation: Generation, - /// Configurable settings for a ClickHouse replica server node. - pub settings: ServerSettings, -} - -#[derive(Debug, Serialize, Deserialize, JsonSchema)] -pub struct KeeperConfigurableSettings { - /// A unique identifier for the configuration generation. - pub generation: Generation, - /// Configurable settings for a ClickHouse keeper node. - pub settings: KeeperSettings, -} +/// API interface for our clickhouse-admin-keeper server +/// +/// We separate the admin interface for the keeper and server APIs because they +/// are completely disjoint. We only run a clickhouse keeper *or* clickhouse +/// server in a given zone, and therefore each admin api is only useful in one +/// of the zones. Using separate APIs and clients prevents us from having to +/// mark a given endpoint `unimplemented` in the case of it not being usable +/// with one of the zone types. +/// +/// Nonetheless, the interfaces themselves are small and serve a similar +/// purpose. Therfore we combine them into the same crate. #[dropshot::api_description] -pub trait ClickhouseAdminApi { +pub trait ClickhouseAdminKeeperApi { type Context; - /// Generate a ClickHouse configuration file for a server node on a specified - /// directory and enable the SMF service. - #[endpoint { - method = PUT, - path = "/server/config-and-enable", - }] - async fn generate_server_config_and_enable( - rqctx: RequestContext, - body: TypedBody, - ) -> Result, HttpError>; - /// Generate a ClickHouse configuration file for a keeper node on a specified - /// directory and enable the SMF service. + /// directory and enable the SMF service if not currently enabled. + /// + /// Note that we cannot start the keeper service until there is an initial + /// configuration set via this endpoint. #[endpoint { method = PUT, - path = "/keeper/config-and-enable", + path = "/config", }] - async fn generate_keeper_config_and_enable( + async fn generate_config( rqctx: RequestContext, body: TypedBody, ) -> Result, HttpError>; @@ -95,3 +79,30 @@ pub trait ClickhouseAdminApi { rqctx: RequestContext, ) -> Result, HttpError>; } + +/// API interface for our clickhouse-admin-server server +/// +/// We separate the admin interface for the keeper and server APIs because they +/// are completely disjoint. We only run a clickhouse keeper *or* clickhouse +/// server in a given zone, and therefore each admin api is only useful in one +/// of the zones. Using separate APIs and clients prevents us from having to +/// mark a given endpoint `unimplemented` in the case of it not being usable +/// with one of the zone types. +/// +/// Nonetheless, the interfaces themselves are small and serve a similar +/// purpose. Therfore we combine them into the same crate. +#[dropshot::api_description] +pub trait ClickhouseAdminServerApi { + type Context; + + /// Generate a ClickHouse configuration file for a server node on a specified + /// directory and enable the SMF service. + #[endpoint { + method = PUT, + path = "/config" + }] + async fn generate_config( + rqctx: RequestContext, + body: TypedBody, + ) -> Result, HttpError>; +} diff --git a/clickhouse-admin/src/bin/clickhouse-admin.rs b/clickhouse-admin/src/bin/clickhouse-admin-keeper.rs similarity index 90% rename from clickhouse-admin/src/bin/clickhouse-admin.rs rename to clickhouse-admin/src/bin/clickhouse-admin-keeper.rs index 3391a3459a..4ec998920b 100644 --- a/clickhouse-admin/src/bin/clickhouse-admin.rs +++ b/clickhouse-admin/src/bin/clickhouse-admin-keeper.rs @@ -2,7 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Executable program to run the Omicron ClickHouse admin interface +//! Executable program to run the Omicron ClickHouse admin interface for +//! clickhouse keepers. use anyhow::anyhow; use camino::Utf8PathBuf; @@ -14,8 +15,8 @@ use std::net::{SocketAddr, SocketAddrV6}; #[derive(Debug, Parser)] #[clap( - name = "clickhouse-admin", - about = "Omicron ClickHouse cluster admin server" + name = "clickhouse-admin-keeper", + about = "Omicron ClickHouse cluster admin server for keepers" )] enum Args { /// Start the ClickHouse admin server @@ -57,7 +58,7 @@ async fn main_impl() -> Result<(), CmdError> { let clickhouse_cli = ClickhouseCli::new(binary_path, listen_address); - let server = omicron_clickhouse_admin::start_server( + let server = omicron_clickhouse_admin::start_keeper_admin_server( clickward, clickhouse_cli, config, diff --git a/clickhouse-admin/src/bin/clickhouse-admin-server.rs b/clickhouse-admin/src/bin/clickhouse-admin-server.rs new file mode 100644 index 0000000000..1258f75805 --- /dev/null +++ b/clickhouse-admin/src/bin/clickhouse-admin-server.rs @@ -0,0 +1,75 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Executable program to run the Omicron ClickHouse admin interface for +//! clickhouse servers. + +use anyhow::anyhow; +use camino::Utf8PathBuf; +use clap::Parser; +use omicron_clickhouse_admin::{ClickhouseCli, Clickward, Config}; +use omicron_common::cmd::fatal; +use omicron_common::cmd::CmdError; +use std::net::{SocketAddr, SocketAddrV6}; + +#[derive(Debug, Parser)] +#[clap( + name = "clickhouse-admin-server", + about = "Omicron ClickHouse cluster admin server for replica servers" +)] +enum Args { + /// Start the ClickHouse admin server + Run { + /// Address on which this server should run + #[clap(long, short = 'a', action)] + http_address: SocketAddrV6, + + /// Path to the server configuration file + #[clap(long, short, action)] + config: Utf8PathBuf, + + /// Address on which the clickhouse server or keeper is listening on + #[clap(long, short = 'l', action)] + listen_address: SocketAddrV6, + + /// Path to the clickhouse binary + #[clap(long, short, action)] + binary_path: Utf8PathBuf, + }, +} + +#[tokio::main] +async fn main() { + if let Err(err) = main_impl().await { + fatal(err); + } +} + +async fn main_impl() -> Result<(), CmdError> { + let args = Args::parse(); + + match args { + Args::Run { http_address, config, listen_address, binary_path } => { + let mut config = Config::from_file(&config) + .map_err(|err| CmdError::Failure(anyhow!(err)))?; + config.dropshot.bind_address = SocketAddr::V6(http_address); + let clickward = Clickward::new(); + let clickhouse_cli = + ClickhouseCli::new(binary_path, listen_address); + + let server = omicron_clickhouse_admin::start_server_admin_server( + clickward, + clickhouse_cli, + config, + ) + .await + .map_err(|err| CmdError::Failure(anyhow!(err)))?; + server.await.map_err(|err| { + CmdError::Failure(anyhow!( + "server failed after starting: {err}" + )) + }) + } + } +} diff --git a/clickhouse-admin/src/clickward.rs b/clickhouse-admin/src/clickward.rs index 5202b9b090..ca5d3df3de 100644 --- a/clickhouse-admin/src/clickward.rs +++ b/clickhouse-admin/src/clickward.rs @@ -2,8 +2,9 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; -use clickhouse_admin_types::{KeeperSettings, ServerSettings}; +use clickhouse_admin_types::{ + KeeperConfig, KeeperSettings, ReplicaConfig, ServerSettings, +}; use dropshot::HttpError; use slog_error_chain::{InlineErrorChain, SlogInlineError}; diff --git a/clickhouse-admin/src/http_entrypoints.rs b/clickhouse-admin/src/http_entrypoints.rs index 72cd105862..49138b9cc3 100644 --- a/clickhouse-admin/src/http_entrypoints.rs +++ b/clickhouse-admin/src/http_entrypoints.rs @@ -4,9 +4,10 @@ use crate::context::ServerContext; use clickhouse_admin_api::*; -use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, KeeperConf, Lgif, RaftConfig, + ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig, + KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig, + ServerConfigurableSettings, }; use dropshot::{ HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody, @@ -16,17 +17,22 @@ use std::sync::Arc; type ClickhouseApiDescription = dropshot::ApiDescription>; -pub fn api() -> ClickhouseApiDescription { - clickhouse_admin_api_mod::api_description::() +pub fn clickhouse_admin_server_api() -> ClickhouseApiDescription { + clickhouse_admin_server_api_mod::api_description::() .expect("registered entrypoints") } -enum ClickhouseAdminImpl {} +pub fn clickhouse_admin_keeper_api() -> ClickhouseApiDescription { + clickhouse_admin_keeper_api_mod::api_description::() + .expect("registered entrypoints") +} -impl ClickhouseAdminApi for ClickhouseAdminImpl { +enum ClickhouseAdminServerImpl {} + +impl ClickhouseAdminServerApi for ClickhouseAdminServerImpl { type Context = Arc; - async fn generate_server_config_and_enable( + async fn generate_config( rqctx: RequestContext, body: TypedBody, ) -> Result, HttpError> { @@ -41,8 +47,14 @@ impl ClickhouseAdminApi for ClickhouseAdminImpl { Ok(HttpResponseCreated(output)) } +} + +enum ClickhouseAdminKeeperImpl {} + +impl ClickhouseAdminKeeperApi for ClickhouseAdminKeeperImpl { + type Context = Arc; - async fn generate_keeper_config_and_enable( + async fn generate_config( rqctx: RequestContext, body: TypedBody, ) -> Result, HttpError> { diff --git a/clickhouse-admin/src/lib.rs b/clickhouse-admin/src/lib.rs index 511a32dd50..1697d24adc 100644 --- a/clickhouse-admin/src/lib.rs +++ b/clickhouse-admin/src/lib.rs @@ -33,8 +33,9 @@ pub enum StartError { pub type Server = dropshot::HttpServer>; -/// Start the dropshot server -pub async fn start_server( +/// Start the dropshot server for `clickhouse-admin-server` which +/// manages clickhouse replica servers. +pub async fn start_server_admin_server( clickward: Clickward, clickhouse_cli: ClickhouseCli, server_config: Config, @@ -42,7 +43,7 @@ pub async fn start_server( let (drain, registration) = slog_dtrace::with_drain( server_config .log - .to_logger("clickhouse-admin") + .to_logger("clickhouse-admin-server") .map_err(StartError::InitializeLogger)?, ); let log = slog::Logger::root(drain.fuse(), slog::o!(FileKv)); @@ -65,7 +66,49 @@ pub async fn start_server( ); let http_server_starter = dropshot::HttpServerStarter::new( &server_config.dropshot, - http_entrypoints::api(), + http_entrypoints::clickhouse_admin_server_api(), + Arc::new(context), + &log.new(slog::o!("component" => "dropshot")), + ) + .map_err(StartError::InitializeHttpServer)?; + + Ok(http_server_starter.start()) +} + +/// Start the dropshot server for `clickhouse-admin-server` which +/// manages clickhouse replica servers. +pub async fn start_keeper_admin_server( + clickward: Clickward, + clickhouse_cli: ClickhouseCli, + server_config: Config, +) -> Result { + let (drain, registration) = slog_dtrace::with_drain( + server_config + .log + .to_logger("clickhouse-admin-keeper") + .map_err(StartError::InitializeLogger)?, + ); + let log = slog::Logger::root(drain.fuse(), slog::o!(FileKv)); + match registration { + ProbeRegistration::Success => { + debug!(log, "registered DTrace probes"); + } + ProbeRegistration::Failed(err) => { + let err = StartError::RegisterDtraceProbes(err); + error!(log, "failed to register DTrace probes"; &err); + return Err(err); + } + } + + let context = ServerContext::new( + clickward, + clickhouse_cli + .with_log(log.new(slog::o!("component" => "ClickhouseCli"))), + log.new(slog::o!("component" => "ServerContext")), + ); + let http_server_starter = dropshot::HttpServerStarter::new( + &server_config.dropshot, + http_entrypoints::clickhouse_admin_keeper_api(), Arc::new(context), &log.new(slog::o!("component" => "dropshot")), ) diff --git a/clickhouse-admin/tests/integration_test.rs b/clickhouse-admin/tests/integration_test.rs index fd706722a3..eb26bec668 100644 --- a/clickhouse-admin/tests/integration_test.rs +++ b/clickhouse-admin/tests/integration_test.rs @@ -4,10 +4,9 @@ use anyhow::Context; use camino::Utf8PathBuf; -use clickhouse_admin_types::config::ClickhouseHost; use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, KeeperId, KeeperServerInfo, - KeeperServerType, RaftConfig, + ClickhouseHost, ClickhouseKeeperClusterMembership, KeeperId, + KeeperServerInfo, KeeperServerType, RaftConfig, }; use clickward::{BasePorts, Deployment, DeploymentConfig}; use dropshot::test_util::log_prefix_for_test; diff --git a/clickhouse-admin/types/src/config.rs b/clickhouse-admin/types/src/config.rs index 3cbf4c7fe2..20aa3d71f8 100644 --- a/clickhouse-admin/types/src/config.rs +++ b/clickhouse-admin/types/src/config.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::{KeeperId, ServerId, OXIMETER_CLUSTER}; +use crate::{path_schema, KeeperId, ServerId, OXIMETER_CLUSTER}; use anyhow::{bail, Error}; use camino::Utf8PathBuf; use omicron_common::address::{ @@ -10,23 +10,11 @@ use omicron_common::address::{ CLICKHOUSE_KEEPER_RAFT_PORT, CLICKHOUSE_KEEPER_TCP_PORT, CLICKHOUSE_TCP_PORT, }; -use schemars::{ - gen::SchemaGenerator, - schema::{Schema, SchemaObject}, - JsonSchema, -}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::net::{Ipv4Addr, Ipv6Addr}; use std::{fmt::Display, str::FromStr}; -// Used for schemars to be able to be used with camino: -// See https://github.com/camino-rs/camino/issues/91#issuecomment-2027908513 -pub fn path_schema(gen: &mut SchemaGenerator) -> Schema { - let mut schema: SchemaObject = ::json_schema(gen).into(); - schema.format = Some("Utf8PathBuf".to_owned()); - schema.into() -} - /// Configuration for a ClickHouse replica server #[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] pub struct ReplicaConfig { diff --git a/clickhouse-admin/types/src/lib.rs b/clickhouse-admin/types/src/lib.rs index 83d56be1a1..021b9db356 100644 --- a/clickhouse-admin/types/src/lib.rs +++ b/clickhouse-admin/types/src/lib.rs @@ -7,7 +7,12 @@ use atomicwrites::AtomicFile; use camino::Utf8PathBuf; use derive_more::{Add, AddAssign, Display, From}; use itertools::Itertools; -use schemars::JsonSchema; +use omicron_common::api::external::Generation; +use schemars::{ + gen::SchemaGenerator, + schema::{Schema, SchemaObject}, + JsonSchema, +}; use serde::{Deserialize, Serialize}; use slog::{info, Logger}; use std::collections::BTreeSet; @@ -16,8 +21,20 @@ use std::io::{ErrorKind, Write}; use std::net::Ipv6Addr; use std::str::FromStr; -pub mod config; -use config::*; +mod config; +pub use config::{ + ClickhouseHost, KeeperConfig, KeeperConfigsForReplica, KeeperNodeConfig, + LogConfig, LogLevel, Macros, NodeType, RaftServerConfig, + RaftServerSettings, RaftServers, ReplicaConfig, ServerNodeConfig, +}; + +// Used for schemars to be able to be used with camino: +// See https://github.com/camino-rs/camino/issues/91#issuecomment-2027908513 +pub fn path_schema(gen: &mut SchemaGenerator) -> Schema { + let mut schema: SchemaObject = ::json_schema(gen).into(); + schema.format = Some("Utf8PathBuf".to_owned()); + schema.into() +} pub const OXIMETER_CLUSTER: &str = "oximeter_cluster"; @@ -59,6 +76,26 @@ pub struct KeeperId(pub u64); )] pub struct ServerId(pub u64); +/// The top most type for configuring clickhouse-servers via +/// clickhouse-admin-server-api +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct ServerConfigurableSettings { + /// A unique identifier for the configuration generation. + pub generation: Generation, + /// Configurable settings for a ClickHouse replica server node. + pub settings: ServerSettings, +} + +/// The top most type for configuring clickhouse-servers via +/// clickhouse-admin-keeper-api +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct KeeperConfigurableSettings { + /// A unique identifier for the configuration generation. + pub generation: Generation, + /// Configurable settings for a ClickHouse keeper node. + pub settings: KeeperSettings, +} + /// Configurable settings for a ClickHouse replica server node. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "snake_case")] diff --git a/clients/clickhouse-admin-client/Cargo.toml b/clients/clickhouse-admin-keeper-client/Cargo.toml similarity index 85% rename from clients/clickhouse-admin-client/Cargo.toml rename to clients/clickhouse-admin-keeper-client/Cargo.toml index c1e05c4f71..1b8839ae64 100644 --- a/clients/clickhouse-admin-client/Cargo.toml +++ b/clients/clickhouse-admin-keeper-client/Cargo.toml @@ -1,12 +1,11 @@ [package] -name = "clickhouse-admin-client" +name = "clickhouse-admin-keeper-client" version = "0.1.0" edition = "2021" [dependencies] chrono.workspace = true clickhouse-admin-types.workspace = true -clickhouse-admin-api.workspace = true omicron-uuid-kinds.workspace = true progenitor.workspace = true reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] } diff --git a/clients/clickhouse-admin-client/src/lib.rs b/clients/clickhouse-admin-keeper-client/src/lib.rs similarity index 74% rename from clients/clickhouse-admin-client/src/lib.rs rename to clients/clickhouse-admin-keeper-client/src/lib.rs index 067c3e6939..502b8ccbdd 100644 --- a/clients/clickhouse-admin-client/src/lib.rs +++ b/clients/clickhouse-admin-keeper-client/src/lib.rs @@ -2,11 +2,11 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Interface for making API requests to a clickhouse-admin server running in -//! an omicron zone. +//! Interface for making API requests to a clickhouse-admin-keeper server +//! running in an omicron zone. progenitor::generate_api!( - spec = "../../openapi/clickhouse-admin.json", + spec = "../../openapi/clickhouse-admin-keeper.json", inner_type = slog::Logger, pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { slog::debug!(log, "client request"; @@ -21,8 +21,7 @@ progenitor::generate_api!( derives = [schemars::JsonSchema], replace = { TypedUuidForOmicronZoneKind = omicron_uuid_kinds::OmicronZoneUuid, - KeeperConfigurableSettings = clickhouse_admin_api::KeeperConfigurableSettings, - ServerConfigurableSettings = clickhouse_admin_api::ServerConfigurableSettings, + KeeperConfigurableSettings = clickhouse_admin_types::KeeperConfigurableSettings, ClickhouseKeeperClusterMembership = clickhouse_admin_types::ClickhouseKeeperClusterMembership, KeeperId = clickhouse_admin_types::KeeperId } diff --git a/clients/clickhouse-admin-server-client/Cargo.toml b/clients/clickhouse-admin-server-client/Cargo.toml new file mode 100644 index 0000000000..61142f8a7b --- /dev/null +++ b/clients/clickhouse-admin-server-client/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "clickhouse-admin-server-client" +version = "0.1.0" +edition = "2021" + +[dependencies] +chrono.workspace = true +clickhouse-admin-types.workspace = true +omicron-uuid-kinds.workspace = true +progenitor.workspace = true +reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] } +schemars.workspace = true +serde.workspace = true +slog.workspace = true +omicron-workspace-hack.workspace = true + +[lints] +workspace = true diff --git a/clients/clickhouse-admin-server-client/src/lib.rs b/clients/clickhouse-admin-server-client/src/lib.rs new file mode 100644 index 0000000000..3092160d65 --- /dev/null +++ b/clients/clickhouse-admin-server-client/src/lib.rs @@ -0,0 +1,26 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Interface for making API requests to a clickhouse-admin-server server +//! running in an omicron zone. + +progenitor::generate_api!( + spec = "../../openapi/clickhouse-admin-server.json", + inner_type = slog::Logger, + pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + }), + post_hook = (|log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + }), + derives = [schemars::JsonSchema], + replace = { + TypedUuidForOmicronZoneKind = omicron_uuid_kinds::OmicronZoneUuid, + ServerConfigurableSettings = clickhouse_admin_types::ServerConfigurableSettings, + } +); diff --git a/dev-tools/ls-apis/api-manifest.toml b/dev-tools/ls-apis/api-manifest.toml index b9ca4bc5df..f91aba74aa 100644 --- a/dev-tools/ls-apis/api-manifest.toml +++ b/dev-tools/ls-apis/api-manifest.toml @@ -139,11 +139,20 @@ label = "Bootstrap Agent" server_package_name = "bootstrap-agent-api" [[apis]] -client_package_name = "clickhouse-admin-client" -label = "Clickhouse Cluster Admin" +client_package_name = "clickhouse-admin-keeper-client" +label = "Clickhouse Cluster Admin for Keepers" server_package_name = "clickhouse-admin-api" notes = """ -This is the server running inside multi-node Clickhouse zones that's \ +This is the server running inside multi-node Clickhouse keeper zones that's \ +responsible for local configuration and monitoring. +""" + +[[apis]] +client_package_name = "clickhouse-admin-server-client" +label = "Clickhouse Cluster Admin for Servers" +server_package_name = "clickhouse-admin-api" +notes = """ +This is the server running inside multi-node Clickhouse server zones that's \ responsible for local configuration and monitoring. """ diff --git a/dev-tools/ls-apis/src/system_apis.rs b/dev-tools/ls-apis/src/system_apis.rs index ad61f31f33..60b9b9246a 100644 --- a/dev-tools/ls-apis/src/system_apis.rs +++ b/dev-tools/ls-apis/src/system_apis.rs @@ -444,22 +444,6 @@ impl<'a> ServerComponentsTracker<'a> { return; } - // TODO Work around omicron#6829. - // Through the dependency tree, omicron-nexus appears to export the - // clickhouse-admin-api, but it doesn't really. - // This is just like the Crucible Pantry one above in that we can't - // build our data model unless we ignore it so we have to hardcode this - // here rather than use "dependency_filter_rules". - if **server_pkgname == "omicron-nexus" - && *api.client_package_name == "clickhouse-admin-client" - { - eprintln!( - "note: ignoring Cargo dependency from crucible-pantry -> \ - ... -> crucible-control-client", - ); - return; - } - if let Some((previous, _)) = self.api_producers.insert( api.client_package_name.clone(), (server_pkgname.clone(), dep_path.clone()), diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out index 62da29db42..a902484a75 100644 --- a/dev-tools/ls-apis/tests/api_dependencies.out +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -2,7 +2,10 @@ Bootstrap Agent (client: bootstrap-agent-client) consumed by: omicron-sled-agent (omicron/sled-agent) consumed by: wicketd (omicron/wicketd) -Clickhouse Cluster Admin (client: clickhouse-admin-client) +Clickhouse Cluster Admin for Keepers (client: clickhouse-admin-keeper-client) + consumed by: omicron-nexus (omicron/nexus) + +Clickhouse Cluster Admin for Servers (client: clickhouse-admin-server-client) consumed by: omicron-nexus (omicron/nexus) CockroachDB Cluster Admin (client: cockroach-admin-client) diff --git a/dev-tools/openapi-manager/src/spec.rs b/dev-tools/openapi-manager/src/spec.rs index 7d734218fc..90e5a9b733 100644 --- a/dev-tools/openapi-manager/src/spec.rs +++ b/dev-tools/openapi-manager/src/spec.rs @@ -27,14 +27,25 @@ pub fn all_apis() -> Vec { extra_validation: None, }, ApiSpec { - title: "ClickHouse Cluster Admin API", + title: "ClickHouse Cluster Admin Keeper API", version: "0.0.1", description: "API for interacting with the Oxide \ - control plane's ClickHouse cluster", + control plane's ClickHouse cluster keepers", boundary: ApiBoundary::Internal, api_description: - clickhouse_admin_api::clickhouse_admin_api_mod::stub_api_description, - filename: "clickhouse-admin.json", + clickhouse_admin_api::clickhouse_admin_keeper_api_mod::stub_api_description, + filename: "clickhouse-admin-keeper.json", + extra_validation: None, + }, + ApiSpec { + title: "ClickHouse Cluster Admin Server API", + version: "0.0.1", + description: "API for interacting with the Oxide \ + control plane's ClickHouse cluster replica servers", + boundary: ApiBoundary::Internal, + api_description: + clickhouse_admin_api::clickhouse_admin_server_api_mod::stub_api_description, + filename: "clickhouse-admin-server.json", extra_validation: None, }, ApiSpec { diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 8e54baf2d3..d430009360 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -22,7 +22,8 @@ camino.workspace = true camino-tempfile.workspace = true clap.workspace = true chrono.workspace = true -clickhouse-admin-client.workspace = true +clickhouse-admin-keeper-client.workspace = true +clickhouse-admin-server-client.workspace = true cockroach-admin-client.workspace = true crucible-agent-client.workspace = true crucible-pantry-client.workspace = true diff --git a/nexus/inventory/Cargo.toml b/nexus/inventory/Cargo.toml index f664523950..b5ca91283e 100644 --- a/nexus/inventory/Cargo.toml +++ b/nexus/inventory/Cargo.toml @@ -11,7 +11,8 @@ workspace = true anyhow.workspace = true base64.workspace = true chrono.workspace = true -clickhouse-admin-client.workspace = true +clickhouse-admin-keeper-client.workspace = true +clickhouse-admin-server-client.workspace = true clickhouse-admin-types.workspace = true futures.workspace = true gateway-client.workspace = true diff --git a/nexus/inventory/src/collector.rs b/nexus/inventory/src/collector.rs index dda000cd8c..eebc692019 100644 --- a/nexus/inventory/src/collector.rs +++ b/nexus/inventory/src/collector.rs @@ -28,7 +28,7 @@ const SLED_AGENT_TIMEOUT: Duration = Duration::from_secs(60); pub struct Collector<'a> { log: slog::Logger, mgs_clients: Vec, - keeper_admin_clients: Vec, + keeper_admin_clients: Vec, sled_agent_lister: &'a (dyn SledAgentEnumerator + Send + Sync), in_progress: CollectionBuilder, } @@ -37,7 +37,7 @@ impl<'a> Collector<'a> { pub fn new( creator: &str, mgs_clients: Vec, - keeper_admin_clients: Vec, + keeper_admin_clients: Vec, sled_agent_lister: &'a (dyn SledAgentEnumerator + Send + Sync), log: slog::Logger, ) -> Self { @@ -384,7 +384,7 @@ impl<'a> Collector<'a> { /// Collect inventory about one keeper from one `ClickhouseAdminKeeper` async fn collect_one_keeper( - client: &clickhouse_admin_client::Client, + client: &clickhouse_admin_keeper_client::Client, log: &slog::Logger, in_progress: &mut CollectionBuilder, ) { diff --git a/nexus/reconfigurator/execution/Cargo.toml b/nexus/reconfigurator/execution/Cargo.toml index e3ccddf846..21d861ef51 100644 --- a/nexus/reconfigurator/execution/Cargo.toml +++ b/nexus/reconfigurator/execution/Cargo.toml @@ -12,8 +12,8 @@ omicron-rpaths.workspace = true [dependencies] anyhow.workspace = true camino.workspace = true -clickhouse-admin-api.workspace = true -clickhouse-admin-client.workspace = true +clickhouse-admin-keeper-client.workspace = true +clickhouse-admin-server-client.workspace = true clickhouse-admin-types.workspace = true cockroach-admin-client.workspace = true chrono.workspace = true diff --git a/nexus/reconfigurator/execution/src/clickhouse.rs b/nexus/reconfigurator/execution/src/clickhouse.rs index 926a608e2c..e623952ed9 100644 --- a/nexus/reconfigurator/execution/src/clickhouse.rs +++ b/nexus/reconfigurator/execution/src/clickhouse.rs @@ -7,12 +7,13 @@ use anyhow::anyhow; use camino::Utf8PathBuf; -use clickhouse_admin_api::KeeperConfigurableSettings; -use clickhouse_admin_api::ServerConfigurableSettings; -use clickhouse_admin_client::Client; -use clickhouse_admin_types::config::ClickhouseHost; -use clickhouse_admin_types::config::RaftServerSettings; +use clickhouse_admin_keeper_client::Client as ClickhouseKeeperClient; +use clickhouse_admin_server_client::Client as ClickhouseServerClient; +use clickhouse_admin_types::ClickhouseHost; +use clickhouse_admin_types::KeeperConfigurableSettings; use clickhouse_admin_types::KeeperSettings; +use clickhouse_admin_types::RaftServerSettings; +use clickhouse_admin_types::ServerConfigurableSettings; use clickhouse_admin_types::ServerSettings; use futures::future::Either; use futures::stream::FuturesUnordered; @@ -91,23 +92,19 @@ pub(crate) async fn deploy_nodes( let admin_url = format!("http://{admin_addr}"); let log = log.new(slog::o!("admin_url" => admin_url.clone())); futs.push(Either::Left(async move { - let client = Client::new(&admin_url, log.clone()); - client - .generate_keeper_config_and_enable(&config) - .await - .map(|_| ()) - .map_err(|e| { - anyhow!( - concat!( - "failed to send config for clickhouse keeper ", - "with id {} to clickhouse-admin; admin_url = {}", - "error = {}" - ), - config.settings.id, - admin_url, - e - ) - }) + let client = ClickhouseKeeperClient::new(&admin_url, log.clone()); + client.generate_config(&config).await.map(|_| ()).map_err(|e| { + anyhow!( + concat!( + "failed to send config for clickhouse keeper ", + "with id {} to clickhouse-admin-keeper; admin_url = {}", + "error = {}" + ), + config.settings.id, + admin_url, + e + ) + }) })); } for config in server_configs { @@ -120,23 +117,19 @@ pub(crate) async fn deploy_nodes( let admin_url = format!("http://{admin_addr}"); let log = opctx.log.new(slog::o!("admin_url" => admin_url.clone())); futs.push(Either::Right(async move { - let client = Client::new(&admin_url, log.clone()); - client - .generate_server_config_and_enable(&config) - .await - .map(|_| ()) - .map_err(|e| { - anyhow!( - concat!( - "failed to send config for clickhouse server ", - "with id {} to clickhouse-admin; admin_url = {}", - "error = {}" - ), - config.settings.id, - admin_url, - e - ) - }) + let client = ClickhouseServerClient::new(&admin_url, log.clone()); + client.generate_config(&config).await.map(|_| ()).map_err(|e| { + anyhow!( + concat!( + "failed to send config for clickhouse server ", + "with id {} to clickhouse-admin-server; admin_url = {}", + "error = {}" + ), + config.settings.id, + admin_url, + e + ) + }) })); } @@ -285,7 +278,7 @@ fn keeper_configs( #[cfg(test)] mod test { use super::*; - use clickhouse_admin_types::config::ClickhouseHost; + use clickhouse_admin_types::ClickhouseHost; use clickhouse_admin_types::KeeperId; use clickhouse_admin_types::ServerId; use nexus_sled_agent_shared::inventory::OmicronZoneDataset; diff --git a/nexus/src/app/background/tasks/inventory_collection.rs b/nexus/src/app/background/tasks/inventory_collection.rs index e0b2f25365..c4271c58d8 100644 --- a/nexus/src/app/background/tasks/inventory_collection.rs +++ b/nexus/src/app/background/tasks/inventory_collection.rs @@ -142,7 +142,7 @@ async fn inventory_activate( let url = format!("http://{}", sockaddr); let log = opctx.log.new(o!("clickhouse_admin_keeper_url" => url.clone())); - clickhouse_admin_client::Client::new(&url, log) + clickhouse_admin_keeper_client::Client::new(&url, log) }) .collect::>(); diff --git a/openapi/clickhouse-admin.json b/openapi/clickhouse-admin-keeper.json similarity index 77% rename from openapi/clickhouse-admin.json rename to openapi/clickhouse-admin-keeper.json index 5f5b4932ed..3c48a082b7 100644 --- a/openapi/clickhouse-admin.json +++ b/openapi/clickhouse-admin-keeper.json @@ -1,8 +1,8 @@ { "openapi": "3.0.3", "info": { - "title": "ClickHouse Cluster Admin API", - "description": "API for interacting with the Oxide control plane's ClickHouse cluster", + "title": "ClickHouse Cluster Admin Keeper API", + "description": "API for interacting with the Oxide control plane's ClickHouse cluster keepers", "contact": { "url": "https://oxide.computer", "email": "api@oxide.computer" @@ -10,17 +10,28 @@ "version": "0.0.1" }, "paths": { - "/keeper/cluster-membership": { - "get": { - "summary": "Retrieve cluster membership information from a keeper node.", - "operationId": "keeper_cluster_membership", + "/config": { + "put": { + "summary": "Generate a ClickHouse configuration file for a keeper node on a specified", + "description": "directory and enable the SMF service if not currently enabled.\n\nNote that we cannot start the keeper service until there is an initial configuration set via this endpoint.", + "operationId": "generate_config", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/KeeperConfigurableSettings" + } + } + }, + "required": true + }, "responses": { - "200": { - "description": "successful operation", + "201": { + "description": "successful creation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ClickhouseKeeperClusterMembership" + "$ref": "#/components/schemas/KeeperConfig" } } } @@ -34,17 +45,17 @@ } } }, - "/keeper/conf": { + "/keeper/cluster-membership": { "get": { - "summary": "Retrieve configuration information from a keeper node.", - "operationId": "keeper_conf", + "summary": "Retrieve cluster membership information from a keeper node.", + "operationId": "keeper_cluster_membership", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/KeeperConf" + "$ref": "#/components/schemas/ClickhouseKeeperClusterMembership" } } } @@ -58,28 +69,17 @@ } } }, - "/keeper/config-and-enable": { - "put": { - "summary": "Generate a ClickHouse configuration file for a keeper node on a specified", - "description": "directory and enable the SMF service.", - "operationId": "generate_keeper_config_and_enable", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/KeeperConfigurableSettings" - } - } - }, - "required": true - }, + "/keeper/conf": { + "get": { + "summary": "Retrieve configuration information from a keeper node.", + "operationId": "keeper_conf", "responses": { - "201": { - "description": "successful creation", + "200": { + "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/KeeperConfig" + "$ref": "#/components/schemas/KeeperConf" } } } @@ -142,41 +142,6 @@ } } } - }, - "/server/config-and-enable": { - "put": { - "summary": "Generate a ClickHouse configuration file for a server node on a specified", - "description": "directory and enable the SMF service.", - "operationId": "generate_server_config_and_enable", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ServerConfigurableSettings" - } - } - }, - "required": true - }, - "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ReplicaConfig" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } } }, "components": { @@ -593,21 +558,8 @@ "tcp_port" ] }, - "KeeperConfigsForReplica": { - "type": "object", - "properties": { - "nodes": { - "type": "array", - "items": { - "$ref": "#/components/schemas/KeeperNodeConfig" - } - } - }, - "required": [ - "nodes" - ] - }, "KeeperConfigurableSettings": { + "description": "The top most type for configuring clickhouse-servers via clickhouse-admin-keeper-api", "type": "object", "properties": { "generation": { @@ -661,23 +613,6 @@ "format": "uint64", "minimum": 0 }, - "KeeperNodeConfig": { - "type": "object", - "properties": { - "host": { - "$ref": "#/components/schemas/ClickhouseHost" - }, - "port": { - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "host", - "port" - ] - }, "KeeperServerInfo": { "type": "object", "properties": { @@ -880,27 +815,6 @@ "debug" ] }, - "Macros": { - "type": "object", - "properties": { - "cluster": { - "type": "string" - }, - "replica": { - "$ref": "#/components/schemas/ServerId" - }, - "shard": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - "required": [ - "cluster", - "replica", - "shard" - ] - }, "RaftConfig": { "description": "Keeper raft configuration information", "type": "object", @@ -966,204 +880,6 @@ "required": [ "servers" ] - }, - "RemoteServers": { - "type": "object", - "properties": { - "cluster": { - "type": "string" - }, - "replicas": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ServerNodeConfig" - } - }, - "secret": { - "type": "string" - } - }, - "required": [ - "cluster", - "replicas", - "secret" - ] - }, - "ReplicaConfig": { - "description": "Configuration for a ClickHouse replica server", - "type": "object", - "properties": { - "data_path": { - "description": "Directory for all files generated by ClickHouse itself", - "type": "string", - "format": "Utf8PathBuf" - }, - "http_port": { - "description": "Port for HTTP connections", - "type": "integer", - "format": "uint16", - "minimum": 0 - }, - "interserver_http_port": { - "description": "Port for interserver HTTP connections", - "type": "integer", - "format": "uint16", - "minimum": 0 - }, - "keepers": { - "description": "Contains settings that allow ClickHouse servers to interact with a Keeper cluster", - "allOf": [ - { - "$ref": "#/components/schemas/KeeperConfigsForReplica" - } - ] - }, - "listen_host": { - "description": "Address the server is listening on", - "type": "string", - "format": "ipv6" - }, - "logger": { - "description": "Logging settings", - "allOf": [ - { - "$ref": "#/components/schemas/LogConfig" - } - ] - }, - "macros": { - "description": "Parameter substitutions for replicated tables", - "allOf": [ - { - "$ref": "#/components/schemas/Macros" - } - ] - }, - "remote_servers": { - "description": "Configuration of clusters used by the Distributed table engine and bythe cluster table function", - "allOf": [ - { - "$ref": "#/components/schemas/RemoteServers" - } - ] - }, - "tcp_port": { - "description": "Port for TCP connections", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "data_path", - "http_port", - "interserver_http_port", - "keepers", - "listen_host", - "logger", - "macros", - "remote_servers", - "tcp_port" - ] - }, - "ServerConfigurableSettings": { - "type": "object", - "properties": { - "generation": { - "description": "A unique identifier for the configuration generation.", - "allOf": [ - { - "$ref": "#/components/schemas/Generation" - } - ] - }, - "settings": { - "description": "Configurable settings for a ClickHouse replica server node.", - "allOf": [ - { - "$ref": "#/components/schemas/ServerSettings" - } - ] - } - }, - "required": [ - "generation", - "settings" - ] - }, - "ServerId": { - "description": "A unique ID for a Clickhouse Server", - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - "ServerNodeConfig": { - "type": "object", - "properties": { - "host": { - "$ref": "#/components/schemas/ClickhouseHost" - }, - "port": { - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "host", - "port" - ] - }, - "ServerSettings": { - "description": "Configurable settings for a ClickHouse replica server node.", - "type": "object", - "properties": { - "config_dir": { - "description": "Directory for the generated server configuration XML file", - "type": "string", - "format": "Utf8PathBuf" - }, - "datastore_path": { - "description": "Directory for all files generated by ClickHouse itself", - "type": "string", - "format": "Utf8PathBuf" - }, - "id": { - "description": "Unique ID of the server node", - "allOf": [ - { - "$ref": "#/components/schemas/ServerId" - } - ] - }, - "keepers": { - "description": "Addresses for each of the individual nodes in the Keeper cluster", - "type": "array", - "items": { - "$ref": "#/components/schemas/ClickhouseHost" - } - }, - "listen_addr": { - "description": "Address the server is listening on", - "type": "string", - "format": "ipv6" - }, - "remote_servers": { - "description": "Addresses for each of the individual replica servers", - "type": "array", - "items": { - "$ref": "#/components/schemas/ClickhouseHost" - } - } - }, - "required": [ - "config_dir", - "datastore_path", - "id", - "keepers", - "listen_addr", - "remote_servers" - ] } }, "responses": { diff --git a/openapi/clickhouse-admin-server.json b/openapi/clickhouse-admin-server.json new file mode 100644 index 0000000000..9dc2506eb1 --- /dev/null +++ b/openapi/clickhouse-admin-server.json @@ -0,0 +1,423 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "ClickHouse Cluster Admin Server API", + "description": "API for interacting with the Oxide control plane's ClickHouse cluster replica servers", + "contact": { + "url": "https://oxide.computer", + "email": "api@oxide.computer" + }, + "version": "0.0.1" + }, + "paths": { + "/config": { + "put": { + "summary": "Generate a ClickHouse configuration file for a server node on a specified", + "description": "directory and enable the SMF service.", + "operationId": "generate_config", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServerConfigurableSettings" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReplicaConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "ClickhouseHost": { + "oneOf": [ + { + "type": "object", + "properties": { + "ipv6": { + "type": "string", + "format": "ipv6" + } + }, + "required": [ + "ipv6" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ipv4": { + "type": "string", + "format": "ipv4" + } + }, + "required": [ + "ipv4" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "domain_name": { + "type": "string" + } + }, + "required": [ + "domain_name" + ], + "additionalProperties": false + } + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "Generation": { + "description": "Generation numbers stored in the database, used for optimistic concurrency control", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "KeeperConfigsForReplica": { + "type": "object", + "properties": { + "nodes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/KeeperNodeConfig" + } + } + }, + "required": [ + "nodes" + ] + }, + "KeeperNodeConfig": { + "type": "object", + "properties": { + "host": { + "$ref": "#/components/schemas/ClickhouseHost" + }, + "port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "host", + "port" + ] + }, + "LogConfig": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "errorlog": { + "type": "string", + "format": "Utf8PathBuf" + }, + "level": { + "$ref": "#/components/schemas/LogLevel" + }, + "log": { + "type": "string", + "format": "Utf8PathBuf" + }, + "size": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "count", + "errorlog", + "level", + "log", + "size" + ] + }, + "LogLevel": { + "type": "string", + "enum": [ + "trace", + "debug" + ] + }, + "Macros": { + "type": "object", + "properties": { + "cluster": { + "type": "string" + }, + "replica": { + "$ref": "#/components/schemas/ServerId" + }, + "shard": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "cluster", + "replica", + "shard" + ] + }, + "RemoteServers": { + "type": "object", + "properties": { + "cluster": { + "type": "string" + }, + "replicas": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ServerNodeConfig" + } + }, + "secret": { + "type": "string" + } + }, + "required": [ + "cluster", + "replicas", + "secret" + ] + }, + "ReplicaConfig": { + "description": "Configuration for a ClickHouse replica server", + "type": "object", + "properties": { + "data_path": { + "description": "Directory for all files generated by ClickHouse itself", + "type": "string", + "format": "Utf8PathBuf" + }, + "http_port": { + "description": "Port for HTTP connections", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "interserver_http_port": { + "description": "Port for interserver HTTP connections", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "keepers": { + "description": "Contains settings that allow ClickHouse servers to interact with a Keeper cluster", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperConfigsForReplica" + } + ] + }, + "listen_host": { + "description": "Address the server is listening on", + "type": "string", + "format": "ipv6" + }, + "logger": { + "description": "Logging settings", + "allOf": [ + { + "$ref": "#/components/schemas/LogConfig" + } + ] + }, + "macros": { + "description": "Parameter substitutions for replicated tables", + "allOf": [ + { + "$ref": "#/components/schemas/Macros" + } + ] + }, + "remote_servers": { + "description": "Configuration of clusters used by the Distributed table engine and bythe cluster table function", + "allOf": [ + { + "$ref": "#/components/schemas/RemoteServers" + } + ] + }, + "tcp_port": { + "description": "Port for TCP connections", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "data_path", + "http_port", + "interserver_http_port", + "keepers", + "listen_host", + "logger", + "macros", + "remote_servers", + "tcp_port" + ] + }, + "ServerConfigurableSettings": { + "description": "The top most type for configuring clickhouse-servers via clickhouse-admin-server-api", + "type": "object", + "properties": { + "generation": { + "description": "A unique identifier for the configuration generation.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "settings": { + "description": "Configurable settings for a ClickHouse replica server node.", + "allOf": [ + { + "$ref": "#/components/schemas/ServerSettings" + } + ] + } + }, + "required": [ + "generation", + "settings" + ] + }, + "ServerId": { + "description": "A unique ID for a Clickhouse Server", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "ServerNodeConfig": { + "type": "object", + "properties": { + "host": { + "$ref": "#/components/schemas/ClickhouseHost" + }, + "port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "host", + "port" + ] + }, + "ServerSettings": { + "description": "Configurable settings for a ClickHouse replica server node.", + "type": "object", + "properties": { + "config_dir": { + "description": "Directory for the generated server configuration XML file", + "type": "string", + "format": "Utf8PathBuf" + }, + "datastore_path": { + "description": "Directory for all files generated by ClickHouse itself", + "type": "string", + "format": "Utf8PathBuf" + }, + "id": { + "description": "Unique ID of the server node", + "allOf": [ + { + "$ref": "#/components/schemas/ServerId" + } + ] + }, + "keepers": { + "description": "Addresses for each of the individual nodes in the Keeper cluster", + "type": "array", + "items": { + "$ref": "#/components/schemas/ClickhouseHost" + } + }, + "listen_addr": { + "description": "Address the server is listening on", + "type": "string", + "format": "ipv6" + }, + "remote_servers": { + "description": "Addresses for each of the individual replica servers", + "type": "array", + "items": { + "$ref": "#/components/schemas/ClickhouseHost" + } + } + }, + "required": [ + "config_dir", + "datastore_path", + "id", + "keepers", + "listen_addr", + "remote_servers" + ] + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/package-manifest.toml b/package-manifest.toml index 1a93e0d165..e724486e77 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -243,13 +243,14 @@ output.intermediate_only = true setup_hint = "Run `cargo xtask download clickhouse` to download the necessary binaries" [package.omicron-clickhouse-admin] -service_name = "clickhouse-admin" +service_name = "omicron-clickhouse-admin" only_for_targets.image = "standard" source.type = "local" -source.rust.binary_names = ["clickhouse-admin"] +source.rust.binary_names = ["clickhouse-admin-keeper", "clickhouse-admin-server"] source.rust.release = true source.paths = [ - { from = "smf/clickhouse-admin", to = "/var/svc/manifest/site/clickhouse-admin" }, + { from = "smf/clickhouse-admin-keeper", to = "/var/svc/manifest/site/clickhouse-admin-keeper" }, + { from = "smf/clickhouse-admin-server", to = "/var/svc/manifest/site/clickhouse-admin-server" }, ] output.type = "zone" output.intermediate_only = true diff --git a/smf/clickhouse-admin/config.toml b/smf/clickhouse-admin-keeper/config.toml similarity index 100% rename from smf/clickhouse-admin/config.toml rename to smf/clickhouse-admin-keeper/config.toml diff --git a/smf/clickhouse-admin/manifest.xml b/smf/clickhouse-admin-keeper/manifest.xml similarity index 73% rename from smf/clickhouse-admin/manifest.xml rename to smf/clickhouse-admin-keeper/manifest.xml index 379c738f5d..46b0a33e3c 100644 --- a/smf/clickhouse-admin/manifest.xml +++ b/smf/clickhouse-admin-keeper/manifest.xml @@ -1,9 +1,9 @@ - + - + @@ -35,10 +35,10 @@ diff --git a/smf/clickhouse-admin-server/config.toml b/smf/clickhouse-admin-server/config.toml new file mode 100644 index 0000000000..86ee2c5d4b --- /dev/null +++ b/smf/clickhouse-admin-server/config.toml @@ -0,0 +1,10 @@ +[dropshot] +# 1 MiB; we don't expect any requests of more than nominal size. +request_body_max_bytes = 1048576 + +[log] +# Show log messages of this level and more severe +level = "info" +mode = "file" +path = "/dev/stdout" +if_exists = "append" diff --git a/smf/clickhouse-admin-server/manifest.xml b/smf/clickhouse-admin-server/manifest.xml new file mode 100644 index 0000000000..0b72b56323 --- /dev/null +++ b/smf/clickhouse-admin-server/manifest.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +