Skip to content

Commit

Permalink
Split clickhouse admin server (#6837)
Browse files Browse the repository at this point in the history
Fixes #6779
Fixes #6829
  • Loading branch information
andrewjstone authored Oct 12, 2024
1 parent 4d17944 commit e634215
Show file tree
Hide file tree
Showing 32 changed files with 913 additions and 487 deletions.
28 changes: 22 additions & 6 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 6 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ members = [
"clickhouse-admin",
"clickhouse-admin/api",
"clients/bootstrap-agent-client",
"clients/clickhouse-admin-client",
"clients/clickhouse-admin-keeper-client",
"clients/clickhouse-admin-server-client",
"clients/cockroach-admin-client",
"clients/ddm-admin-client",
"clients/dns-service-client",
Expand Down Expand Up @@ -128,7 +129,8 @@ default-members = [
"clickhouse-admin/api",
"clickhouse-admin/types",
"clients/bootstrap-agent-client",
"clients/clickhouse-admin-client",
"clients/clickhouse-admin-keeper-client",
"clients/clickhouse-admin-server-client",
"clients/cockroach-admin-client",
"clients/ddm-admin-client",
"clients/dns-service-client",
Expand Down Expand Up @@ -313,7 +315,8 @@ chrono = { version = "0.4", features = [ "serde" ] }
ciborium = "0.2.2"
clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] }
clickhouse-admin-api = { path = "clickhouse-admin/api" }
clickhouse-admin-client = { path = "clients/clickhouse-admin-client" }
clickhouse-admin-keeper-client = { path = "clients/clickhouse-admin-keeper-client" }
clickhouse-admin-server-client = { path = "clients/clickhouse-admin-server-client" }
clickhouse-admin-types = { path = "clickhouse-admin/types" }
clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "ceec762e6a87d2a22bf56792a3025e145caa095e" }
cockroach-admin-api = { path = "cockroach-admin/api" }
Expand Down
85 changes: 48 additions & 37 deletions clickhouse-admin/api/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,56 +2,40 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
use clickhouse_admin_types::{
ClickhouseKeeperClusterMembership, KeeperConf, KeeperSettings, Lgif,
RaftConfig, ServerSettings,
ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig,
KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig,
ServerConfigurableSettings,
};
use dropshot::{
HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody,
};
use omicron_common::api::external::Generation;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};

#[derive(Debug, Serialize, Deserialize, JsonSchema)]
pub struct ServerConfigurableSettings {
/// A unique identifier for the configuration generation.
pub generation: Generation,
/// Configurable settings for a ClickHouse replica server node.
pub settings: ServerSettings,
}

#[derive(Debug, Serialize, Deserialize, JsonSchema)]
pub struct KeeperConfigurableSettings {
/// A unique identifier for the configuration generation.
pub generation: Generation,
/// Configurable settings for a ClickHouse keeper node.
pub settings: KeeperSettings,
}

/// API interface for our clickhouse-admin-keeper server
///
/// We separate the admin interface for the keeper and server APIs because they
/// are completely disjoint. We only run a clickhouse keeper *or* clickhouse
/// server in a given zone, and therefore each admin api is only useful in one
/// of the zones. Using separate APIs and clients prevents us from having to
/// mark a given endpoint `unimplemented` in the case of it not being usable
/// with one of the zone types.
///
/// Nonetheless, the interfaces themselves are small and serve a similar
/// purpose. Therfore we combine them into the same crate.
#[dropshot::api_description]
pub trait ClickhouseAdminApi {
pub trait ClickhouseAdminKeeperApi {
type Context;

/// Generate a ClickHouse configuration file for a server node on a specified
/// directory and enable the SMF service.
#[endpoint {
method = PUT,
path = "/server/config-and-enable",
}]
async fn generate_server_config_and_enable(
rqctx: RequestContext<Self::Context>,
body: TypedBody<ServerConfigurableSettings>,
) -> Result<HttpResponseCreated<ReplicaConfig>, HttpError>;

/// Generate a ClickHouse configuration file for a keeper node on a specified
/// directory and enable the SMF service.
/// directory and enable the SMF service if not currently enabled.
///
/// Note that we cannot start the keeper service until there is an initial
/// configuration set via this endpoint.
#[endpoint {
method = PUT,
path = "/keeper/config-and-enable",
path = "/config",
}]
async fn generate_keeper_config_and_enable(
async fn generate_config(
rqctx: RequestContext<Self::Context>,
body: TypedBody<KeeperConfigurableSettings>,
) -> Result<HttpResponseCreated<KeeperConfig>, HttpError>;
Expand Down Expand Up @@ -95,3 +79,30 @@ pub trait ClickhouseAdminApi {
rqctx: RequestContext<Self::Context>,
) -> Result<HttpResponseOk<ClickhouseKeeperClusterMembership>, HttpError>;
}

/// API interface for our clickhouse-admin-server server
///
/// We separate the admin interface for the keeper and server APIs because they
/// are completely disjoint. We only run a clickhouse keeper *or* clickhouse
/// server in a given zone, and therefore each admin api is only useful in one
/// of the zones. Using separate APIs and clients prevents us from having to
/// mark a given endpoint `unimplemented` in the case of it not being usable
/// with one of the zone types.
///
/// Nonetheless, the interfaces themselves are small and serve a similar
/// purpose. Therfore we combine them into the same crate.
#[dropshot::api_description]
pub trait ClickhouseAdminServerApi {
type Context;

/// Generate a ClickHouse configuration file for a server node on a specified
/// directory and enable the SMF service.
#[endpoint {
method = PUT,
path = "/config"
}]
async fn generate_config(
rqctx: RequestContext<Self::Context>,
body: TypedBody<ServerConfigurableSettings>,
) -> Result<HttpResponseCreated<ReplicaConfig>, HttpError>;
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

//! Executable program to run the Omicron ClickHouse admin interface
//! Executable program to run the Omicron ClickHouse admin interface for
//! clickhouse keepers.

use anyhow::anyhow;
use camino::Utf8PathBuf;
Expand All @@ -14,8 +15,8 @@ use std::net::{SocketAddr, SocketAddrV6};

#[derive(Debug, Parser)]
#[clap(
name = "clickhouse-admin",
about = "Omicron ClickHouse cluster admin server"
name = "clickhouse-admin-keeper",
about = "Omicron ClickHouse cluster admin server for keepers"
)]
enum Args {
/// Start the ClickHouse admin server
Expand Down Expand Up @@ -57,7 +58,7 @@ async fn main_impl() -> Result<(), CmdError> {
let clickhouse_cli =
ClickhouseCli::new(binary_path, listen_address);

let server = omicron_clickhouse_admin::start_server(
let server = omicron_clickhouse_admin::start_keeper_admin_server(
clickward,
clickhouse_cli,
config,
Expand Down
75 changes: 75 additions & 0 deletions clickhouse-admin/src/bin/clickhouse-admin-server.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

//! Executable program to run the Omicron ClickHouse admin interface for
//! clickhouse servers.

use anyhow::anyhow;
use camino::Utf8PathBuf;
use clap::Parser;
use omicron_clickhouse_admin::{ClickhouseCli, Clickward, Config};
use omicron_common::cmd::fatal;
use omicron_common::cmd::CmdError;
use std::net::{SocketAddr, SocketAddrV6};

#[derive(Debug, Parser)]
#[clap(
name = "clickhouse-admin-server",
about = "Omicron ClickHouse cluster admin server for replica servers"
)]
enum Args {
/// Start the ClickHouse admin server
Run {
/// Address on which this server should run
#[clap(long, short = 'a', action)]
http_address: SocketAddrV6,

/// Path to the server configuration file
#[clap(long, short, action)]
config: Utf8PathBuf,

/// Address on which the clickhouse server or keeper is listening on
#[clap(long, short = 'l', action)]
listen_address: SocketAddrV6,

/// Path to the clickhouse binary
#[clap(long, short, action)]
binary_path: Utf8PathBuf,
},
}

#[tokio::main]
async fn main() {
if let Err(err) = main_impl().await {
fatal(err);
}
}

async fn main_impl() -> Result<(), CmdError> {
let args = Args::parse();

match args {
Args::Run { http_address, config, listen_address, binary_path } => {
let mut config = Config::from_file(&config)
.map_err(|err| CmdError::Failure(anyhow!(err)))?;
config.dropshot.bind_address = SocketAddr::V6(http_address);
let clickward = Clickward::new();
let clickhouse_cli =
ClickhouseCli::new(binary_path, listen_address);

let server = omicron_clickhouse_admin::start_server_admin_server(
clickward,
clickhouse_cli,
config,
)
.await
.map_err(|err| CmdError::Failure(anyhow!(err)))?;
server.await.map_err(|err| {
CmdError::Failure(anyhow!(
"server failed after starting: {err}"
))
})
}
}
}
5 changes: 3 additions & 2 deletions clickhouse-admin/src/clickward.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
use clickhouse_admin_types::{KeeperSettings, ServerSettings};
use clickhouse_admin_types::{
KeeperConfig, KeeperSettings, ReplicaConfig, ServerSettings,
};
use dropshot::HttpError;
use slog_error_chain::{InlineErrorChain, SlogInlineError};

Expand Down
28 changes: 20 additions & 8 deletions clickhouse-admin/src/http_entrypoints.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@

use crate::context::ServerContext;
use clickhouse_admin_api::*;
use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
use clickhouse_admin_types::{
ClickhouseKeeperClusterMembership, KeeperConf, Lgif, RaftConfig,
ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig,
KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig,
ServerConfigurableSettings,
};
use dropshot::{
HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody,
Expand All @@ -16,17 +17,22 @@ use std::sync::Arc;

type ClickhouseApiDescription = dropshot::ApiDescription<Arc<ServerContext>>;

pub fn api() -> ClickhouseApiDescription {
clickhouse_admin_api_mod::api_description::<ClickhouseAdminImpl>()
pub fn clickhouse_admin_server_api() -> ClickhouseApiDescription {
clickhouse_admin_server_api_mod::api_description::<ClickhouseAdminServerImpl>()
.expect("registered entrypoints")
}

enum ClickhouseAdminImpl {}
pub fn clickhouse_admin_keeper_api() -> ClickhouseApiDescription {
clickhouse_admin_keeper_api_mod::api_description::<ClickhouseAdminKeeperImpl>()
.expect("registered entrypoints")
}

impl ClickhouseAdminApi for ClickhouseAdminImpl {
enum ClickhouseAdminServerImpl {}

impl ClickhouseAdminServerApi for ClickhouseAdminServerImpl {
type Context = Arc<ServerContext>;

async fn generate_server_config_and_enable(
async fn generate_config(
rqctx: RequestContext<Self::Context>,
body: TypedBody<ServerConfigurableSettings>,
) -> Result<HttpResponseCreated<ReplicaConfig>, HttpError> {
Expand All @@ -41,8 +47,14 @@ impl ClickhouseAdminApi for ClickhouseAdminImpl {

Ok(HttpResponseCreated(output))
}
}

enum ClickhouseAdminKeeperImpl {}

impl ClickhouseAdminKeeperApi for ClickhouseAdminKeeperImpl {
type Context = Arc<ServerContext>;

async fn generate_keeper_config_and_enable(
async fn generate_config(
rqctx: RequestContext<Self::Context>,
body: TypedBody<KeeperConfigurableSettings>,
) -> Result<HttpResponseCreated<KeeperConfig>, HttpError> {
Expand Down
Loading

0 comments on commit e634215

Please sign in to comment.