Skip to content

Commit

Permalink
Split clickhouse admin keeper and replica servers
Browse files Browse the repository at this point in the history
Also, split the clients and remove the dependency
of clickhouse-admin-api from nexus.
  • Loading branch information
andrewjstone committed Oct 11, 2024
1 parent 1f3de04 commit f039746
Show file tree
Hide file tree
Showing 24 changed files with 1,675 additions and 142 deletions.
28 changes: 22 additions & 6 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 6 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ members = [
"clickhouse-admin",
"clickhouse-admin/api",
"clients/bootstrap-agent-client",
"clients/clickhouse-admin-client",
"clients/clickhouse-admin-keeper-client",
"clients/clickhouse-admin-server-client",
"clients/cockroach-admin-client",
"clients/ddm-admin-client",
"clients/dns-service-client",
Expand Down Expand Up @@ -128,7 +129,8 @@ default-members = [
"clickhouse-admin/api",
"clickhouse-admin/types",
"clients/bootstrap-agent-client",
"clients/clickhouse-admin-client",
"clients/clickhouse-admin-keeper-client",
"clients/clickhouse-admin-server-client",
"clients/cockroach-admin-client",
"clients/ddm-admin-client",
"clients/dns-service-client",
Expand Down Expand Up @@ -313,7 +315,8 @@ chrono = { version = "0.4", features = [ "serde" ] }
ciborium = "0.2.2"
clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] }
clickhouse-admin-api = { path = "clickhouse-admin/api" }
clickhouse-admin-client = { path = "clients/clickhouse-admin-client" }
clickhouse-admin-keeper-client = { path = "clients/clickhouse-admin-keeper-client" }
clickhouse-admin-server-client = { path = "clients/clickhouse-admin-server-client" }
clickhouse-admin-types = { path = "clickhouse-admin/types" }
clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "ceec762e6a87d2a22bf56792a3025e145caa095e" }
cockroach-admin-api = { path = "cockroach-admin/api" }
Expand Down
63 changes: 26 additions & 37 deletions clickhouse-admin/api/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,56 +2,29 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
use clickhouse_admin_types::{
ClickhouseKeeperClusterMembership, KeeperConf, KeeperSettings, Lgif,
RaftConfig, ServerSettings,
ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig,
KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig,
ServerConfigurableSettings,
};
use dropshot::{
HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody,
};
use omicron_common::api::external::Generation;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};

#[derive(Debug, Serialize, Deserialize, JsonSchema)]
pub struct ServerConfigurableSettings {
/// A unique identifier for the configuration generation.
pub generation: Generation,
/// Configurable settings for a ClickHouse replica server node.
pub settings: ServerSettings,
}

#[derive(Debug, Serialize, Deserialize, JsonSchema)]
pub struct KeeperConfigurableSettings {
/// A unique identifier for the configuration generation.
pub generation: Generation,
/// Configurable settings for a ClickHouse keeper node.
pub settings: KeeperSettings,
}

#[dropshot::api_description]
pub trait ClickhouseAdminApi {
pub trait ClickhouseAdminKeeperApi {
type Context;

/// Generate a ClickHouse configuration file for a server node on a specified
/// directory and enable the SMF service.
#[endpoint {
method = PUT,
path = "/server/config-and-enable",
}]
async fn generate_server_config_and_enable(
rqctx: RequestContext<Self::Context>,
body: TypedBody<ServerConfigurableSettings>,
) -> Result<HttpResponseCreated<ReplicaConfig>, HttpError>;

/// Generate a ClickHouse configuration file for a keeper node on a specified
/// directory and enable the SMF service.
/// directory and enable the SMF service if not currently enabled.
///
/// Note that we cannot start the keeper service until there is an initial
/// configuration set via this endpoint.
#[endpoint {
method = PUT,
path = "/keeper/config-and-enable",
path = "/config",
}]
async fn generate_keeper_config_and_enable(
async fn generate_config(
rqctx: RequestContext<Self::Context>,
body: TypedBody<KeeperConfigurableSettings>,
) -> Result<HttpResponseCreated<KeeperConfig>, HttpError>;
Expand Down Expand Up @@ -95,3 +68,19 @@ pub trait ClickhouseAdminApi {
rqctx: RequestContext<Self::Context>,
) -> Result<HttpResponseOk<ClickhouseKeeperClusterMembership>, HttpError>;
}

#[dropshot::api_description]
pub trait ClickhouseAdminServerApi {
type Context;

/// Generate a ClickHouse configuration file for a server node on a specified
/// directory and enable the SMF service.
#[endpoint {
method = PUT,
path = "/config"
}]
async fn generate_config(
rqctx: RequestContext<Self::Context>,
body: TypedBody<ServerConfigurableSettings>,
) -> Result<HttpResponseCreated<ReplicaConfig>, HttpError>;
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

//! Executable program to run the Omicron ClickHouse admin interface
//! Executable program to run the Omicron ClickHouse admin interface for
//! clickhouse keepers.

use anyhow::anyhow;
use camino::Utf8PathBuf;
Expand All @@ -14,8 +15,8 @@ use std::net::{SocketAddr, SocketAddrV6};

#[derive(Debug, Parser)]
#[clap(
name = "clickhouse-admin",
about = "Omicron ClickHouse cluster admin server"
name = "clickhouse-admin-keeper",
about = "Omicron ClickHouse cluster admin server for keepers"
)]
enum Args {
/// Start the ClickHouse admin server
Expand Down Expand Up @@ -57,7 +58,7 @@ async fn main_impl() -> Result<(), CmdError> {
let clickhouse_cli =
ClickhouseCli::new(binary_path, listen_address);

let server = omicron_clickhouse_admin::start_server(
let server = omicron_clickhouse_admin::start_keeper_admin_server(
clickward,
clickhouse_cli,
config,
Expand Down
75 changes: 75 additions & 0 deletions clickhouse-admin/src/bin/clickhouse-admin-server.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

//! Executable program to run the Omicron ClickHouse admin interface for
//! clickhouse servers.

use anyhow::anyhow;
use camino::Utf8PathBuf;
use clap::Parser;
use omicron_clickhouse_admin::{ClickhouseCli, Clickward, Config};
use omicron_common::cmd::fatal;
use omicron_common::cmd::CmdError;
use std::net::{SocketAddr, SocketAddrV6};

#[derive(Debug, Parser)]
#[clap(
name = "clickhouse-admin-server",
about = "Omicron ClickHouse cluster admin server for replica servers"
)]
enum Args {
/// Start the ClickHouse admin server
Run {
/// Address on which this server should run
#[clap(long, short = 'a', action)]
http_address: SocketAddrV6,

/// Path to the server configuration file
#[clap(long, short, action)]
config: Utf8PathBuf,

/// Address on which the clickhouse server or keeper is listening on
#[clap(long, short = 'l', action)]
listen_address: SocketAddrV6,

/// Path to the clickhouse binary
#[clap(long, short, action)]
binary_path: Utf8PathBuf,
},
}

#[tokio::main]
async fn main() {
if let Err(err) = main_impl().await {
fatal(err);
}
}

async fn main_impl() -> Result<(), CmdError> {
let args = Args::parse();

match args {
Args::Run { http_address, config, listen_address, binary_path } => {
let mut config = Config::from_file(&config)
.map_err(|err| CmdError::Failure(anyhow!(err)))?;
config.dropshot.bind_address = SocketAddr::V6(http_address);
let clickward = Clickward::new();
let clickhouse_cli =
ClickhouseCli::new(binary_path, listen_address);

let server = omicron_clickhouse_admin::start_server_admin_server(
clickward,
clickhouse_cli,
config,
)
.await
.map_err(|err| CmdError::Failure(anyhow!(err)))?;
server.await.map_err(|err| {
CmdError::Failure(anyhow!(
"server failed after starting: {err}"
))
})
}
}
}
5 changes: 3 additions & 2 deletions clickhouse-admin/src/clickward.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
use clickhouse_admin_types::{KeeperSettings, ServerSettings};
use clickhouse_admin_types::{
KeeperConfig, KeeperSettings, ReplicaConfig, ServerSettings,
};
use dropshot::HttpError;
use slog_error_chain::{InlineErrorChain, SlogInlineError};

Expand Down
28 changes: 20 additions & 8 deletions clickhouse-admin/src/http_entrypoints.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@

use crate::context::ServerContext;
use clickhouse_admin_api::*;
use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig};
use clickhouse_admin_types::{
ClickhouseKeeperClusterMembership, KeeperConf, Lgif, RaftConfig,
ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig,
KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig,
ServerConfigurableSettings,
};
use dropshot::{
HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody,
Expand All @@ -16,17 +17,22 @@ use std::sync::Arc;

type ClickhouseApiDescription = dropshot::ApiDescription<Arc<ServerContext>>;

pub fn api() -> ClickhouseApiDescription {
clickhouse_admin_api_mod::api_description::<ClickhouseAdminImpl>()
pub fn clickhouse_admin_server_api() -> ClickhouseApiDescription {
clickhouse_admin_server_api_mod::api_description::<ClickhouseAdminServerImpl>()
.expect("registered entrypoints")
}

enum ClickhouseAdminImpl {}
pub fn clickhouse_admin_keeper_api() -> ClickhouseApiDescription {
clickhouse_admin_keeper_api_mod::api_description::<ClickhouseAdminKeeperImpl>()
.expect("registered entrypoints")
}

impl ClickhouseAdminApi for ClickhouseAdminImpl {
enum ClickhouseAdminServerImpl {}

impl ClickhouseAdminServerApi for ClickhouseAdminServerImpl {
type Context = Arc<ServerContext>;

async fn generate_server_config_and_enable(
async fn generate_config(
rqctx: RequestContext<Self::Context>,
body: TypedBody<ServerConfigurableSettings>,
) -> Result<HttpResponseCreated<ReplicaConfig>, HttpError> {
Expand All @@ -41,8 +47,14 @@ impl ClickhouseAdminApi for ClickhouseAdminImpl {

Ok(HttpResponseCreated(output))
}
}

enum ClickhouseAdminKeeperImpl {}

impl ClickhouseAdminKeeperApi for ClickhouseAdminKeeperImpl {
type Context = Arc<ServerContext>;

async fn generate_keeper_config_and_enable(
async fn generate_config(
rqctx: RequestContext<Self::Context>,
body: TypedBody<KeeperConfigurableSettings>,
) -> Result<HttpResponseCreated<KeeperConfig>, HttpError> {
Expand Down
Loading

0 comments on commit f039746

Please sign in to comment.