WIP: Single-node ClickHouse database initialization

Cargo.toml

@@ -8,6 +8,7 @@ members = [
     "clients/bootstrap-agent-client",
     "clients/clickhouse-admin-keeper-client",
     "clients/clickhouse-admin-server-client",
+    "clients/clickhouse-admin-single-client",
     "clients/cockroach-admin-client",
     "clients/ddm-admin-client",
     "clients/dns-service-client",
@@ -131,6 +132,7 @@ default-members = [
     "clients/bootstrap-agent-client",
     "clients/clickhouse-admin-keeper-client",
     "clients/clickhouse-admin-server-client",
+    "clients/clickhouse-admin-single-client",
     "clients/cockroach-admin-client",
     "clients/ddm-admin-client",
     "clients/dns-service-client",
@@ -318,6 +320,7 @@ clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] }
 clickhouse-admin-api = { path = "clickhouse-admin/api" }
 clickhouse-admin-keeper-client = { path = "clients/clickhouse-admin-keeper-client" }
 clickhouse-admin-server-client = { path = "clients/clickhouse-admin-server-client" }
+clickhouse-admin-single-client = { path = "clients/clickhouse-admin-single-client" }
 clickhouse-admin-types = { path = "clickhouse-admin/types" }
 clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "ceec762e6a87d2a22bf56792a3025e145caa095e" }
 cockroach-admin-api = { path = "cockroach-admin/api" }

clickhouse-admin/Cargo.toml

@@ -16,6 +16,7 @@ http.workspace = true
 illumos-utils.workspace = true
 omicron-common.workspace = true
 omicron-uuid-kinds.workspace = true
+oximeter-db.workspace = true
 schemars.workspace = true
 slog.workspace = true
 slog-async.workspace = true
@@ -35,7 +36,6 @@ dropshot.workspace = true
 expectorate.workspace = true
 nexus-test-utils.workspace = true
 omicron-test-utils.workspace = true
-oximeter-db.workspace = true
 oximeter-test-utils.workspace = true
 openapi-lint.workspace = true
 openapiv3.workspace = true

clickhouse-admin/api/src/lib.rs

@@ -8,7 +8,8 @@ use clickhouse_admin_types::{
     ServerConfigurableSettings,
 };
 use dropshot::{
-    HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody,
+    HttpError, HttpResponseCreated, HttpResponseOk,
+    HttpResponseUpdatedNoContent, RequestContext, TypedBody,
 };
 
 /// API interface for our clickhouse-admin-keeper server
@@ -106,3 +107,24 @@ pub trait ClickhouseAdminServerApi {
         body: TypedBody<ServerConfigurableSettings>,
     ) -> Result<HttpResponseCreated<ReplicaConfig>, HttpError>;
 }
+
+/// API interface for our clickhouse-admin-single server
+///
+/// The single-node server is distinct from the both the multi-node servers
+/// and its keepers. The sole purpose of this API is to serialize database
+/// initialization requests from reconfigurator execution. Multi-node clusters
+/// must eventually implement a similar interface, but the implementation will
+/// obviously be more complex.
+#[dropshot::api_description]
+pub trait ClickhouseAdminSingleApi {
+    type Context;
+
+    /// Idempotently initialize a single-node ClickHouse database.
+    #[endpoint {
+        method = PUT,
+        path = "/init"
+    }]
+    async fn init_db(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError>;
+}

clickhouse-admin/src/bin/clickhouse-admin-single.rs

@@ -0,0 +1,72 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Single-node ClickHouse database admin binary.
+
+use anyhow::anyhow;
+use camino::Utf8PathBuf;
+use clap::Parser;
+use omicron_clickhouse_admin::{ClickhouseCli, Config};
+use omicron_common::cmd::fatal;
+use omicron_common::cmd::CmdError;
+use std::net::{SocketAddr, SocketAddrV6};
+
+#[derive(Debug, Parser)]
+#[clap(
+    name = "clickhouse-admin-single",
+    about = "Single-node ClickHouse admin server"
+)]
+enum Args {
+    /// Start the single-node ClickHouse admin server
+    Run {
+        /// Address on which this server should run
+        #[clap(long, short = 'a', action)]
+        http_address: SocketAddrV6,
+
+        /// Path to the server configuration file
+        #[clap(long, short, action)]
+        config: Utf8PathBuf,
+
+        /// Address of the ClickHouse single-node database server
+        #[clap(long, short = 'l', action)]
+        listen_address: SocketAddrV6,
+
+        /// Path to the clickhouse binary
+        #[clap(long, short, action)]
+        binary_path: Utf8PathBuf,
+    },
+}
+
+#[tokio::main]
+async fn main() {
+    if let Err(err) = main_impl().await {
+        fatal(err);
+    }
+}
+
+async fn main_impl() -> Result<(), CmdError> {
+    let args = Args::parse();
+
+    match args {
+        Args::Run { http_address, config, listen_address, binary_path } => {
+            let mut config = Config::from_file(&config)
+                .map_err(|err| CmdError::Failure(anyhow!(err)))?;
+            config.dropshot.bind_address = SocketAddr::V6(http_address);
+            let clickhouse_cli =
+                ClickhouseCli::new(binary_path, listen_address);
+
+            let server = omicron_clickhouse_admin::start_single_admin_server(
+                clickhouse_cli,
+                config,
+            )
+            .await
+            .map_err(|err| CmdError::Failure(anyhow!(err)))?;
+            server.await.map_err(|err| {
+                CmdError::Failure(anyhow!(
+                    "server failed after starting: {err}"
+                ))
+            })
+        }
+    }
+}

clickhouse-admin/src/context.rs

@@ -4,6 +4,8 @@
 
 use crate::{ClickhouseCli, Clickward};
 use slog::Logger;
+use std::sync::Arc;
+use tokio::sync::Mutex;
 
 pub struct ServerContext {
     clickward: Clickward,
@@ -28,3 +30,22 @@ impl ServerContext {
         &self.clickhouse_cli
     }
 }
+
+pub struct SingleServerContext {
+    clickhouse_cli: ClickhouseCli,
+    db_initialized: Arc<Mutex<bool>>,
+}
+
+impl SingleServerContext {
+    pub fn new(clickhouse_cli: ClickhouseCli) -> Self {
+        Self { clickhouse_cli, db_initialized: Arc::new(Mutex::new(false)) }
+    }
+
+    pub fn clickhouse_cli(&self) -> &ClickhouseCli {
+        &self.clickhouse_cli
+    }
+
+    pub fn db_initialized(&self) -> Arc<Mutex<bool>> {
+        self.db_initialized.clone()
+    }
+}

clickhouse-admin/src/http_entrypoints.rs

@@ -2,31 +2,38 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use crate::context::ServerContext;
+use crate::context::{ServerContext, SingleServerContext};
 use clickhouse_admin_api::*;
 use clickhouse_admin_types::{
     ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig,
     KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig,
     ServerConfigurableSettings,
 };
 use dropshot::{
-    HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody,
+    ApiDescription, HttpError, HttpResponseCreated, HttpResponseOk,
+    HttpResponseUpdatedNoContent, RequestContext, TypedBody,
 };
 use illumos_utils::svcadm::Svcadm;
+use oximeter_db::{Client as OximeterClient, OXIMETER_VERSION};
+use slog::debug;
 use std::sync::Arc;
 
-type ClickhouseApiDescription = dropshot::ApiDescription<Arc<ServerContext>>;
-
-pub fn clickhouse_admin_server_api() -> ClickhouseApiDescription {
+pub fn clickhouse_admin_server_api() -> ApiDescription<Arc<ServerContext>> {
     clickhouse_admin_server_api_mod::api_description::<ClickhouseAdminServerImpl>()
         .expect("registered entrypoints")
 }
 
-pub fn clickhouse_admin_keeper_api() -> ClickhouseApiDescription {
+pub fn clickhouse_admin_keeper_api() -> ApiDescription<Arc<ServerContext>> {
     clickhouse_admin_keeper_api_mod::api_description::<ClickhouseAdminKeeperImpl>()
         .expect("registered entrypoints")
 }
 
+pub fn clickhouse_admin_single_api() -> ApiDescription<Arc<SingleServerContext>>
+{
+    clickhouse_admin_single_api_mod::api_description::<ClickhouseAdminSingleImpl>()
+        .expect("registered entrypoints")
+}
+
 enum ClickhouseAdminServerImpl {}
 
 impl ClickhouseAdminServerApi for ClickhouseAdminServerImpl {
@@ -102,3 +109,38 @@ impl ClickhouseAdminKeeperApi for ClickhouseAdminKeeperImpl {
         Ok(HttpResponseOk(output))
     }
 }
+
+enum ClickhouseAdminSingleImpl {}
+
+impl ClickhouseAdminSingleApi for ClickhouseAdminSingleImpl {
+    type Context = Arc<SingleServerContext>;
+
+    async fn init_db(
+        rqctx: RequestContext<Self::Context>,
+    ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+        let log = &rqctx.log;
+        let ctx = rqctx.context();
+        let initialized = ctx.db_initialized();
+        let mut initialized = initialized.lock().await;
+        if !*initialized {
+            let address = ctx.clickhouse_cli().listen_address;
+            let client = OximeterClient::new(address.into(), log);
+            debug!(
+                log,
+                "initializing single-node ClickHouse \
+                 at {address} to version {OXIMETER_VERSION}"
+            );
+            client
+                .initialize_db_with_version(false, OXIMETER_VERSION)
+                .await
+                .map_err(|e| {
+                    HttpError::for_internal_error(format!(
+                        "can't initialize single-node ClickHouse \
+                         at {address} to version {OXIMETER_VERSION}: {e}",
+                    ))
+                })?;
+            *initialized = true;
+        }
+        Ok(HttpResponseUpdatedNoContent())
+    }
+}

clickhouse-admin/src/lib.rs

@@ -2,7 +2,8 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use context::ServerContext;
+use context::{ServerContext, SingleServerContext};
+use dropshot::{HttpServer, HttpServerStarter};
 use omicron_common::FileKv;
 use slog::{debug, error, Drain};
 use slog_dtrace::ProbeRegistration;
@@ -31,15 +32,13 @@ pub enum StartError {
     InitializeHttpServer(#[source] Box<dyn Error + Send + Sync>),
 }
 
-pub type Server = dropshot::HttpServer<Arc<ServerContext>>;
-
 /// Start the dropshot server for `clickhouse-admin-server` which
 /// manages clickhouse replica servers.
 pub async fn start_server_admin_server(
     clickward: Clickward,
     clickhouse_cli: ClickhouseCli,
     server_config: Config,
-) -> Result<Server, StartError> {
+) -> Result<HttpServer<Arc<ServerContext>>, StartError> {
     let (drain, registration) = slog_dtrace::with_drain(
         server_config
             .log
@@ -64,7 +63,7 @@ pub async fn start_server_admin_server(
             .with_log(log.new(slog::o!("component" => "ClickhouseCli"))),
         log.new(slog::o!("component" => "ServerContext")),
     );
-    let http_server_starter = dropshot::HttpServerStarter::new(
+    let http_server_starter = HttpServerStarter::new(
         &server_config.dropshot,
         http_entrypoints::clickhouse_admin_server_api(),
         Arc::new(context),
@@ -81,7 +80,7 @@ pub async fn start_keeper_admin_server(
     clickward: Clickward,
     clickhouse_cli: ClickhouseCli,
     server_config: Config,
-) -> Result<Server, StartError> {
+) -> Result<HttpServer<Arc<ServerContext>>, StartError> {
     let (drain, registration) = slog_dtrace::with_drain(
         server_config
             .log
@@ -106,7 +105,7 @@ pub async fn start_keeper_admin_server(
             .with_log(log.new(slog::o!("component" => "ClickhouseCli"))),
         log.new(slog::o!("component" => "ServerContext")),
     );
-    let http_server_starter = dropshot::HttpServerStarter::new(
+    let http_server_starter = HttpServerStarter::new(
         &server_config.dropshot,
         http_entrypoints::clickhouse_admin_keeper_api(),
         Arc::new(context),
@@ -116,3 +115,39 @@ pub async fn start_keeper_admin_server(
 
     Ok(http_server_starter.start())
 }
+
+/// Start the dropshot server for `clickhouse-admin-single` which
+/// manages a single-node ClickHouse database.
+pub async fn start_single_admin_server(
+    clickhouse_cli: ClickhouseCli,
+    server_config: Config,
+) -> Result<HttpServer<Arc<SingleServerContext>>, StartError> {
+    let (drain, registration) = slog_dtrace::with_drain(
+        server_config
+            .log
+            .to_logger("clickhouse-admin-keeper")
+            .map_err(StartError::InitializeLogger)?,
+    );
+    let log = slog::Logger::root(drain.fuse(), slog::o!(FileKv));
+    match registration {
+        ProbeRegistration::Success => {
+            debug!(log, "registered DTrace probes");
+        }
+        ProbeRegistration::Failed(err) => {
+            let err = StartError::RegisterDtraceProbes(err);
+            error!(log, "failed to register DTrace probes"; &err);
+            return Err(err);
+        }
+    }
+
+    let context = SingleServerContext::new(clickhouse_cli);
+    let http_server_starter = HttpServerStarter::new(
+        &server_config.dropshot,
+        http_entrypoints::clickhouse_admin_single_api(),
+        Arc::new(context),
+        &log.new(slog::o!("component" => "dropshot")),
+    )
+    .map_err(StartError::InitializeHttpServer)?;
+
+    Ok(http_server_starter.start())
+}