diff --git a/Cargo.lock b/Cargo.lock index 0347ff2..4595c33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4302,7 +4302,7 @@ dependencies = [ [[package]] name = "wasmcloud-operator" -version = "0.1.1" +version = "0.2.0" dependencies = [ "anyhow", "async-nats", diff --git a/Cargo.toml b/Cargo.toml index 67927f1..450babe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wasmcloud-operator" -version = "0.1.1" +version = "0.2.0" edition = "2021" [[bin]] diff --git a/Dockerfile.local b/Dockerfile.local index 44263d5..f386202 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM rust:1.75-bookworm as builder +FROM rust:1.77-bookworm as builder WORKDIR /app COPY . . diff --git a/crates/types/src/v1alpha1/wasmcloud_host_config.rs b/crates/types/src/v1alpha1/wasmcloud_host_config.rs index bd0c4f0..e1157b8 100644 --- a/crates/types/src/v1alpha1/wasmcloud_host_config.rs +++ b/crates/types/src/v1alpha1/wasmcloud_host_config.rs @@ -1,8 +1,8 @@ -use k8s_openapi::api::core::v1::ResourceRequirements; +use k8s_openapi::api::core::v1::{PodSpec, ResourceRequirements}; use kube::CustomResource; -use schemars::JsonSchema; +use schemars::{gen::SchemaGenerator, schema::Schema, JsonSchema}; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, JsonSchema)] #[cfg_attr(test, derive(Default))] @@ -36,10 +36,6 @@ pub struct WasmCloudHostConfigSpec { pub enable_structured_logging: Option, /// Name of a secret containing the registry credentials pub registry_credentials_secret: Option, - /// Kubernetes resources to allocate for the host. See - /// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for valid - /// values to use here. - pub resources: Option, /// The control topic prefix to use for the host. pub control_topic_prefix: Option, /// The leaf node domain to use for the NATS sidecar. Defaults to "leaf". @@ -57,9 +53,39 @@ pub struct WasmCloudHostConfigSpec { /// The log level to use for the host. Defaults to "INFO". #[serde(default = "default_log_level")] pub log_level: String, + /// Kubernetes scheduling options for the wasmCloud host. + pub scheduling_options: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] +pub struct KubernetesSchedulingOptions { /// Run hosts as a DaemonSet instead of a Deployment. #[serde(default)] pub daemonset: bool, + /// Kubernetes resources to allocate for the host. See + /// https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for valid + /// values to use here. + pub resources: Option, + #[schemars(schema_with = "pod_schema")] + /// Any other pod template spec options to set for the underlying wasmCloud host pods. + pub pod_template_additions: Option, +} + +/// This is a workaround for the fact that we can't override the PodSpec schema to make containers +/// an optional field. It generates the OpenAPI schema for the PodSpec type the same way that +/// kube.rs does while dropping any required fields. +fn pod_schema(_gen: &mut SchemaGenerator) -> Schema { + let gen = schemars::gen::SchemaSettings::openapi3() + .with(|s| { + s.inline_subschemas = true; + s.meta_schema = None; + }) + .with_visitor(kube::core::schema::StructuralSchemaRewriter) + .into_generator(); + let mut val = gen.into_root_schema_for::(); + // Drop `containers` as a required field, along with any others. + val.schema.object.as_mut().unwrap().required = BTreeSet::new(); + val.schema.into() } fn default_host_replicas() -> u32 { diff --git a/sample.yaml b/sample.yaml index e3c038a..8f4e879 100644 --- a/sample.yaml +++ b/sample.yaml @@ -18,5 +18,23 @@ spec: secretName: cluster-secrets logLevel: INFO natsAddress: nats://nats-cluster.default.svc.cluster.local - # Enable the following to run the wasmCloud hosts as a DaemonSet - #daemonset: true + # Additional options to control how the underlying wasmCloud hosts are scheduled in Kubernetes. + # This includes setting resource requirements for the nats and wasmCloud host + # containers along with any additional pot template settings. + #schedulingOptions: + # Enable the following to run the wasmCloud hosts as a DaemonSet + #daemonset: true + # Set the resource requirements for the nats and wasmCloud host containers. + #resources: + # nats: + # requests: + # cpu: 100m + # wasmCloudHost: + # requests: + # cpu: 100m + # Any additional pod template settings to apply to the wasmCloud host pods. + # See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#podspec-v1-core for all valid options. + # Note that you *cannot* set the `containers` field here as it is managed by the controller. + #pod_template_additions: + # nodeSelector: + # kubernetes.io/os: linux diff --git a/src/controller.rs b/src/controller.rs index c11a005..a83267d 100644 --- a/src/controller.rs +++ b/src/controller.rs @@ -315,9 +315,11 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc) -> PodTemplate let mut nats_resources: Option = None; let mut wasmcloud_resources: Option = None; - if let Some(resources) = &config.spec.resources { - nats_resources = resources.nats.clone(); - wasmcloud_resources = resources.wasmcloud.clone(); + if let Some(scheduling_options) = &config.spec.scheduling_options { + if let Some(resources) = &scheduling_options.resources { + nats_resources = resources.nats.clone(); + wasmcloud_resources = resources.wasmcloud.clone(); + } } let containers = vec![ @@ -371,14 +373,34 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc) -> PodTemplate ..Default::default() }, ]; - PodTemplateSpec { + + let mut volumes = vec![ + Volume { + name: "nats-config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: Some(config.name_any()), + ..Default::default() + }), + ..Default::default() + }, + Volume { + name: "nats-creds".to_string(), + secret: Some(SecretVolumeSource { + secret_name: Some(config.spec.secret_name.clone()), + ..Default::default() + }), + ..Default::default() + }, + ]; + let service_account = config.name_any(); + let mut template = PodTemplateSpec { metadata: Some(ObjectMeta { labels: Some(labels), ..Default::default() }), spec: Some(PodSpec { service_account: Some(config.name_any()), - containers, + containers: containers.clone(), volumes: Some(vec![ Volume { name: "nats-config".to_string(), @@ -399,7 +421,21 @@ fn pod_template(config: &WasmCloudHostConfig, _ctx: Arc) -> PodTemplate ]), ..Default::default() }), - } + }; + + if let Some(scheduling_options) = &config.spec.scheduling_options { + if let Some(pod_overrides) = &scheduling_options.pod_template_additions { + let mut overrides = pod_overrides.clone(); + overrides.service_account_name = Some(service_account); + overrides.containers = containers.clone(); + if let Some(vols) = overrides.volumes { + volumes.extend(vols); + } + overrides.volumes = Some(volumes); + template.spec = Some(overrides); + } + }; + template } fn deployment_spec(config: &WasmCloudHostConfig, ctx: Arc) -> DeploymentSpec { @@ -504,31 +540,34 @@ async fn configure_hosts(config: &WasmCloudHostConfig, ctx: Arc) -> Res ]; } - if config.spec.daemonset { - let mut spec = daemonset_spec(config, ctx.clone()); - spec.template.spec.as_mut().unwrap().containers[1] - .env - .as_mut() - .unwrap() - .append(&mut env_vars); - let ds = DaemonSet { - metadata: ObjectMeta { - name: Some(config.name_any()), - namespace: Some(config.namespace().unwrap()), - owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]), + if let Some(scheduling_options) = &config.spec.scheduling_options { + if scheduling_options.daemonset { + let mut spec = daemonset_spec(config, ctx.clone()); + spec.template.spec.as_mut().unwrap().containers[1] + .env + .as_mut() + .unwrap() + .append(&mut env_vars); + let ds = DaemonSet { + metadata: ObjectMeta { + name: Some(config.name_any()), + namespace: Some(config.namespace().unwrap()), + owner_references: Some(vec![config.controller_owner_ref(&()).unwrap()]), + ..Default::default() + }, + spec: Some(spec), ..Default::default() - }, - spec: Some(spec), - ..Default::default() - }; - - let api = Api::::namespaced(ctx.client.clone(), &config.namespace().unwrap()); - api.patch( - &config.name_any(), - &PatchParams::apply(CLUSTER_CONFIG_FINALIZER), - &Patch::Apply(ds), - ) - .await?; + }; + + let api = + Api::::namespaced(ctx.client.clone(), &config.namespace().unwrap()); + api.patch( + &config.name_any(), + &PatchParams::apply(CLUSTER_CONFIG_FINALIZER), + &Patch::Apply(ds), + ) + .await?; + } } else { let mut spec = deployment_spec(config, ctx.clone()); spec.template.spec.as_mut().unwrap().containers[1]