diff --git a/.github/workflows/kubernetes.yml b/.github/workflows/kubernetes.yml index 3b2698b610b..3892ae0d3db 100644 --- a/.github/workflows/kubernetes.yml +++ b/.github/workflows/kubernetes.yml @@ -46,6 +46,8 @@ jobs: strip target/release/linera-proxy strip target/release/linera-server strip target/release/linera-db + - name: Setup helmfile + uses: mamezou-tech/setup-helmfile@v1.3.0 - name: Run Wasm e2e test uses: nick-fields/retry@v2 with: diff --git a/Cargo.lock b/Cargo.lock index 23ed225e66e..6d36105630e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2226,6 +2226,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fungible" version = "0.1.0" @@ -3434,6 +3440,7 @@ dependencies = [ "dirs", "file-lock", "fs-err", + "fs_extra", "fungible", "futures", "hex", diff --git a/Cargo.toml b/Cargo.toml index e71f0cc780d..9ce0470d72b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ ed25519-dalek = { version = "1.0.1", features = ["batch", "serde"] } either = "1.9.0" frunk = "0.4.2" fs-err = { version = "2.11.0", features = ["tokio"] } +fs_extra = "1.3.0" futures = "0.3.29" generic-array = { version = "0.14.7", features = ["serde"] } hex = "0.4.3" diff --git a/docker/server-entrypoint.sh b/docker/server-entrypoint.sh index 544ba38551d..c14d26d1094 100644 --- a/docker/server-entrypoint.sh +++ b/docker/server-entrypoint.sh @@ -4,7 +4,7 @@ ORDINAL="${HOSTNAME##*-}" exec ./linera-server run \ - --storage scylladb:tcp:scylladb.default.svc.cluster.local:9042 \ + --storage scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042 \ --server /config/server.json \ --shard $ORDINAL \ --genesis /config/genesis.json diff --git a/docker/server-init.sh b/docker/server-init.sh index e4d9157313e..34d4b4ea3e3 100644 --- a/docker/server-init.sh +++ b/docker/server-init.sh @@ -1,7 +1,7 @@ #!/bin/sh while true; do - ./linera-db check_existence --storage "scylladb:tcp:scylladb.default.svc.cluster.local:9042" + ./linera-db check_existence --storage "scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042" status=$? if [ $status -eq 0 ]; then @@ -10,7 +10,7 @@ while true; do elif [ $status -eq 1 ]; then echo "Database does not exist, attempting to initialize..." if ./linera-server initialize \ - --storage scylladb:tcp:scylladb.default.svc.cluster.local:9042 \ + --storage scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042 \ --genesis /config/genesis.json; then echo "Initialization successful." exit 0 diff --git a/kubernetes/linera-validator/Chart.lock b/kubernetes/linera-validator/Chart.lock index fcac64c02f0..c4d2e00ef72 100644 --- a/kubernetes/linera-validator/Chart.lock +++ b/kubernetes/linera-validator/Chart.lock @@ -6,4 +6,4 @@ dependencies: repository: https://grafana.github.io/helm-charts version: 2.8.9 digest: sha256:da3a8808fb9479bdd183b307e70d7855794739cafc1796047bde8febc78d539c -generated: "2023-10-23T17:50:47.813398009+01:00" +generated: "2023-11-30T17:19:14.544457977Z" diff --git a/kubernetes/linera-validator/helmfile.yaml b/kubernetes/linera-validator/helmfile.yaml new file mode 100644 index 00000000000..f262b2d3c41 --- /dev/null +++ b/kubernetes/linera-validator/helmfile.yaml @@ -0,0 +1,59 @@ +repositories: + - name: scylla + url: https://scylla-operator-charts.storage.googleapis.com/stable + - name: jetstack + url: https://charts.jetstack.io + - name: prometheus-community + url: https://prometheus-community.github.io/helm-charts + - name: grafana + url: https://grafana.github.io/helm-charts + +releases: + - name: linera-core + namespace: default + chart: . + timeout: 900 + needs: + - scylla/scylla + values: + - {{ env "LINERA_HELMFILE_VALUES_LINERA_CORE" | default "values-local.yaml" }} + set: + - name: installCRDs + value: "true" + - name: validator.serverConfig + value: {{ env "LINERA_HELMFILE_SET_SERVER_CONFIG" | default "working/server_1.json" }} + - name: validator.genesisConfig + value: {{ env "LINERA_HELMFILE_SET_GENESIS_CONFIG" | default "working/genesis.json" }} + - name: numShards + value: {{ env "LINERA_HELMFILE_SET_NUM_SHARDS" | default 10 }} + - name: scylla + namespace: scylla + chart: scylla/scylla + timeout: 900 + needs: + - scylla-manager/scylla-manager + - scylla-operator/scylla-operator + values: + - {{ env "LINERA_HELMFILE_VALUES_SCYLLA" | default "scylla.values.yaml" }} + - name: scylla-manager + namespace: scylla-manager + chart: scylla/scylla-manager + timeout: 900 + needs: + - scylla-operator/scylla-operator + - name: scylla-operator + namespace: scylla-operator + chart: scylla/scylla-operator + timeout: 900 + needs: + - cert-manager/cert-manager + values: + - {{ env "LINERA_HELMFILE_VALUES_SCYLLA_OPERATOR" | default "scylla-operator.values.yaml" }} + - name: cert-manager + namespace: cert-manager + chart: jetstack/cert-manager + timeout: 900 + set: + - name: installCRDs + value: "true" + diff --git a/kubernetes/linera-validator/scylla-operator.values.yaml b/kubernetes/linera-validator/scylla-operator.values.yaml new file mode 100644 index 00000000000..dfe5d2bb153 --- /dev/null +++ b/kubernetes/linera-validator/scylla-operator.values.yaml @@ -0,0 +1,3 @@ +webhook: + createSelfSignedCertificate: true + certificateSecretName: "" diff --git a/kubernetes/linera-validator/scylla.values.yaml b/kubernetes/linera-validator/scylla.values.yaml new file mode 100644 index 00000000000..99afc9b0028 --- /dev/null +++ b/kubernetes/linera-validator/scylla.values.yaml @@ -0,0 +1,16 @@ +developerMode: true +sysctls: + - "fs.aio-max-nr=4082080" +datacenter: validator +racks: + - name: rack-1 + members: 1 + storage: + capacity: 1Gi + resources: + limits: + cpu: 1 + memory: 0.2Gi + requests: + cpu: 1 + memory: 0.2Gi diff --git a/kubernetes/linera-validator/templates/scylladb.yaml b/kubernetes/linera-validator/templates/scylladb.yaml deleted file mode 100644 index 0ff70e53c54..00000000000 --- a/kubernetes/linera-validator/templates/scylladb.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: scylladb - labels: - app: scylladb -spec: - ports: - - port: 9042 - name: cql - clusterIP: None - selector: - app: scylladb - ---- - -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: scylladb -spec: - serviceName: "scylladb" - replicas: 1 - selector: - matchLabels: - app: scylladb - template: - metadata: - labels: - app: scylladb - spec: - # If you're not on Linux, Docker will spin up a Linux VM to run Linux containers - # All containers (regardless of what clusters they're in) will share the same Linux VM kernel - # Because of that, when we're running with kind clusters, we'll need to alter - # aio-max-nr to make sure it can handle multiple scyllaDB instances (for multiple Validators - # running locally in parallel) - # Each ScyllaDB instance seems to request 510260 of AIO, so setting this to 8 * 510260 - # so we have some buffer - initContainers: - {{- if eq .Values.environment "kind" }} - - name: increase-aio-max-nr - image: busybox - command: ['sh', '-c', 'echo 4082080 > /proc/sys/fs/aio-max-nr'] - securityContext: - privileged: true - {{- end }} - containers: - - name: scylladb - image: scylladb/scylla:5.2 - ports: - - containerPort: 9042 - name: cql - volumeMounts: - - name: scylladb-data - mountPath: /var/lib/scylla - volumes: - - name: scylladb-config - emptyDir: {} - - volumeClaimTemplates: - - metadata: - name: scylladb-data - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: "standard" - resources: - requests: - storage: 10Gi diff --git a/kubernetes/linera-validator/values-local.yaml b/kubernetes/linera-validator/values-local.yaml index 9ca3bd5b37a..5e59cb47502 100644 --- a/kubernetes/linera-validator/values-local.yaml +++ b/kubernetes/linera-validator/values-local.yaml @@ -48,5 +48,5 @@ environment: "kind" # Validator validator: - serverConfig: "" # Needs to be set by the CLI `--set validator.serverConfig=server_1.json` - genesisConfig: "" # Needs to be set by the CLI `--set validator.genesisConfig=genesis.json` + serverConfig: "" # Is set by helmfile. + genesisConfig: "" # Is set by helmfile. diff --git a/linera-service/Cargo.toml b/linera-service/Cargo.toml index a784b36a1ba..c2c15833269 100644 --- a/linera-service/Cargo.toml +++ b/linera-service/Cargo.toml @@ -19,7 +19,7 @@ wasmtime = ["linera-execution/wasmtime", "linera-storage/wasmtime"] rocksdb = ["linera-views/rocksdb", "linera-core/rocksdb", "linera-storage/rocksdb"] aws = ["linera-views/aws", "linera-core/aws", "linera-storage/aws"] scylladb = ["linera-views/scylladb", "linera-core/scylladb", "linera-storage/scylladb"] -kubernetes = ["dep:k8s-openapi", "dep:kube", "dep:pathdiff"] +kubernetes = ["dep:k8s-openapi", "dep:kube", "dep:pathdiff", "dep:fs_extra"] [dependencies] anyhow = { workspace = true } @@ -38,6 +38,7 @@ current_platform = "0.2.0" dirs = { workspace = true } file-lock = "2.1.10" fs-err = { workspace = true } +fs_extra = { workspace = true, optional = true } fungible = { workspace = true, optional = true } futures = { workspace = true } hex = { workspace = true } diff --git a/linera-service/src/cli_wrappers/helm.rs b/linera-service/src/cli_wrappers/helm.rs deleted file mode 100644 index fa8e3df25e2..00000000000 --- a/linera-service/src/cli_wrappers/helm.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) Zefchain Labs, Inc. -// SPDX-License-Identifier: Apache-2.0 - -use anyhow::{Context, Result}; -use pathdiff::diff_paths; -use std::path::{Path, PathBuf}; -use tokio::process::Command; - -use crate::util::CommandExt; - -pub struct HelmRelease; - -impl HelmRelease { - pub async fn install( - name: String, - configs_dir: &PathBuf, - server_config_id: usize, - github_root: &Path, - num_shards: usize, - cluster_id: u32, - ) -> Result<()> { - let execution_dir = format!("{}/kubernetes/linera-validator", github_root.display()); - - let configs_dir = diff_paths(configs_dir, execution_dir.clone()) - .context("Getting relative path failed")?; - let configs_dir = configs_dir.to_str().expect("Getting str failed"); - - Command::new("helm") - .current_dir(&execution_dir) - .arg("install") - .arg(&name) - .arg(".") - .args(["--values", "values-local.yaml"]) - .arg("--wait") - .args(["--set", "installCRDs=true"]) - .args([ - "--set", - &format!("validator.serverConfig={configs_dir}/server_{server_config_id}.json"), - ]) - .args([ - "--set", - &format!("validator.genesisConfig={configs_dir}/genesis.json"), - ]) - .args(["--set", &format!("numShards={num_shards}")]) - .args(["--kube-context", &format!("kind-{}", cluster_id)]) - .args(["--timeout", "10m"]) - .spawn_and_wait() - .await - } -} diff --git a/linera-service/src/cli_wrappers/helmfile.rs b/linera-service/src/cli_wrappers/helmfile.rs new file mode 100644 index 00000000000..554e3419cdb --- /dev/null +++ b/linera-service/src/cli_wrappers/helmfile.rs @@ -0,0 +1,37 @@ +// Copyright (c) Zefchain Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use crate::util::CommandExt; +use anyhow::Result; +use fs_extra::dir::CopyOptions; +use std::path::Path; +use tokio::process::Command; + +pub struct HelmFile; + +impl HelmFile { + pub async fn sync( + server_config_id: usize, + github_root: &Path, + num_shards: usize, + cluster_id: u32, + ) -> Result<()> { + let chart_dir = format!("{}/kubernetes/linera-validator", github_root.display()); + + let temp_dir = tempfile::tempdir()?; + fs_extra::copy_items(&[&chart_dir], temp_dir.path(), &CopyOptions::new())?; + + Command::new("helmfile") + .current_dir(&temp_dir.path().join("linera-validator")) + .env( + "LINERA_HELMFILE_SET_SERVER_CONFIG", + &format!("working/server_{server_config_id}.json"), + ) + .env("LINERA_HELMFILE_SET_NUM_SHARDS", num_shards.to_string()) + .arg("sync") + .arg("--wait") + .args(["--kube-context", &format!("kind-{}", cluster_id)]) + .spawn_and_wait() + .await + } +} diff --git a/linera-service/src/cli_wrappers/local_kubernetes_net.rs b/linera-service/src/cli_wrappers/local_kubernetes_net.rs index a89f4b1a440..d5c709cb51b 100644 --- a/linera-service/src/cli_wrappers/local_kubernetes_net.rs +++ b/linera-service/src/cli_wrappers/local_kubernetes_net.rs @@ -2,11 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 use super::{ - docker::DockerImage, helm::HelmRelease, kind::KindCluster, kubectl::KubectlInstance, - util::get_github_root, + docker::DockerImage, kind::KindCluster, kubectl::KubectlInstance, util::get_github_root, }; use crate::{ - cli_wrappers::{ClientWrapper, LineraNet, LineraNetConfig, Network}, + cli_wrappers::{helmfile::HelmFile, ClientWrapper, LineraNet, LineraNetConfig, Network}, util::{self, CommandExt}, }; use anyhow::{anyhow, bail, ensure, Result}; @@ -22,7 +21,7 @@ use std::{path::PathBuf, sync::Arc}; use tempfile::{tempdir, TempDir}; use tokio::process::Command; #[cfg(any(test, feature = "test"))] -use tokio::sync::OnceCell; +use {crate::util::current_binary_parent, tokio::sync::OnceCell}; #[cfg(any(test, feature = "test"))] static SHARED_LOCAL_KUBERNETES_TESTING_NET: OnceCell<( @@ -63,7 +62,29 @@ pub struct LocalKubernetesNet { #[cfg(any(test, feature = "test"))] impl SharedLocalKubernetesNetTestingConfig { pub fn new(network: Network, binaries: Option>) -> Self { - Self { network, binaries } + if binaries.is_none() { + // For cargo test, current binary should be in debug mode + let current_binary_parent = + current_binary_parent().expect("Fetching current binaries path should not fail"); + // But binaries for cluster should be release mode + let binaries_dir = current_binary_parent + .parent() + .expect("Getting parent shuold not fail") + .join("release"); + if binaries_dir.exists() { + // If release exists, use those binaries + Self { + network, + binaries: Some(Some(binaries_dir)), + } + } else { + // If release doesn't exist, pass None to build binaries + // from within Docker container + Self { network, binaries } + } + } else { + Self { network, binaries } + } } } @@ -409,15 +430,7 @@ impl LocalKubernetesNet { base_dir.join(&server_config_filename), )?; - HelmRelease::install( - String::from("linera-core"), - &base_dir, - i, - &github_root, - num_shards, - cluster_id, - ) - .await?; + HelmFile::sync(i, &github_root, num_shards, cluster_id).await?; let mut kubectl_instance = kubectl_instance.lock().await; let output = kubectl_instance.get_pods(cluster_id).await?; diff --git a/linera-service/src/cli_wrappers/mod.rs b/linera-service/src/cli_wrappers/mod.rs index bd3f943dbe3..d1754e97582 100644 --- a/linera-service/src/cli_wrappers/mod.rs +++ b/linera-service/src/cli_wrappers/mod.rs @@ -7,9 +7,10 @@ #[cfg(feature = "kubernetes")] /// How to run docker operations mod docker; + #[cfg(feature = "kubernetes")] -/// How to run helm operations -mod helm; +/// How to run helmfile operations +mod helmfile; #[cfg(feature = "kubernetes")] /// How to run kind operations mod kind;