Move to Helmfile (linera-io#1412)

* Added scylla-operator chart and helmfiles

* Replaced Helm wrapper with Helmfile wrapper

* Added a step in CI to install Helmfile

* Create new temporary directory for every Helmfile deployment

* Increase installations timeouts

* Fix CI

* Make sure CI uses already built binaries

* Fix Helmfile installation

* Removed create_cert script

---------

Co-authored-by: Andre da Silva <[email protected]>

.github/workflows/kubernetes.yml

@@ -46,6 +46,8 @@ jobs:
           strip target/release/linera-proxy
           strip target/release/linera-server
           strip target/release/linera-db
+      - name: Setup helmfile
+        uses: mamezou-tech/[email protected]
       - name: Run Wasm e2e test
         uses: nick-fields/retry@v2
         with:

Cargo.toml

@@ -61,6 +61,7 @@ ed25519-dalek = { version = "1.0.1", features = ["batch", "serde"] }
 either = "1.9.0"
 frunk = "0.4.2"
 fs-err = { version = "2.11.0", features = ["tokio"] }
+fs_extra = "1.3.0"
 futures = "0.3.29"
 generic-array = { version = "0.14.7", features = ["serde"] }
 hex = "0.4.3"

docker/server-entrypoint.sh

@@ -4,7 +4,7 @@
 ORDINAL="${HOSTNAME##*-}"
 
 exec ./linera-server run \
-  --storage scylladb:tcp:scylladb.default.svc.cluster.local:9042 \
+  --storage scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042 \
   --server /config/server.json \
   --shard $ORDINAL \
   --genesis /config/genesis.json

docker/server-init.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 while true; do
-  ./linera-db check_existence --storage "scylladb:tcp:scylladb.default.svc.cluster.local:9042"
+  ./linera-db check_existence --storage "scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042"
   status=$?
 
   if [ $status -eq 0 ]; then
@@ -10,7 +10,7 @@ while true; do
   elif [ $status -eq 1 ]; then
     echo "Database does not exist, attempting to initialize..."
     if ./linera-server initialize \
-      --storage scylladb:tcp:scylladb.default.svc.cluster.local:9042 \
+      --storage scylladb:tcp:scylla-client.scylla.svc.cluster.local:9042 \
       --genesis /config/genesis.json; then
       echo "Initialization successful."
       exit 0

kubernetes/linera-validator/Chart.lock

@@ -6,4 +6,4 @@ dependencies:
   repository: https://grafana.github.io/helm-charts
   version: 2.8.9
 digest: sha256:da3a8808fb9479bdd183b307e70d7855794739cafc1796047bde8febc78d539c
-generated: "2023-10-23T17:50:47.813398009+01:00"
+generated: "2023-11-30T17:19:14.544457977Z"

kubernetes/linera-validator/helmfile.yaml

@@ -0,0 +1,59 @@
+repositories:
+  - name: scylla
+    url: https://scylla-operator-charts.storage.googleapis.com/stable
+  - name: jetstack
+    url: https://charts.jetstack.io
+  - name: prometheus-community
+    url: https://prometheus-community.github.io/helm-charts
+  - name: grafana
+    url: https://grafana.github.io/helm-charts
+
+releases:
+  - name: linera-core
+    namespace: default
+    chart: .
+    timeout: 900
+    needs:
+      - scylla/scylla
+    values:
+      - {{ env "LINERA_HELMFILE_VALUES_LINERA_CORE" | default "values-local.yaml" }}
+    set:
+      - name: installCRDs
+        value: "true"
+      - name: validator.serverConfig
+        value: {{ env "LINERA_HELMFILE_SET_SERVER_CONFIG" | default "working/server_1.json" }}
+      - name: validator.genesisConfig
+        value: {{ env "LINERA_HELMFILE_SET_GENESIS_CONFIG" | default "working/genesis.json" }}
+      - name: numShards
+        value: {{ env "LINERA_HELMFILE_SET_NUM_SHARDS" | default 10 }}
+  - name: scylla
+    namespace: scylla
+    chart: scylla/scylla
+    timeout: 900
+    needs:
+      - scylla-manager/scylla-manager
+      - scylla-operator/scylla-operator
+    values:
+      - {{ env "LINERA_HELMFILE_VALUES_SCYLLA" | default "scylla.values.yaml" }}
+  - name: scylla-manager
+    namespace: scylla-manager
+    chart: scylla/scylla-manager
+    timeout: 900
+    needs:
+      - scylla-operator/scylla-operator
+  - name: scylla-operator
+    namespace: scylla-operator
+    chart: scylla/scylla-operator
+    timeout: 900
+    needs:
+      - cert-manager/cert-manager
+    values:
+      - {{ env "LINERA_HELMFILE_VALUES_SCYLLA_OPERATOR" | default "scylla-operator.values.yaml" }}
+  - name: cert-manager
+    namespace: cert-manager
+    chart: jetstack/cert-manager
+    timeout: 900
+    set:
+      - name: installCRDs
+        value: "true"
+

kubernetes/linera-validator/scylla-operator.values.yaml

@@ -0,0 +1,3 @@
+webhook:
+  createSelfSignedCertificate: true
+  certificateSecretName: ""

kubernetes/linera-validator/scylla.values.yaml

@@ -0,0 +1,16 @@
+developerMode: true
+sysctls:
+  - "fs.aio-max-nr=4082080"
+datacenter: validator
+racks:
+  - name: rack-1
+    members: 1
+    storage:
+      capacity: 1Gi
+    resources:
+      limits:
+        cpu: 1
+        memory: 0.2Gi
+      requests:
+        cpu: 1
+        memory: 0.2Gi

kubernetes/linera-validator/values-local.yaml

@@ -48,5 +48,5 @@ environment: "kind"
 
 # Validator
 validator:
-  serverConfig: "" # Needs to be set by the CLI `--set validator.serverConfig=server_1.json`
-  genesisConfig: "" # Needs to be set by the CLI `--set validator.genesisConfig=genesis.json`
+  serverConfig: "" # Is set by helmfile.
+  genesisConfig: "" # Is set by helmfile.

linera-service/Cargo.toml

@@ -19,7 +19,7 @@ wasmtime = ["linera-execution/wasmtime", "linera-storage/wasmtime"]
 rocksdb = ["linera-views/rocksdb", "linera-core/rocksdb", "linera-storage/rocksdb"]
 aws = ["linera-views/aws", "linera-core/aws", "linera-storage/aws"]
 scylladb = ["linera-views/scylladb", "linera-core/scylladb", "linera-storage/scylladb"]
-kubernetes = ["dep:k8s-openapi", "dep:kube", "dep:pathdiff"]
+kubernetes = ["dep:k8s-openapi", "dep:kube", "dep:pathdiff", "dep:fs_extra"]
 
 [dependencies]
 anyhow = { workspace = true }
@@ -38,6 +38,7 @@ current_platform = "0.2.0"
 dirs = { workspace = true }
 file-lock = "2.1.10"
 fs-err = { workspace = true }
+fs_extra = { workspace = true, optional = true }
 fungible = { workspace = true, optional = true }
 futures = { workspace = true }
 hex = { workspace = true }

linera-service/src/cli_wrappers/helmfile.rs

@@ -0,0 +1,37 @@
+// Copyright (c) Zefchain Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::util::CommandExt;
+use anyhow::Result;
+use fs_extra::dir::CopyOptions;
+use std::path::Path;
+use tokio::process::Command;
+
+pub struct HelmFile;
+
+impl HelmFile {
+    pub async fn sync(
+        server_config_id: usize,
+        github_root: &Path,
+        num_shards: usize,
+        cluster_id: u32,
+    ) -> Result<()> {
+        let chart_dir = format!("{}/kubernetes/linera-validator", github_root.display());
+
+        let temp_dir = tempfile::tempdir()?;
+        fs_extra::copy_items(&[&chart_dir], temp_dir.path(), &CopyOptions::new())?;
+
+        Command::new("helmfile")
+            .current_dir(&temp_dir.path().join("linera-validator"))
+            .env(
+                "LINERA_HELMFILE_SET_SERVER_CONFIG",
+                &format!("working/server_{server_config_id}.json"),
+            )
+            .env("LINERA_HELMFILE_SET_NUM_SHARDS", num_shards.to_string())
+            .arg("sync")
+            .arg("--wait")
+            .args(["--kube-context", &format!("kind-{}", cluster_id)])
+            .spawn_and_wait()
+            .await
+    }
+}