chore: NVME bootstrapping (#9)

* chore: NVME bootstrapping

* Better handling of tolerations

* One proxy per network

* Fix proxy per network approach

* Improve health check

bootstrap/cell/main.tf

@@ -0,0 +1,33 @@
+module "pvc" {
+  source = "../pvc"
+
+  namespace     = var.namespace
+  name          = "pvc-${var.salt}"
+  storage_size  = var.storage_size
+  storage_class = var.storage_class
+  volume_name   = var.volume_name
+}
+
+module "instances" {
+  for_each = var.instances
+  source   = "../instance"
+
+  namespace     = var.namespace
+  tolerations   = var.tolerations
+  salt          = var.salt
+  instance_name = each.key
+  network       = each.key
+  pvc_name      = "pvc-${var.salt}"
+  dolos_version = each.value.dolos_version
+  replicas      = coalesce(each.value.replicas, 1)
+  resources = coalesce(each.value.resources, {
+    requests = {
+      cpu    = "50m"
+      memory = "512Mi"
+    }
+    limits = {
+      cpu    = "1000m"
+      memory = "512Mi"
+    }
+  })
+}

bootstrap/cell/variables.tf

@@ -0,0 +1,74 @@
+variable "namespace" {
+  type = string
+}
+
+variable "salt" {
+  type = string
+}
+
+variable "extension_subdomain" {
+  type = string
+}
+
+variable "dns_zone" {
+  default = "demeter.run"
+}
+
+variable "storage_size" {
+  type = string
+}
+
+variable "storage_class" {
+  type = string
+}
+
+variable "volume_name" {
+  type = string
+}
+
+variable "tolerations" {
+  type = list(object({
+    effect   = string
+    key      = string
+    operator = string
+    value    = string
+  }))
+  default = [
+    {
+      effect   = "NoSchedule"
+      key      = "demeter.run/compute-profile"
+      operator = "Equal"
+      value    = "disk-intensive"
+    },
+    {
+      effect   = "NoSchedule"
+      key      = "demeter.run/compute-arch"
+      operator = "Equal"
+      value    = "arm64"
+    },
+    {
+      effect   = "NoSchedule"
+      key      = "demeter.run/availability-sla"
+      operator = "Equal"
+      value    = "consistent"
+    }
+  ]
+}
+
+// Instances
+variable "instances" {
+  type = map(object({
+    dolos_version = string
+    replicas      = optional(number)
+    resources = optional(object({
+      limits = object({
+        cpu    = string
+        memory = string
+      })
+      requests = object({
+        cpu    = string
+        memory = string
+      })
+    }))
+  }))
+}

bootstrap/cloudflared/config.tf

@@ -14,9 +14,9 @@ resource "kubernetes_config_map" "tunnel-config" {
       tunnel_id    = var.tunnel_id
       metrics_port = var.metrics_port
       hostname     = var.hostname
-      service      = "proxy"
       namespace    = var.namespace
       port         = 8080
+      networks     = var.networks
     })}"
   }
 }

bootstrap/cloudflared/config.yml.tfpl

@@ -15,9 +15,11 @@ no-autoupdate: true
 # from the internet to cloudflared, run `cloudflared tunnel route dns <tunnel> <hostname>`.
 # E.g. `cloudflared tunnel route dns example-tunnel tunnel.example.com`.
 ingress:
-  - hostname: ${hostname}
-    service: https://${service}.${namespace}.svc.cluster.local:${port}
+%{ for network in networks ~}
+  - hostname: ${network}-${hostname}
+    service: https://proxy-${network}.${namespace}.svc.cluster.local:${port}
     originRequest:
       noTLSVerify: true
       http2Origin: true
+%{ endfor ~}
   - service: http_status:404

bootstrap/cloudflared/deployment.tf

@@ -1,9 +1,13 @@
+locals {
+  name = "cloudflared"
+}
+
 resource "kubernetes_deployment" "cloudflared" {
   wait_for_rollout = false
   depends_on       = [kubernetes_secret.tunnel_credentials]
 
   metadata {
-    name      = "cloudflared"
+    name      = local.name
     namespace = var.namespace
 
     labels = {
@@ -102,23 +106,15 @@ resource "kubernetes_deployment" "cloudflared" {
           }
         }
 
-        toleration {
-          effect   = "NoSchedule"
-          key      = "demeter.run/compute-profile"
-          operator = "Exists"
-        }
-
-        toleration {
-          effect   = "NoSchedule"
-          key      = "demeter.run/compute-arch"
-          operator = "Exists"
-        }
+        dynamic "toleration" {
+          for_each = var.tolerations
 
-        toleration {
-          effect   = "NoSchedule"
-          key      = "demeter.run/availability-sla"
-          operator = "Equal"
-          value    = "consistent"
+          content {
+            effect   = toleration.value.effect
+            key      = toleration.value.key
+            operator = toleration.value.operator
+            value    = toleration.value.value
+          }
         }
       }
     }

bootstrap/cloudflared/main.tf

@@ -2,6 +2,10 @@ variable "namespace" {
   type = string
 }
 
+variable "networks" {
+  type = list(string)
+}
+
 variable "tunnel_id" {
   type = string
 }
@@ -25,6 +29,35 @@ variable "replicas" {
   default = 2
 }
 
+variable "tolerations" {
+  type = list(object({
+    effect   = string
+    key      = string
+    operator = string
+    value    = string
+  }))
+  default = [
+    {
+      effect   = "NoSchedule"
+      key      = "demeter.run/compute-profile"
+      operator = "Equal"
+      value    = "general-purpose"
+    },
+    {
+      effect   = "NoSchedule"
+      key      = "demeter.run/compute-arch"
+      operator = "Equal"
+      value    = "x86"
+    },
+    {
+      effect   = "NoSchedule"
+      key      = "demeter.run/availability-sla"
+      operator = "Equal"
+      value    = "best-effort"
+    }
+  ]
+}
+
 variable "resources" {
   type = object({
     limits = object({

bootstrap/configs/mainnet.toml

@@ -4,7 +4,7 @@ network_magic = 764824073
 is_testnet = false
 
 [storage]
-path = "/var/data/db"
+path = "/var/data/mainnet/db"
 
 [genesis]
 byron_path = "/etc/genesis/mainnet/byron.json"

bootstrap/configs/preprod.toml

@@ -4,7 +4,7 @@ network_magic = 1
 is_testnet = true
 
 [storage]
-path = "/var/data/db"
+path = "/var/data/preprod/db"
 
 [genesis]
 byron_path = "/etc/genesis/preprod/byron.json"

bootstrap/configs/preview.toml

@@ -4,7 +4,7 @@ network_magic = 2
 is_testnet = true
 
 [storage]
-path = "/var/data/db"
+path = "/var/data/preview/db"
 
 [genesis]
 byron_path = "/etc/genesis/preview/byron.json"

bootstrap/configs/vector-testnet.toml

@@ -5,11 +5,11 @@ network_id = 1
 phase1_validation_enabled = false
 
 [rolldb]
-path = "/var/data/rolldb"
+path = "/var/data/vector-testnet/rolldb"
 k_param = 1000
 
 [applydb]
-path = "/var/data/applydb"
+path = "/var/data/vector-testnet/applydb"
 
 
 [serve.grpc]

bootstrap/proxy/certs.tf → bootstrap/feature/certs.tf

@@ -1,7 +1,7 @@
 resource "kubernetes_config_map" "proxy-certs" {
   metadata {
     namespace = var.namespace
-    name      = local.certs_configmap
+    name      = var.certs_configmap
   }
 
   data = {

bootstrap/feature/main.tf

@@ -15,6 +15,11 @@ variable "extension_subdomain" {
   default = "utxorpc-m0"
 }
 
+variable "certs_configmap" {
+  type    = string
+  default = "proxy-certs"
+}
+
 variable "dns_zone" {
   default = "demeter.run"
 }

bootstrap/feature/operator.tf

@@ -80,14 +80,14 @@ resource "kubernetes_deployment_v1" "operator" {
         toleration {
           effect   = "NoSchedule"
           key      = "demeter.run/compute-profile"
-          operator = "Equal"
-          value    = "general-purpose"
+          operator = "Exists"
         }
 
         toleration {
           effect   = "NoSchedule"
           key      = "demeter.run/compute-arch"
-          operator = "Exists"
+          operator = "Equal"
+          value    = "x86"
         }
 
         toleration {