Initial Helm Chart for all zk_stack resources (#21)

* Add initial Helm chart with services * Add ingresses to helm chart * Add missing services, healthcheck and reuse port values * Disable rate limit and remove proxy from RPC URLs * Fix external-secrets version syntax * Add Cloudflare DNS01 issuer and use it * Use full price GPU nodes for instant availability * Change location of GPU nodes * [WIP] GPU Nodepool with 4 NVIDIA L4 GPUs * Upgrade kubernetes version * Change GPU location and name * Comment x4 GPUs nodes * Remove deletion protections
adi-foundation · Sep 26, 2024 · e75d3bb · e75d3bb
1 parent 5cffd53
commit e75d3bb
Show file tree

Hide file tree

Showing 24 changed files with 365 additions and 42 deletions.
diff --git a/infra/helm-charts/zk_stack/.helmignore b/infra/helm-charts/zk_stack/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/infra/helm-charts/zk_stack/Chart.yaml b/infra/helm-charts/zk_stack/Chart.yaml
@@ -0,0 +1,15 @@
+apiVersion: v2
+name: zk_stack
+description: A Helm chart that deploys all the components of the zk_stack
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"
diff --git a/infra/helm-charts/zk_stack/templates/backendconfig.yaml b/infra/helm-charts/zk_stack/templates/backendconfig.yaml
@@ -0,0 +1,11 @@
+apiVersion: cloud.google.com/v1
+kind: BackendConfig
+metadata:
+  name: health-check-config
+  namespace: default
+spec:
+  healthCheck:
+    checkIntervalSec: {{ .Values.healthCheck.checkIntervalSec }}
+    port: {{ .Values.healthCheck.port }}
+    type: HTTP
+    requestPath: /health
diff --git a/infra/helm-charts/zk_stack/templates/ingress.yaml b/infra/helm-charts/zk_stack/templates/ingress.yaml
@@ -0,0 +1,30 @@
+{{- range $app, $values := .Values.ingresses }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ $app }}-ingress
+  namespace: {{ $.Values.namespace }}
+  annotations:
+    kubernetes.io/ingress.global-static-ip-name: {{ $values.gcp_global_ip_name }}
+    cert-manager.io/issuer: "dns01-cloudflare-issuer"
+    acme.cert-manager.io/http01-edit-in-place: "true"
+    networking.gke.io/v1beta1.FrontendConfig: "http-https-redirect"
+  labels:
+    app: {{ $app }}
+spec:
+  rules:
+    - host: {{ $values.dns }}
+      http:
+        paths:
+        - pathType: ImplementationSpecific
+          backend:
+            service:
+              name: {{ $app }}
+              port:
+                number: {{ $values.port }}
+  tls:
+    - hosts:
+      - {{ $values.dns }}
+      secretName: {{ $app }}-ssl-cert
+---
+{{- end}}
diff --git a/infra/helm-charts/zk_stack/templates/service.yaml b/infra/helm-charts/zk_stack/templates/service.yaml
@@ -0,0 +1,21 @@
+{{- range $app, $values := .Values.services }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $app }}-service
+  namespace: {{ $.Values.namespace }}
+  labels:
+    app: {{ $app }}
+spec:
+  type: {{ $values.type }}
+  ports:
+    {{- range $values.ports }}
+    - name: {{ .name }}
+      port: {{ .port }}
+      targetPort: {{ .port }}
+      protocol: TCP
+    {{- end }}
+  selector:
+    app: {{ $app }}
+---
+{{- end }}
diff --git a/infra/helm-charts/zk_stack/values.yaml b/infra/helm-charts/zk_stack/values.yaml
@@ -0,0 +1,138 @@
+# Default values for zk_stack.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+namespace: default
+explorer-app:
+  http-port: &explorer-app-http-port 3010
+explorer-api:
+  http-port: &explorer-api-http-port 3020
+explorer-data-fetcher:
+  http-port: &explorer-data-fetcher-http-port 3040
+explorer-data-fetcher:
+  http-port: &explorer-data-fetcher-http-port 3040
+external-node:
+  http-port: &external-node-http-port 3050
+portal:
+  http-port: &portal-http-port 3000
+server:
+  http-port: &server-http-port 3050
+grafana:
+  http-port: &grafana-http-port 3000
+healthCheck:
+  port: &healthcheck-port 3071
+  checkIntervalSec: 15
+services:
+  explorer-app:
+      type: NodePort
+      ports:
+        - name: http
+          port: *explorer-app-http-port
+  explorer-api:
+      type: NodePort
+      ports:
+        - name: http
+          port: *explorer-api-http-port
+  explorer-data-fetcher:
+      type: ClusterIP
+      ports:
+        - name: http
+          port: *explorer-data-fetcher-http-port
+  external-node:
+      type: NodePort
+      ports:
+        - name: en-http
+          port: *external-node-http-port
+        - name: en-healthcheck
+          port: *healthcheck-port
+        - name: en-prometheu0
+          port: 3312
+  portal:
+      type: NodePort
+      ports:
+        - name: http
+          port: *portal-http-port
+  server:
+      type: NodePort
+      ports:
+        - name: server-rpc
+          port: *server-http-port
+        - name: server-proof-generation
+          port: 3320
+        - name: server-readiness
+          port: *healthcheck-port
+        - name: server-prometheus-listener
+          port: 3312
+  grafana:
+      type: NodePort
+      ports:
+        - name: grafana
+          port: *grafana-http-port
+  prometheus:
+      type: NodePort
+      ports:
+        - name: prometheus
+          port: 9090
+  pushgateway:
+      type: NodePort
+      ports:
+        - name: pushgateway
+          port: 9091
+  nodeexporter:
+      type: NodePort
+      ports:
+        - name: nodeexporter
+          port: 9091
+  alertmanager:
+      type: NodePort
+      ports:
+        - name: alertmanager
+          port: 9093
+  en01-grafana:
+      type: NodePort
+      ports:
+        - name: en01-grafana
+          port: *grafana-http-port
+  en01-prometheus:
+      type: NodePort
+      ports:
+        - name: en01-prometheus
+          port: 9090
+  en01-pushgateway:
+      type: NodePort
+      ports:
+        - name: en01-pushgateway
+          port: 9091
+  en01-nodeexporter:
+      type: NodePort
+      ports:
+        - name: en01-nodeexporter
+          port: 9100
+ingresses:
+  explorer-app:
+    dns: k8s.explorer.sepolia.zk-stack.lambdaclass.com
+    gcp_global_ip_name: explorer-app-ip
+    port: *explorer-app-http-port
+  explorer-api:
+    dns: k8s.explorer.api.sepolia.zk-stack.lambdaclass.com
+    gcp_global_ip_name: explorer-api-ip
+    port: *explorer-api-http-port
+  external-node:
+    dns: k8s.en01.rpc.sepolia.zk-stack.lambdaclass.com
+    gcp_global_ip_name: external-node-ip
+    port: *external-node-http-port
+  portal:
+    dns: k8s.portal.sepolia.zk-stack.lambdaclass.com
+    gcp_global_ip_name: portal-ip
+    port: *portal-http-port
+  server:
+    dns: k8s.rpc.sepolia.zk-stack.lambdaclass.com
+    gcp_global_ip_name: server-ip
+    port: *server-http-port
+  grafana:
+    dns: k8s.grafana.sepolia.zk-stack.lambdaclass.com
+    gcp_global_ip_name: grafana-ip
+    port: *grafana-http-port
+  en01-grafana:
+    dns: k8s.en01.grafana.sepolia.zk-stack.lambdaclass.com
+    gcp_global_ip_name: en01-grafana-ip
+    port: *grafana-http-port
diff --git a/infra/kubernetes/ingress/en-grafana.yaml b/infra/kubernetes/ingress/en-grafana.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: default
   annotations:
     kubernetes.io/ingress.global-static-ip-name: en01-grafana-ip
-    cert-manager.io/issuer: http01-issuer
+    cert-manager.io/issuer: dns01-cloudflare-issuer
     acme.cert-manager.io/http01-edit-in-place: "true"
     networking.gke.io/v1beta1.FrontendConfig: "http-https-redirect"
   labels:

diff --git a/infra/kubernetes/ingress/explorer-api.yaml b/infra/kubernetes/ingress/explorer-api.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: default
   annotations:
     kubernetes.io/ingress.global-static-ip-name: explorer-api-ip
-    cert-manager.io/issuer: http01-issuer
+    cert-manager.io/issuer: dns01-cloudflare-issuer
     acme.cert-manager.io/http01-edit-in-place: "true"
     networking.gke.io/v1beta1.FrontendConfig: "http-https-redirect"
   labels:

diff --git a/infra/kubernetes/ingress/explorer-app.yaml b/infra/kubernetes/ingress/explorer-app.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: default
   annotations:
     kubernetes.io/ingress.global-static-ip-name: explorer-app-ip
-    cert-manager.io/issuer: http01-issuer
+    cert-manager.io/issuer: dns01-cloudflare-issuer
     acme.cert-manager.io/http01-edit-in-place: "true"
     networking.gke.io/v1beta1.FrontendConfig: "http-https-redirect"
   labels:

diff --git a/infra/kubernetes/ingress/external-node.yaml b/infra/kubernetes/ingress/external-node.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: default
   annotations:
     kubernetes.io/ingress.global-static-ip-name: external-node-ip
-    cert-manager.io/issuer: http01-issuer
+    cert-manager.io/issuer: dns01-cloudflare-issuer
     acme.cert-manager.io/http01-edit-in-place: "true"
     networking.gke.io/v1beta1.FrontendConfig: "http-https-redirect"
   labels:

diff --git a/infra/kubernetes/ingress/grafana.yaml b/infra/kubernetes/ingress/grafana.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: default
   annotations:
     kubernetes.io/ingress.global-static-ip-name: grafana-ip
-    cert-manager.io/issuer: http01-issuer
+    cert-manager.io/issuer: dns01-cloudflare-issuer
     acme.cert-manager.io/http01-edit-in-place: "true"
     networking.gke.io/v1beta1.FrontendConfig: "http-https-redirect"
   labels:

diff --git a/infra/kubernetes/ingress/portal.yaml b/infra/kubernetes/ingress/portal.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: default
   annotations:
     kubernetes.io/ingress.global-static-ip-name: portal-ip
-    cert-manager.io/issuer: http01-issuer
+    cert-manager.io/issuer: dns01-cloudflare-issuer
     acme.cert-manager.io/http01-edit-in-place: "true"
     networking.gke.io/v1beta1.FrontendConfig: "http-https-redirect"
   labels:

diff --git a/infra/kubernetes/ingress/server.yaml b/infra/kubernetes/ingress/server.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: default
   annotations:
     kubernetes.io/ingress.global-static-ip-name: server-ip
-    cert-manager.io/issuer: http01-issuer
+    cert-manager.io/issuer: dns01-cloudflare-issuer
     acme.cert-manager.io/http01-edit-in-place: "true"
     networking.gke.io/v1beta1.FrontendConfig: "http-https-redirect"
   labels:

diff --git a/infra/terraform/dev/us-central1/db/zksync_dev_01.tf b/infra/terraform/dev/us-central1/db/zksync_dev_01.tf
@@ -2,6 +2,7 @@ resource "google_sql_database_instance" "zksync_dev_01" {
   name             = "zksync-dev-01"
   database_version = "POSTGRES_14"
   region           = "us-east1"
+  deletion_protection = false
 
   settings {
     tier              = "db-custom-4-15360"

diff --git a/infra/terraform/dev/us-central1/db/zksync_dev_prover_01.tf b/infra/terraform/dev/us-central1/db/zksync_dev_prover_01.tf
@@ -2,6 +2,7 @@ resource "google_sql_database_instance" "zksync_dev_prover_01" {
   name             = "zksync-dev-prover-01"
   database_version = "POSTGRES_14"
   region           = "us-central1"
+  deletion_protection = false
 
   settings {
     tier              = "db-custom-4-15360"

diff --git a/infra/terraform/dev/us-central1/db/zksync_en_dev_01.tf b/infra/terraform/dev/us-central1/db/zksync_en_dev_01.tf
@@ -2,6 +2,7 @@ resource "google_sql_database_instance" "zksync_en_dev_01" {
   name             = "zksync-en-dev-01"
   database_version = "POSTGRES_14"
   region           = "us-east1"
+  deletion_protection = false
 
   settings {
     tier              = "db-custom-4-15360"

diff --git a/infra/terraform/dev/us-central1/gke/buckets.tf b/infra/terraform/dev/us-central1/gke/buckets.tf
@@ -6,13 +6,17 @@ resource "google_storage_bucket" "object-store-dev" {
   uniform_bucket_level_access = true
 
   public_access_prevention = "enforced"
+  # Allow Terraform to delete all objects in the bucket
+  force_destroy = true
 }
 
 resource "google_storage_bucket" "public-object-store-dev" {
   name          = "public-object-store-dev"
   location      = "US-CENTRAL1"
 
   uniform_bucket_level_access = true
+  # Allow Terraform to delete all objects in the bucket
+  force_destroy = true
 }
 
 resource "google_storage_bucket" "prover-object-store-dev" {
@@ -22,20 +26,26 @@ resource "google_storage_bucket" "prover-object-store-dev" {
   uniform_bucket_level_access = true
 
   public_access_prevention = "enforced"
+  # Allow Terraform to delete all objects in the bucket
+  force_destroy = true
 }
 
 resource "google_storage_bucket" "snapshots-object-store-dev" {
   name          = "snapshots-object-store-dev"
   location      = "US-CENTRAL1"
 
   uniform_bucket_level_access = true
+  # Allow Terraform to delete all objects in the bucket
+  force_destroy = true
 }
 
 resource "google_storage_bucket" "prover-setup-data" {
   name          = "prover-setup-data"
   location      = "US-CENTRAL1"
 
   uniform_bucket_level_access = true
+  # Allow Terraform to delete all objects in the bucket
+  force_destroy = true
 }
 
 # Public read access