ci: separate deploy logic into charts

deployments/README.md

@@ -16,7 +16,7 @@ The networks are completely separate.
 ```
 .
 ├── ci.sh # runner script for executing a deploy against k8s
-├── charts/ # helm charts used to configure full-node/validator layout
+├── charts/ # helm charts used to configure genesis, nodes, and metrics
 ├── networks/ # logic specific to network, e.g. "testnet" or "testnet-preview"
 │  └── testnet/
 └── terraform/ # server and cluster provisioning logic
@@ -66,6 +66,25 @@ kubectl get svc -n infra traefik --output jsonpath='{.status.loadBalancer.ingres
 The Traefik reverse proxy is used for fronting pd's grpc service, because Traefik supports h2c.
 See for details: https://github.com/penumbra-zone/penumbra/issues/2341
 
+## Generating and storing public IPs for P2P connections
+
+There's a chicken-or-egg problem when creating a new network: the deployment will trigger the creation
+of LoadBalancer objects with public IPv4 addresses. Those public IP addresses are needed at genesis
+creation time, so that the validator configs are generated with an external address field
+in the Tendermint configs. To resolve, there's a special var `only_lb_svc=true` that will
+deploy just the P2P LBs. You can then poll the IPs, store them as additional vars, and rerun
+with `only_lb_svc=false`.
+
+```
+helmfile sync -f helmfile.d/penumbra-devnet.yaml --args --set=only_lb_svc=true
+./scripts/get-lb-ips penumbra-devnet
+helmfile sync -f helmfile.d/penumbra-devnet.yaml
+```
+
+This two-step process is only required the *first* time a given network is deployed.
+Thereafter, resource retention policies will preserve the LBs, so that the IPs remain reserved,
+and can be reused on subsequent deployments of that network.
+
 ## Dude, where's my logs?
 
 There's web-based access for viewing logs from the testnet deployment:

deployments/charts/penumbra-metrics/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployments/charts/penumbra/Chart.yaml → deployments/charts/penumbra-metrics/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
-name: penumbra
-description: Run Penumbra nodes and/or validators
+name: penumbra-metrics
+description: Deploy Grafana and Prometheus for monitoring Penumbra deployments.
 
 # A chart can be either an 'application' or a 'library' chart.
 #
@@ -21,8 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-
-# Pre-mainnet, we set this to a tag name for the given testnet, e.g. '036-iocaste.2'.
-# Container images are automatically tagged for each testnet; we can also use "latest"
-# for the HEAD of main branch in the penumbra git repository.
-appVersion: "037-megaclite.1"
+appVersion: "1.16.0"

deployments/charts/penumbra-metrics/templates/NOTES.txt

@@ -0,0 +1,11 @@
+Penumbra metrics dashabords have been deployed.
+{{- if .Values.ingressRoute.enabled }}
+You can access the Grafana dashboard here:
+
+  https://{{.Values.ingressRoute.hosts.grafana }}
+
+You can view Prometheus dashboard by using `kubectl port-forward`.
+{{- else }}
+You have not enabled an ingress (via `ingressRoute.enabled=true`),
+so you'll need to forward ports via kubectl to access Grafana.
+{{- end }}

deployments/charts/penumbra-metrics/templates/_helpers.tpl

@@ -0,0 +1,65 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "penumbra-metrics.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "penumbra-metrics.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "penumbra-metrics.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "penumbra-metrics.labels" -}}
+helm.sh/chart: {{ include "penumbra-metrics.chart" . }}
+{{ include "penumbra-metrics.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+{{- if .Values.part_of }}
+app.kubernetes.io/part-of: {{ .Values.part_of }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "penumbra-metrics.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "penumbra-metrics.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "penumbra-metrics.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "penumbra-metrics.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

deployments/charts/penumbra/templates/metrics-config.yaml → deployments/charts/penumbra-metrics/templates/configmap.yaml

@@ -1,10 +1,9 @@
-{{ $depl_name := printf "%s-metrics" (include "penumbra.fullname" .) }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: config-{{ $depl_name }}
+  name: config-{{ include "penumbra-metrics.fullname" . }}
   labels:
-    {{- include "penumbra.labels" . | nindent 4 }}
+    {{- include "penumbra-metrics.labels" . | nindent 4 }}
 data:
   grafana_datasources.yml: |
     datasources:
@@ -16,39 +15,37 @@ data:
       editable: false
   prometheus_scrape_configs: |
     scrape_configs:
-{{ $count := (.Values.numFullNodes | int) }}
+{{ $count := (.Values.scrape_configs.numFullNodes | int) }}
 {{ range $i,$e := until $count }}
-{{ $fn_name := printf "%s-fn-%d" (include "penumbra.fullname" $) $i }}
+{{ $fn_name := printf $.Values.scrape_configs.fmtFullNodeSvc $i }}
+
       - job_name: 'Tendermint Fullnode {{ $i }}'
         scrape_interval: 10s
         scheme: http
         metrics_path: metrics
         static_configs:
-          - targets: ['{{ $fn_name }}-metrics:26660']
-{{/*
-N.B. `pd` doesn't actually expose metrics yet, so this doesn't actually do
-anything interesting...
-*/}}
+          - targets: ['{{ $fn_name }}:26660']
+
       - job_name: 'Penumbra Daemon Fullnode {{ $i }}'
         scrape_interval: 10s
         scheme: http
         metrics_path: metrics
         static_configs:
-          - targets: ['{{ $fn_name }}-metrics:9000']
+          - targets: ['{{ $fn_name }}:9000']
 {{ end }}
-{{ $count := (.Values.numValidators | int) }}
+{{ $count := (.Values.scrape_configs.numValidators | int) }}
 {{ range $i,$e := until $count }}
-{{ $val_name := printf "%s-val-%d" (include "penumbra.fullname" $) $i }}
+{{ $val_name := printf $.Values.scrape_configs.fmtValidatorSvc $i }}
       - job_name: 'Tendermint Validator {{ $i }}'
         scrape_interval: 10s
         scheme: http
         metrics_path: metrics
         static_configs:
-          - targets: ['{{ $val_name }}-metrics:26660']
+          - targets: ['{{ $val_name }}:26660']
       - job_name: 'Penumbra Daemon Validator {{ $i }}'
         scrape_interval: 10s
         scheme: http
         metrics_path: metrics
         static_configs:
-          - targets: ['{{ $val_name }}-metrics:9000']
+          - targets: ['{{ $val_name }}:9000']
 {{ end }}

deployments/charts/penumbra-metrics/templates/deployment.yaml

@@ -0,0 +1,117 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}
+  labels:
+    app.kubernetes.io/component: metrics
+    {{- include "penumbra-metrics.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "penumbra-metrics.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "penumbra-metrics.selectorLabels" . | nindent 8 }}
+    spec:
+      volumes:
+        - name: grafana-datasources
+          configMap:
+            name: config-{{ include "penumbra-metrics.fullname" . }}
+            items:
+              - key: "grafana_datasources.yml"
+                path: "all.yml"
+        - name: prometheus-scrape-configs
+          configMap:
+            name: config-{{ include "penumbra-metrics.fullname" . }}
+            items:
+              - key: "prometheus_scrape_configs"
+                path: "prometheus.yml"
+        - name: prometheus-data
+      {{- if .Values.persistence.enabled }}
+          persistentVolumeClaim:
+            claimName: {{ include "penumbra-metrics.fullname" . }}
+      {{- else }}
+          emptyDir: {}
+      {{- end }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "penumbra-metrics.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      initContainers:
+        - name: prom-config
+          image: "docker.io/debian:stable"
+          command:
+            - sh
+            - -cex
+            - >
+              id &&
+              cat /etc/prometheus/prometheus.yml &&
+              chown -R 65534:65534 /prometheus
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: prometheus-scrape-configs
+              mountPath: "/etc/prometheus/"
+              readOnly: true
+            - name: prometheus-data
+              mountPath: "/prometheus"
+      containers:
+        - name: grafana
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - containerPort: 3000
+              protocol: TCP
+              name: grafana-http
+          volumeMounts:
+            - name: grafana-datasources
+              mountPath: "/etc/grafana/provisioning/datasources/"
+              readOnly: true
+          readinessProbe:
+            tcpSocket:
+              port: grafana-http
+            timeoutSeconds: 10
+            initialDelaySeconds: 2
+        - name: prometheus
+          image: "{{ .Values.prometheus.repository }}:{{ .Values.prometheus.tag }}"
+          args:
+            - "--config.file=/etc/prometheus/prometheus.yml"
+            - "--log.level=debug"
+          imagePullPolicy: {{ .Values.prometheus.pullPolicy }}
+          ports:
+            - containerPort: 9090
+              protocol: TCP
+              name: prometheus-http
+          readinessProbe:
+            tcpSocket:
+              port: prometheus-http
+            timeoutSeconds: 10
+            initialDelaySeconds: 2
+          volumeMounts:
+            - name: prometheus-scrape-configs
+              mountPath: "/etc/prometheus/"
+              readOnly: true
+            - name: prometheus-data
+              mountPath: "/prometheus"
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

deployments/charts/penumbra-metrics/templates/ingress.yaml

@@ -0,0 +1,62 @@
+{{- if .Values.ingress.enabled -}}
+{{- $fullName := include "penumbra-node.fullname" . -}}
+{{- $svcPort := .Values.service.port -}}
+{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
+  {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
+  {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
+  {{- end }}
+{{- end }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  labels:
+    {{- include "penumbra-node.labels" . | nindent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- end }}
+  {{- if .Values.ingress.tls }}
+  tls:
+    {{- range .Values.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
+            pathType: {{ .pathType }}
+            {{- end }}
+            backend:
+              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+              service:
+                name: {{ $fullName }}-tendermint
+                port:
+                  # number: {{ $svcPort }}
+                  number: 26657
+              {{- else }}
+              serviceName: {{ $fullName }}
+              servicePort: {{ $svcPort }}
+              {{- end }}
+          {{- end }}
+    {{- end }}
+{{- end }}