From 8c23ca98cc0a737212221b2be184f908bf7d5a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= Date: Fri, 1 Dec 2023 19:09:16 +0100 Subject: [PATCH 01/34] cleaning up all charts, updating versions --- charts/dirk/Chart.yaml | 18 -- charts/dirk/values.yaml | 4 +- charts/execution-beacon/Chart.yaml | 2 +- .../templates/statefulset.yaml | 4 - charts/mev-boost/Chart.yaml | 20 +- charts/posmoni/Chart.yaml | 20 +- charts/validator-ejector/Chart.yaml | 20 +- charts/validator-ejector/values.yaml | 2 +- charts/validator-kapi/Chart.yaml | 22 +- charts/validators/Chart.yaml | 22 +- charts/validators/templates/_helpers.tpl | 4 +- charts/validators/templates/configmap.yaml | 190 ------------------ charts/validators/templates/statefulset.yaml | 56 ++---- charts/validators/values.yaml | 107 +++++++--- charts/vouch/Chart.yaml | 18 -- charts/vouch/values.yaml | 4 +- charts/web3signer/Chart.yaml | 22 +- charts/web3signer/templates/statefulset.yaml | 4 +- charts/web3signer/values.yaml | 8 +- 19 files changed, 115 insertions(+), 432 deletions(-) diff --git a/charts/dirk/Chart.yaml b/charts/dirk/Chart.yaml index 2a6c577e2..27af54f87 100644 --- a/charts/dirk/Chart.yaml +++ b/charts/dirk/Chart.yaml @@ -1,26 +1,8 @@ apiVersion: v2 name: dirk description: A Helm chart for installing and configuring large scale ETH staking infrastructure on top of the Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) version: 1.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. appVersion: "v22.10.0" keywords: diff --git a/charts/dirk/values.yaml b/charts/dirk/values.yaml index 9d3de93e8..534429519 100644 --- a/charts/dirk/values.yaml +++ b/charts/dirk/values.yaml @@ -24,8 +24,8 @@ initImage: ## CLI image is used to fetch private keys. ## cliImage: - repository: 238155366538.dkr.ecr.eu-west-1.amazonaws.com/keystores-cli - tag: "dirk-0.0.16" + repository: nethermindeth/keystores-cli + tag: "v1.0.0" pullPolicy: IfNotPresent externalSecrets: diff --git a/charts/execution-beacon/Chart.yaml b/charts/execution-beacon/Chart.yaml index 20384488c..08373cb6b 100644 --- a/charts/execution-beacon/Chart.yaml +++ b/charts/execution-beacon/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: execution-beacon -version: 0.0.9 +version: 1.0.0 appVersion: 0.0.1 kubeVersion: "^1.23.0-0" description: Execution and Beacon diff --git a/charts/execution-beacon/templates/statefulset.yaml b/charts/execution-beacon/templates/statefulset.yaml index a8ea447b8..c5e322883 100644 --- a/charts/execution-beacon/templates/statefulset.yaml +++ b/charts/execution-beacon/templates/statefulset.yaml @@ -133,11 +133,7 @@ spec: */}} {{- if eq .Values.execution.client "nethermind" }} exec /nethermind/Nethermind.Runner - {{- if eq .Values.global.network "gnosis" }} - --config=xdai - {{- else }} --config={{ .Values.global.network }} - {{- end }} --datadir=/data/execution {{- if .Values.execution.jsonrpc.enabled }} --JsonRpc.Enabled={{ .Values.execution.jsonrpc.enabled }} diff --git a/charts/mev-boost/Chart.yaml b/charts/mev-boost/Chart.yaml index 69d8eaedf..1d8e4c27f 100644 --- a/charts/mev-boost/Chart.yaml +++ b/charts/mev-boost/Chart.yaml @@ -1,31 +1,13 @@ apiVersion: v2 name: mev-boost description: mev-boost allows proof-of-stake Ethereum consensus clients to outsource block construction +version: 1.0.0 home: https://boost.flashbots.net/ keywords: - ethereum - blockchain - mev-boost - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.2 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. appVersion: "1.6.0" dependencies: diff --git a/charts/posmoni/Chart.yaml b/charts/posmoni/Chart.yaml index ca446e556..8d211bfbe 100644 --- a/charts/posmoni/Chart.yaml +++ b/charts/posmoni/Chart.yaml @@ -1,26 +1,8 @@ apiVersion: v2 name: posmoni description: A Helm chart for installing and configuring Posmoni - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.1 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. +version: 1.0.0 appVersion: "v0.0.1" keywords: diff --git a/charts/validator-ejector/Chart.yaml b/charts/validator-ejector/Chart.yaml index 1c45fe756..09ed578a7 100644 --- a/charts/validator-ejector/Chart.yaml +++ b/charts/validator-ejector/Chart.yaml @@ -1,26 +1,8 @@ apiVersion: v2 name: validator-ejector description: A Helm chart for installing and configuring Lido's validator-ejector - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.1 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. +version: 1.0.0 appVersion: "v0.0.1" keywords: diff --git a/charts/validator-ejector/values.yaml b/charts/validator-ejector/values.yaml index 98a3f1295..9fdc58487 100644 --- a/charts/validator-ejector/values.yaml +++ b/charts/validator-ejector/values.yaml @@ -38,7 +38,7 @@ global: loader: repository: nethermindeth/eth-exit-messages pullPolicy: IfNotPresent - tag: "v0.0.15" + tag: "v0.0.26" loader: EIP2335_PASSWORD: "test" diff --git a/charts/validator-kapi/Chart.yaml b/charts/validator-kapi/Chart.yaml index cdbaa9401..c49f011ed 100644 --- a/charts/validator-kapi/Chart.yaml +++ b/charts/validator-kapi/Chart.yaml @@ -1,26 +1,8 @@ apiVersion: v2 name: validator-kapi description: A Helm chart for installing and configuring Lido's validator-kapi - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.1 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. +version: 1.0.0 appVersion: "v0.0.1" keywords: @@ -36,6 +18,6 @@ dependencies: repository: file://../common version: 1.0.0 - name: postgresql - version: 12.2.6 + version: 13.2.23 repository: https://charts.bitnami.com/bitnami/ condition: postgresql.enabled diff --git a/charts/validators/Chart.yaml b/charts/validators/Chart.yaml index 078652704..fbd7d342b 100644 --- a/charts/validators/Chart.yaml +++ b/charts/validators/Chart.yaml @@ -1,27 +1,9 @@ apiVersion: v2 name: validators description: A Helm chart for installing validators with the web3signer. - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.8 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "v0.0.8" +version: 1.0.0 +appVersion: "v1.0.0" keywords: - ethereum diff --git a/charts/validators/templates/_helpers.tpl b/charts/validators/templates/_helpers.tpl index 534e81577..e379a2aea 100644 --- a/charts/validators/templates/_helpers.tpl +++ b/charts/validators/templates/_helpers.tpl @@ -106,7 +106,9 @@ Validator beacon node {{- else if eq $.Values.type "teku" }} - "--beacon-node-api-endpoint={{ $.Values.beaconChainRpcEndpoints | join "," }}" {{- else if eq $.Values.type "nimbus" }} -- "--beacon-node={{ $.Values.beaconChainRpcEndpoints | join "," }}" +{{- range $.Values.beaconChainRpcEndpoints }} +- "--beacon-node={{ . }}" +{{- end }} {{- else if eq $.Values.type "lodestar" }} - "--beaconNodes={{ $.Values.beaconChainRpcEndpoints | join "," }}" {{- end }} diff --git a/charts/validators/templates/configmap.yaml b/charts/validators/templates/configmap.yaml index 0ce2f4521..cfafa4c6f 100644 --- a/charts/validators/templates/configmap.yaml +++ b/charts/validators/templates/configmap.yaml @@ -1,193 +1,3 @@ ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "validators.fullname" . }}-gnosis-prysm - labels: - {{- include "validators.labels" . | nindent 4 }} - annotations: - "helm.sh/resource-policy": keep - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "-5" -data: - config.yaml: |- - CONFIG_NAME: gnosis - PRESET_BASE: mainnet - - # Misc - # --------------------------------------------------------------- - # 2**6 (= 64) - MAX_COMMITTEES_PER_SLOT: 64 - # 2**7 (= 128) - TARGET_COMMITTEE_SIZE: 128 - # 2**11 (= 2,048) - MAX_VALIDATORS_PER_COMMITTEE: 2048 - # 2**2 (= 4) - MIN_PER_EPOCH_CHURN_LIMIT: 4 - # 2**12 (= 4096) - CHURN_LIMIT_QUOTIENT: 4096 - # See issue 563 - SHUFFLE_ROUND_COUNT: 90 - # `2**12` (= 4096) - MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 4096 - # Dec 08, 2021, 13:00 UTC - MIN_GENESIS_TIME: 1638968400 - # 4 - HYSTERESIS_QUOTIENT: 4 - # 1 (minus 0.25) - HYSTERESIS_DOWNWARD_MULTIPLIER: 1 - # 5 (plus 1.25) - HYSTERESIS_UPWARD_MULTIPLIER: 5 - # Fork Choice - # --------------------------------------------------------------- - # 2**3 (= 8) - SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 8 - # Validator - # --------------------------------------------------------------- - # 2**10 (= 1024) ~1.4 hour - ETH1_FOLLOW_DISTANCE: 1024 - # 2**4 (= 16) - TARGET_AGGREGATORS_PER_COMMITTEE: 16 - # 2**0 (= 1) - RANDOM_SUBNETS_PER_VALIDATOR: 1 - # 2**8 (= 256) - EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION: 256 - # 6 (estimate from xDai mainnet) - SECONDS_PER_ETH1_BLOCK: 6 - - # Deposit contract - # --------------------------------------------------------------- - # xDai Mainnet - DEPOSIT_CHAIN_ID: 100 - DEPOSIT_NETWORK_ID: 100 - # GBC deposit contract on xDai Mainnet - DEPOSIT_CONTRACT_ADDRESS: 0x0B98057eA310F4d31F2a452B414647007d1645d9 - # Gwei values - # --------------------------------------------------------------- - # 2**0 * 10**9 (= 1,000,000,000) Gwei - MIN_DEPOSIT_AMOUNT: 1000000000 - # 2**5 * 10**9 (= 32,000,000,000) Gwei - MAX_EFFECTIVE_BALANCE: 32000000000 - # 2**4 * 10**9 (= 16,000,000,000) Gwei - EJECTION_BALANCE: 16000000000 - # 2**0 * 10**9 (= 1,000,000,000) Gwei - EFFECTIVE_BALANCE_INCREMENT: 1000000000 - # Initial values - # --------------------------------------------------------------- - # GBC area code - GENESIS_FORK_VERSION: 0x00000064 - BLS_WITHDRAWAL_PREFIX: 0x00 - # Time parameters - # --------------------------------------------------------------- - # Customized for GBC: ~1 hour - GENESIS_DELAY: 6000 - # 5 seconds - SECONDS_PER_SLOT: 5 - # 2**0 (= 1) slots 12 seconds - MIN_ATTESTATION_INCLUSION_DELAY: 1 - # 2**4 (= 16) slots 1.87 minutes - SLOTS_PER_EPOCH: 16 - # 2**0 (= 1) epochs 1.87 minutes - MIN_SEED_LOOKAHEAD: 1 - # 2**2 (= 4) epochs 7.47 minutes - MAX_SEED_LOOKAHEAD: 4 - # 2**6 (= 64) epochs ~2 hours - EPOCHS_PER_ETH1_VOTING_PERIOD: 64 - # 2**13 (= 8,192) slots ~15.9 hours - SLOTS_PER_HISTORICAL_ROOT: 8192 - # 2**8 (= 256) epochs ~8 hours - MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 - # 2**8 (= 256) epochs ~8 hours - SHARD_COMMITTEE_PERIOD: 256 - # 2**2 (= 4) epochs 7.47 minutes - MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 - - # State vector lengths - # --------------------------------------------------------------- - # 2**16 (= 65,536) epochs ~85 days - EPOCHS_PER_HISTORICAL_VECTOR: 65536 - # 2**13 (= 8,192) epochs ~10.6 days - EPOCHS_PER_SLASHINGS_VECTOR: 8192 - # 2**24 (= 16,777,216) historical roots, ~15,243 years - HISTORICAL_ROOTS_LIMIT: 16777216 - # 2**40 (= 1,099,511,627,776) validator spots - VALIDATOR_REGISTRY_LIMIT: 1099511627776 - # Reward and penalty quotients - # --------------------------------------------------------------- - # 25 - BASE_REWARD_FACTOR: 25 - # 2**9 (= 512) - WHISTLEBLOWER_REWARD_QUOTIENT: 512 - # 2**3 (= 8) - PROPOSER_REWARD_QUOTIENT: 8 - # 2**26 (= 67,108,864) - INACTIVITY_PENALTY_QUOTIENT: 67108864 - # 2**7 (= 128) (lower safety margin at Phase 0 genesis) - MIN_SLASHING_PENALTY_QUOTIENT: 128 - # 1 (lower safety margin at Phase 0 genesis) - PROPORTIONAL_SLASHING_MULTIPLIER: 1 - # Max operations per block - # --------------------------------------------------------------- - # 2**4 (= 16) - MAX_PROPOSER_SLASHINGS: 16 - # 2**1 (= 2) - MAX_ATTESTER_SLASHINGS: 2 - # 2**7 (= 128) - MAX_ATTESTATIONS: 128 - # 2**4 (= 16) - MAX_DEPOSITS: 16 - # 2**4 (= 16) - MAX_VOLUNTARY_EXITS: 16 - # Signature domains - # --------------------------------------------------------------- - DOMAIN_BEACON_PROPOSER: 0x00000000 - DOMAIN_BEACON_ATTESTER: 0x01000000 - DOMAIN_RANDAO: 0x02000000 - DOMAIN_DEPOSIT: 0x03000000 - DOMAIN_VOLUNTARY_EXIT: 0x04000000 - DOMAIN_SELECTION_PROOF: 0x05000000 - DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 - DOMAIN_SYNC_COMMITTEE: 0x07000000 - DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF: 0x08000000 - DOMAIN_CONTRIBUTION_AND_PROOF: 0x09000000 - - ALTAIR_FORK_VERSION: 0x01000064 - ALTAIR_FORK_EPOCH: 512 - - BELLATRIX_FORK_VERSION: 0x02000064 - BELLATRIX_FORK_EPOCH: 18446744073709551615 - - INACTIVITY_SCORE_BIAS: 4 - # 2**4 (= 16) - INACTIVITY_SCORE_RECOVERY_RATE: 16 - INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 - MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 - PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 - - # Sync committee - # --------------------------------------------------------------- - # 2**9 (= 512) - SYNC_COMMITTEE_SIZE: 512 - # 2**9 (= 512) - # assert EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH <= SLOTS_PER_HISTORICAL_ROOT - EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 512 - -{{- if .Values.feeOverride }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "validators.fullname" . }}-override-fee-recipient - labels: - {{- include "validators.labels" . | nindent 4 }} - annotations: - "helm.sh/resource-policy": keep - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "-5" -data: - feeOverride.json: |- - {{- .Values.feeOverride | nindent 4 }} -{{- end }} {{- if .Values.enableWatcher }} --- apiVersion: v1 diff --git a/charts/validators/templates/statefulset.yaml b/charts/validators/templates/statefulset.yaml index 0eb2b466e..94b26d689 100644 --- a/charts/validators/templates/statefulset.yaml +++ b/charts/validators/templates/statefulset.yaml @@ -1,6 +1,7 @@ {{- if .Values.enabled }} {{- $root := . -}} {{- $counter := 0 }} + {{- range (untilStep (int .Values.validatorsKeyIndex) (int (add .Values.validatorsKeyIndex .Values.validatorsCount)) 1) }} {{ $rpcEndpoints := list }} @@ -100,10 +101,6 @@ spec: volumeMounts: - name: data mountPath: /data - {{- if $root.Values.feeOverride }} - - name: fee-override - mountPath: /fee - {{- end }} - name: prepare image: "{{ $root.Values.initImageBusybox.repository }}:{{ $root.Values.initImageBusybox.tag }}" imagePullPolicy: {{ $root.Values.initImageBusybox.pullPolicy }} @@ -116,17 +113,14 @@ spec: ls -lha /data; mkdir -p /data/lighthouse/validators; cp /data/validator_definitions.yml /data/lighthouse/validators/validator_definitions.yml; - cp /mnt/gnosis-prysm/config.yaml /data/config_gnosis_prysm; cat /data/signer_keys.yml > /data/config; cat /data/config; cat /data/proposerConfig.json; cat /data/proposerConfig.yaml; formatted_content=$(cat /data/signer_keys.yml | grep -o '".*"' | sed -e 's/"//g' | tr ',' '\n'); - echo "$formatted_content" > /data/pubkeys.txt; + echo "$formatted_content" > /data/pubkeys.txt; + echo '{"externalSigner.pubkeys": ['"$(awk '{printf "%s\"%s\"", (NR==1 ? "" : ", "), $0}' pubkeys.txt)"']}' > /data/rcconfig.json; cat /data/pubkeys.txt; - echo "" >> - cat /data/signer_keys.yml >> /data/config_gnosis_prysm; - cat /data/config_gnosis_prysm; chown -R {{ $root.Values.securityContext.runAsUser }}:{{ $root.Values.securityContext.runAsUser }} /data; {{- if eq $root.Values.type "nimbus" }} find /data/nimbus -type d -name '0x*' -exec chmod 0600 {}/remote_keystore.json \; @@ -134,8 +128,6 @@ spec: volumeMounts: - name: data mountPath: /data - - name: gnosis-prysm - mountPath: /mnt/gnosis-prysm containers: {{- if $root.Values.enableWatcher }} - name: watcher @@ -158,10 +150,6 @@ spec: volumeMounts: - name: data mountPath: /data - {{- if $root.Values.feeOverride }} - - name: fee-override - mountPath: /fee - {{- end }} {{- if $root.Values.enableWatcher }} - name: watcher mountPath: /scripts @@ -175,7 +163,6 @@ spec: - sh - -c - > - export PUBKEYS=$(paste -sd, /data/pubkeys.txt); node ./packages/cli/bin/lodestar {{- range (pluck $root.Values.type $root.Values.flags | first) }} {{ . }} @@ -183,9 +170,9 @@ spec: --network={{ $root.Values.network }} --proposerSettingsFile=/data/proposerConfig.yaml --externalSigner.url={{ include "web3signer" $root }} - --externalSigner.pubkeys=$PUBKEYS + --rcConfig=/data/rcconfig.json {{- if $.Values.beaconChainRpcEndpointsRandomized }} - --beaconNodes={{ $rpcEndpoints }}" + --beaconNodes={{ $rpcEndpoints }} {{- else }} {{- include "beacon-rpc-node" $ | nindent 12 }} {{- end }} @@ -204,15 +191,11 @@ spec: - {{ . | quote }} {{- end -}} - {{- if and (eq $root.Values.type "prysm") (ne $root.Values.network "gnosis") }} + {{- if eq $root.Values.type "prysm" }} - "--{{ $root.Values.network }}" - "--config-file=/data/config" - "--validators-external-signer-url={{ include "web3signer" $root }}" - "--proposer-settings-file=/data/proposerConfig.json" - {{- else if and (eq $root.Values.type "prysm") (eq $root.Values.network "gnosis") }} - - "--config-file /data/config_gnosis_prysm" - - "--chain-config-file /data/config_gnosis_prysm" - - "--validators-external-signer-url={{ include "web3signer" $root }}" {{- else if eq $root.Values.type "lighthouse" }} - "--network={{ $root.Values.network }}" {{- else if eq $root.Values.type "teku" }} @@ -223,21 +206,26 @@ spec: {{- if $root.Values.enableBuilder }} - "--validators-proposer-blinded-blocks-enabled=true" {{- end }} + {{- else if eq $root.Values.type "nimbus" }} + - "--validators-dir=/data/nimbus" + {{- if $root.Values.enableBuilder }} + - "--payload-builder=true" + {{- end }} {{- end }} {{- include "validator-graffiti" $ | nindent 12 }} - {{- if $.Values.beaconChainRpcEndpointsRandomized }} + {{- if $root.Values.beaconChainRpcEndpointsRandomized }} {{- if eq $.Values.type "prysm" }} - "--beacon-rpc-provider={{ $rpcEndpoints }}" {{- else if eq $.Values.type "lighthouse" }} - "--beacon-nodes={{ $rpcEndpoints }}" {{- else if eq $.Values.type "teku" }} - {{- $beaconChainRpcEndpointsLen := len $.Values.beaconChainRpcEndpointsRandomized }} - {{- if gt $beaconChainRpcEndpointsLen 1 }} - - "--beacon-node-api-endpoints={{ $rpcEndpoints | join "," }}" - {{- else }} - - "--beacon-node-api-endpoint={{ $rpcEndpoints }}" + - "--beacon-node-api-endpoints={{ $rpcEndpoints }}" + {{- else if eq $.Values.type "nimbus" }} + {{- $valuesList := split "," $rpcEndpoints }} + {{- range $endpoint := $valuesList }} + - "--beacon-node={{ $endpoint }}" {{- end }} {{- end }} {{- else }} @@ -276,14 +264,6 @@ spec: - name: data mountPath: /data volumes: - - name: gnosis-prysm - configMap: - name: {{ template "validators.fullname" $root }}-gnosis-prysm - {{- if $root.Values.feeOverride }} - - name: fee-override - configMap: - name: {{ template "validators.fullname" $root }}-override-fee-recipient - {{- end }} {{- if $root.Values.enableWatcher }} - name: watcher configMap: @@ -293,4 +273,4 @@ spec: emptyDir: {} {{- end }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/validators/values.yaml b/charts/validators/values.yaml index 0286dfb2e..435e52f4a 100644 --- a/charts/validators/values.yaml +++ b/charts/validators/values.yaml @@ -49,8 +49,8 @@ initImageBusybox: ## CLI image is used to fetch public keys. ## cliImage: - repository: 238155366538.dkr.ecr.eu-west-1.amazonaws.com/keystores-cli - tag: "v0.0.2" + repository: nethermindeth/keystores-cli + tag: "v1.0.0" pullPolicy: IfNotPresent externalSecrets: @@ -72,10 +72,10 @@ externalSecrets: enabled: true ## What type of validator to use. -## Options for Ethereum: prysm, lighthouse, teku -## Options for Gnosis: prysm, lighthouse +## Options for Ethereum: prysm, lighthouse, teku, nimbus, lodestar +## Options for Gnosis: teku, lighthouse, nimbus ## -type: prysm +type: teku ## If you want to run multiple validator types (e.g lighouse, teku,...) ## you need to adjust the key index to prevent double signing. @@ -115,31 +115,28 @@ enableBuilder: false ## ref: https://hub.docker.com/r/sigp/lighthouse image: pullPolicy: IfNotPresent - prysm: - repository: "gcr.io/prysmaticlabs/prysm/validator" - tag: "v4.0.8" - prysmGnosis: - repository: "ghcr.io/gnosischain/gbc-prysm-validator" - tag: "v2.1.2-gbc" - lighthouse: - repository: "sigp/lighthouse" - tag: "v4.4.1" - teku: + prysm: + repository: "gcr.io/prylabs-dev/prysm/validator" + tag: "v4.1.1" + teku: repository: "consensys/teku" - tag: "23.9.0" - nimbus: + tag: "23.11.0" + lighthouse: + repository: "sigp/lighthouse" + tag: "v4.5.0" + nimbus: repository: "statusim/nimbus-validator-client" - tag: "multiarch-v23.9.0" - lodestar: + tag: "multiarch-v23.11.0" + lodestar: repository: "chainsafe/lodestar" - tag: "v1.11.0" + tag: "v1.12.0" ## Credentials to fetch images from private registry ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ ## imagePullSecrets: [] -terminationGracePeriodSeconds: 300 +terminationGracePeriodSeconds: 120 ## Spearate service account per validator. ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ @@ -197,6 +194,15 @@ flags: - "validator-client" - "--log-destination=CONSOLE" - "--data-base-path=/data" + nimbus: + - "--data-dir=/data/nimbus" + - "--non-interactive" + - "--log-level=INFO" + - "--doppelganger-detection=off" + lodestar: + - "validator" + - "--dataDir=/data/lodestar" + - "--logLevel=info" ## Validators extra flags ## @@ -228,11 +234,6 @@ beaconChainRpcEndpointsRandomized: [] ## graffiti: "" -## Override fees for solo validators -## used for stakewise solo validators only -## -feeOverride: "" - enableWatcher: false ## Lodestar specific setting @@ -262,13 +263,13 @@ metrics: - "--metrics-host-allowlist=*" - "--metrics-interface=0.0.0.0" - "--metrics-port=9090" - nimbus: - - "--metrics" - - "--metrics-port=9090" - - "--metrics-address=0.0.0.0" - lodestar: - - "--metrics" - - "--metrics.address=0.0.0.0" + nimbus: + - "--metrics" + - "--metrics-port=9090" + - "--metrics-address=0.0.0.0" + lodestar: + - "--metrics" + - "--metrics.address=0.0.0.0" - "--metrics.port=9090" ## Prometheus Service Monitor @@ -377,6 +378,26 @@ readinessProbe: path: /metrics port: metrics scheme: HTTP + nimbus: + initialDelaySeconds: 60 + timeoutSeconds: 1 + periodSeconds: 60 + failureThreshold: 3 + successThreshold: 1 + httpGet: + path: /metrics + port: metrics + scheme: HTTP + lodestar: + initialDelaySeconds: 60 + timeoutSeconds: 1 + periodSeconds: 60 + failureThreshold: 3 + successThreshold: 1 + httpGet: + path: /metrics + port: metrics + scheme: HTTP livenessProbe: prysm: initialDelaySeconds: 60 @@ -408,3 +429,23 @@ livenessProbe: path: /metrics port: metrics scheme: HTTP + nimbus: + initialDelaySeconds: 60 + timeoutSeconds: 1 + periodSeconds: 60 + failureThreshold: 3 + successThreshold: 1 + httpGet: + path: /metrics + port: metrics + scheme: HTTP + lodestar: + initialDelaySeconds: 60 + timeoutSeconds: 1 + periodSeconds: 60 + failureThreshold: 3 + successThreshold: 1 + httpGet: + path: /metrics + port: metrics + scheme: HTTP diff --git a/charts/vouch/Chart.yaml b/charts/vouch/Chart.yaml index c052e2670..11ee259c4 100644 --- a/charts/vouch/Chart.yaml +++ b/charts/vouch/Chart.yaml @@ -1,26 +1,8 @@ apiVersion: v2 name: vouch description: A Helm chart for installing and configuring large scale ETH staking infrastructure on top of the Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) version: 1.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. appVersion: "v22.10.0" keywords: diff --git a/charts/vouch/values.yaml b/charts/vouch/values.yaml index ee4c5946a..95caa339e 100644 --- a/charts/vouch/values.yaml +++ b/charts/vouch/values.yaml @@ -20,8 +20,8 @@ initImage: pullPolicy: IfNotPresent cliImage: - repository: 238155366538.dkr.ecr.eu-west-1.amazonaws.com/keystores-cli - tag: "dirk-0.0.16" + repository: nethermindeth/keystores-cli + tag: "v1.0.0" pullPolicy: IfNotPresent externalSecrets: diff --git a/charts/web3signer/Chart.yaml b/charts/web3signer/Chart.yaml index 52a957338..de40fc293 100644 --- a/charts/web3signer/Chart.yaml +++ b/charts/web3signer/Chart.yaml @@ -1,27 +1,9 @@ apiVersion: v2 name: web3signer description: A Helm chart for installing and configuring Web3signer - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.2 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "23.6.0" +version: 1.0.0 +appVersion: "23.11.0" keywords: - web3signer diff --git a/charts/web3signer/templates/statefulset.yaml b/charts/web3signer/templates/statefulset.yaml index 6762757b1..dde9afaa4 100644 --- a/charts/web3signer/templates/statefulset.yaml +++ b/charts/web3signer/templates/statefulset.yaml @@ -66,8 +66,6 @@ spec: - -password=$(ESO_DB_PASSWORD) - -url=$(ESO_DB_URL) - migrate - # for first time clean may be needed, be careful not remove production db - # flyway -user=$ESO_DB_USERNAME -password=$ESO_DB_PASSWORD -url=$ESO_DB_URL clean -cleanDisabled=false envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -147,7 +145,7 @@ spec: envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} - args: ["sync-web3signer-keys", "--db-url", $(ESO_DB_KEYSTORE_URL), "--output-dir", "/data/keystore", "--decryption-key-env", "ESO_DECRYPTION_KEY_V2", "--refresh-frequency", "60"] + args: ["sync-web3signer-keys", "--db-url", $(ESO_DB_KEYSTORE_URL), "--output-dir", "/data/keystore", "--decryption-key-env", "ESO_DECRYPTION_KEY", "--refresh-frequency", "60"] volumeMounts: - name: data mountPath: /data diff --git a/charts/web3signer/values.yaml b/charts/web3signer/values.yaml index 66c91a917..ab14d81c7 100644 --- a/charts/web3signer/values.yaml +++ b/charts/web3signer/values.yaml @@ -17,20 +17,20 @@ image: repository: consensys/web3signer pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "23.6.0" + tag: "23.11.0" ## Init image is used to chown data volume, etc. ## initImage: repository: busybox - tag: "1.36.1" + tag: "1.36" pullPolicy: IfNotPresent ## CLI image is used to fetch private keys. ## cliImage: - repository: 238155366538.dkr.ecr.eu-west-1.amazonaws.com/keystores-cli - tag: "v0.0.2" + repository: nethermindeth/keystores-cli + tag: "v1.0.0" pullPolicy: IfNotPresent From 8db086e10a55c7f8052a183dead386c02cc5a28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= Date: Fri, 1 Dec 2023 19:15:01 +0100 Subject: [PATCH 02/34] update image tags --- charts/mev-boost/values.yaml | 2 +- charts/validator-ejector/values.yaml | 4 ++-- charts/validator-kapi/values.yaml | 2 +- charts/validators/values.yaml | 2 +- charts/vouch/Chart.yaml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/mev-boost/values.yaml b/charts/mev-boost/values.yaml index 1e8622187..61a94600f 100644 --- a/charts/mev-boost/values.yaml +++ b/charts/mev-boost/values.yaml @@ -37,7 +37,7 @@ image: repository: flashbots/mev-boost pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "1.5.0" + tag: "1.6" imagePullSecrets: [] nameOverride: "" diff --git a/charts/validator-ejector/values.yaml b/charts/validator-ejector/values.yaml index 9fdc58487..e25966103 100644 --- a/charts/validator-ejector/values.yaml +++ b/charts/validator-ejector/values.yaml @@ -27,13 +27,13 @@ global: initImage: repository: "bitnami/kubectl" - tag: "1.26" + tag: "1.28" pullPolicy: IfNotPresent image: repository: lidofinance/validator-ejector pullPolicy: IfNotPresent - tag: "dev" + tag: "1.2.0" loader: repository: nethermindeth/eth-exit-messages diff --git a/charts/validator-kapi/values.yaml b/charts/validator-kapi/values.yaml index 0384d22e3..d42607063 100644 --- a/charts/validator-kapi/values.yaml +++ b/charts/validator-kapi/values.yaml @@ -36,7 +36,7 @@ global: image: repository: lidofinance/lido-keys-api pullPolicy: IfNotPresent - tag: "stable" + tag: "0.10.1" kapi: env: "production" diff --git a/charts/validators/values.yaml b/charts/validators/values.yaml index 435e52f4a..9de0f6e1c 100644 --- a/charts/validators/values.yaml +++ b/charts/validators/values.yaml @@ -43,7 +43,7 @@ rbac: ## initImageBusybox: repository: "busybox" - tag: "1.36.1" + tag: "1.36" pullPolicy: IfNotPresent ## CLI image is used to fetch public keys. diff --git a/charts/vouch/Chart.yaml b/charts/vouch/Chart.yaml index 11ee259c4..9b9995d8f 100644 --- a/charts/vouch/Chart.yaml +++ b/charts/vouch/Chart.yaml @@ -3,7 +3,7 @@ name: vouch description: A Helm chart for installing and configuring large scale ETH staking infrastructure on top of the Kubernetes type: application version: 1.1.0 -appVersion: "v22.10.0" +appVersion: "1.7.6" keywords: - ethereum From 9c9a18d53bfcdadcf2b2e374aea0801d44f81495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= Date: Tue, 5 Dec 2023 19:59:04 +0100 Subject: [PATCH 03/34] add fallbackRpcEndpoints and fix sharedPersistence init --- charts/execution-beacon/templates/configmap.yaml | 2 +- charts/validators/templates/statefulset.yaml | 6 ++++++ charts/validators/values.yaml | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/charts/execution-beacon/templates/configmap.yaml b/charts/execution-beacon/templates/configmap.yaml index aba47dc3b..6e2baab2f 100644 --- a/charts/execution-beacon/templates/configmap.yaml +++ b/charts/execution-beacon/templates/configmap.yaml @@ -8,7 +8,7 @@ metadata: data: init.sh: | #!/bin/sh -{{- if or (and .Values.execution.persistence.enabled .Values.execution.initChownData) (and .Values.beacon.persistence.enabled .Values.beacon.initChownData) }} +{{- if or (and .Values.execution.persistence.enabled .Values.execution.initChownData) (and .Values.beacon.persistence.enabled .Values.beacon.initChownData) (.Values.sharedPersistence.enabled) }} mkdir -p /data && chown -R {{ .Values.global.securityContext.runAsUser }}:{{ .Values.global.securityContext.runAsUser }} /data; mkdir -p /data/beacon && chown -R {{ .Values.global.securityContext.runAsUser }}:{{ .Values.global.securityContext.runAsUser }} /data/beacon; {{- end }} diff --git a/charts/validators/templates/statefulset.yaml b/charts/validators/templates/statefulset.yaml index 94b26d689..7c795f528 100644 --- a/charts/validators/templates/statefulset.yaml +++ b/charts/validators/templates/statefulset.yaml @@ -11,6 +11,12 @@ {{ $rpcList = append $rpcList (index $.Values.beaconChainRpcEndpointsRandomized .) }} {{- end }} {{ $rpcList = append $rpcList $.Values.beaconChainRpcEndpointsRandomized }} + +# Appending the failover endpoints to the rpcList before flattening +{{- if $.Values.fallbackRpcEndpoints }} + {{- $rpcList = append $rpcList $.Values.fallbackRpcEndpoints }} +{{- end }} + {{ $rpcEndpoints = include "flatten" $rpcList }} {{- if ge $counter (len $.Values.beaconChainRpcEndpointsRandomized) }} {{- $counter = 1 }} diff --git a/charts/validators/values.yaml b/charts/validators/values.yaml index 9de0f6e1c..9cbc40a78 100644 --- a/charts/validators/values.yaml +++ b/charts/validators/values.yaml @@ -228,6 +228,12 @@ beaconChainRpcEndpoints: [] ## beaconChainRpcEndpointsRandomized: [] +## if using beaconChainRpcEndpointsRandomized +## fallbackRpcEndpoints will be appended to the list +## always serving as the last, failover endpoint +## +fallbackRpcEndpoints: [] + ## You can use the graffiti to add a string to your proposed blocks, ## which will be seen on the block explorer. ## ref: https://docs.prylabs.network/docs/prysm-usage/parameters#validator-configuration From ddbc13c9a1ba3152d891a1ec2eca4f4f26fb1e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= Date: Tue, 5 Dec 2023 21:49:10 +0100 Subject: [PATCH 04/34] add missing envFrom --- charts/execution-beacon/templates/statefulset.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/execution-beacon/templates/statefulset.yaml b/charts/execution-beacon/templates/statefulset.yaml index c5e322883..5193eee7c 100644 --- a/charts/execution-beacon/templates/statefulset.yaml +++ b/charts/execution-beacon/templates/statefulset.yaml @@ -318,6 +318,11 @@ spec: - name: BESU_OPTS value: {{ .Values.execution.javaOpts.maxHeapSize }} {{- end }} + {{- if .Values.global.externalSecrets.enabled }} + envFrom: + - secretRef: + name: eso-{{ include "common.names.fullname" . }} + {{- end }} ports: {{- if eq .Values.execution.client "erigon" }} - name: grpc-exec From 7daa6e416761534f6b0b997b20d66c9fd77446bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= Date: Wed, 6 Dec 2023 14:55:27 +0100 Subject: [PATCH 05/34] add securityContext nonRoot --- charts/execution-beacon/templates/statefulset.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/execution-beacon/templates/statefulset.yaml b/charts/execution-beacon/templates/statefulset.yaml index 5193eee7c..364ac2c1b 100644 --- a/charts/execution-beacon/templates/statefulset.yaml +++ b/charts/execution-beacon/templates/statefulset.yaml @@ -54,8 +54,9 @@ spec: image: "{{ .Values.global.initImage.repository }}:{{ .Values.global.initImage.tag }}" imagePullPolicy: {{ .Values.global.initImage.pullPolicy }} securityContext: - runAsNonRoot: false - runAsUser: 0 + runAsNonRoot: {{ .Values.securityContext.runAsNonRoot | default true }} + runAsUser: {{ .Values.securityContext.runAsUser | default 10000 }} + readOnlyRootFilesystem: {{ .Values.securityContext.readOnlyRootFilesystem | default true }} env: - name: POD_IP valueFrom: From e0ee790ecc404c33c02b447db27222282fde9bfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= Date: Wed, 6 Dec 2023 14:58:31 +0100 Subject: [PATCH 06/34] fix references --- charts/execution-beacon/templates/statefulset.yaml | 6 +++--- charts/execution-beacon/values.yaml | 8 ++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/charts/execution-beacon/templates/statefulset.yaml b/charts/execution-beacon/templates/statefulset.yaml index 364ac2c1b..9a8960618 100644 --- a/charts/execution-beacon/templates/statefulset.yaml +++ b/charts/execution-beacon/templates/statefulset.yaml @@ -54,9 +54,9 @@ spec: image: "{{ .Values.global.initImage.repository }}:{{ .Values.global.initImage.tag }}" imagePullPolicy: {{ .Values.global.initImage.pullPolicy }} securityContext: - runAsNonRoot: {{ .Values.securityContext.runAsNonRoot | default true }} - runAsUser: {{ .Values.securityContext.runAsUser | default 10000 }} - readOnlyRootFilesystem: {{ .Values.securityContext.readOnlyRootFilesystem | default true }} + runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} + runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} + readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} env: - name: POD_IP valueFrom: diff --git a/charts/execution-beacon/values.yaml b/charts/execution-beacon/values.yaml index 617c8986e..8ff2be7d3 100644 --- a/charts/execution-beacon/values.yaml +++ b/charts/execution-beacon/values.yaml @@ -243,6 +243,7 @@ global: ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ ## securityContext: + runAsNonRoot: true fsGroup: 1001 runAsUser: 1001 @@ -497,12 +498,7 @@ beacon: port: 4000 portName: "rpc" - targetPeersMap: - teku: 50 - prysm: 50 - lighthouse: 50 - nimbus: 50 - lodestar: 50 + targetPeers: 50 ## Sets the total difficulty to manual overrides the default ## TERMINAL_TOTAL_DIFFICULTY value. WARNING: This flag should be used only if you From 30fba23a54887c89a88e506faabd3f08bd6e398f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= Date: Wed, 6 Dec 2023 15:09:11 +0100 Subject: [PATCH 07/34] remove ownership changes --- charts/execution-beacon/templates/configmap.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/execution-beacon/templates/configmap.yaml b/charts/execution-beacon/templates/configmap.yaml index 6e2baab2f..70cb120fc 100644 --- a/charts/execution-beacon/templates/configmap.yaml +++ b/charts/execution-beacon/templates/configmap.yaml @@ -9,8 +9,8 @@ data: init.sh: | #!/bin/sh {{- if or (and .Values.execution.persistence.enabled .Values.execution.initChownData) (and .Values.beacon.persistence.enabled .Values.beacon.initChownData) (.Values.sharedPersistence.enabled) }} - mkdir -p /data && chown -R {{ .Values.global.securityContext.runAsUser }}:{{ .Values.global.securityContext.runAsUser }} /data; - mkdir -p /data/beacon && chown -R {{ .Values.global.securityContext.runAsUser }}:{{ .Values.global.securityContext.runAsUser }} /data/beacon; + mkdir -p /data; + mkdir -p /data/beacon; {{- end }} echo "Namespace: ${POD_NAMESPACE} Pod: ${POD_NAME}"; {{- if .Values.global.p2pNodePort.enabled }} From 8db30196704088c21da6fe6ec9ae00a5cba159f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= Date: Wed, 6 Dec 2023 15:15:22 +0100 Subject: [PATCH 08/34] use root for init --- charts/execution-beacon/templates/configmap.yaml | 4 ++-- charts/execution-beacon/templates/statefulset.yaml | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/charts/execution-beacon/templates/configmap.yaml b/charts/execution-beacon/templates/configmap.yaml index 70cb120fc..6e2baab2f 100644 --- a/charts/execution-beacon/templates/configmap.yaml +++ b/charts/execution-beacon/templates/configmap.yaml @@ -9,8 +9,8 @@ data: init.sh: | #!/bin/sh {{- if or (and .Values.execution.persistence.enabled .Values.execution.initChownData) (and .Values.beacon.persistence.enabled .Values.beacon.initChownData) (.Values.sharedPersistence.enabled) }} - mkdir -p /data; - mkdir -p /data/beacon; + mkdir -p /data && chown -R {{ .Values.global.securityContext.runAsUser }}:{{ .Values.global.securityContext.runAsUser }} /data; + mkdir -p /data/beacon && chown -R {{ .Values.global.securityContext.runAsUser }}:{{ .Values.global.securityContext.runAsUser }} /data/beacon; {{- end }} echo "Namespace: ${POD_NAMESPACE} Pod: ${POD_NAME}"; {{- if .Values.global.p2pNodePort.enabled }} diff --git a/charts/execution-beacon/templates/statefulset.yaml b/charts/execution-beacon/templates/statefulset.yaml index 9a8960618..5193eee7c 100644 --- a/charts/execution-beacon/templates/statefulset.yaml +++ b/charts/execution-beacon/templates/statefulset.yaml @@ -54,9 +54,8 @@ spec: image: "{{ .Values.global.initImage.repository }}:{{ .Values.global.initImage.tag }}" imagePullPolicy: {{ .Values.global.initImage.pullPolicy }} securityContext: - runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} - runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} - readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} + runAsNonRoot: false + runAsUser: 0 env: - name: POD_IP valueFrom: From 6f2a7b3de1ae67fa920c97e93ce7e49af04af0dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= Date: Tue, 12 Dec 2023 12:06:54 +0100 Subject: [PATCH 09/34] remove dataStorage settings for Teku --- charts/execution-beacon/templates/statefulset.yaml | 3 --- charts/execution-beacon/values.yaml | 9 --------- 2 files changed, 12 deletions(-) diff --git a/charts/execution-beacon/templates/statefulset.yaml b/charts/execution-beacon/templates/statefulset.yaml index 5193eee7c..710ab2268 100644 --- a/charts/execution-beacon/templates/statefulset.yaml +++ b/charts/execution-beacon/templates/statefulset.yaml @@ -450,9 +450,6 @@ spec: --network={{ .Values.global.network }} --data-beacon-path=/data/beacon/teku --data-path=/data/beacon - --data-storage-archive-frequency={{ .Values.beacon.dataStorageArchiveFrequency }} - --data-storage-mode={{ .Values.beacon.dataStorageMode }} - --data-storage-non-canonical-blocks-enabled={{ .Values.beacon.dataStorageNonCanonicalBlocksEnabled }} --ee-endpoint=http://localhost:{{ .Values.execution.jsonrpc.engine.port }} {{- if .Values.global.externalSecrets.enabled }} --ee-jwt-secret-file=/external-secrets/JWT_SECRET diff --git a/charts/execution-beacon/values.yaml b/charts/execution-beacon/values.yaml index 8ff2be7d3..4bb72f18a 100644 --- a/charts/execution-beacon/values.yaml +++ b/charts/execution-beacon/values.yaml @@ -463,15 +463,6 @@ beacon: ## MEV Boost endpoint ## builderEndpoint: "" - # Sets the frequency, in slots, at which to store - # finalized states to disk. This option is ignored - # if --data-storage-mode is set to PRUNE - dataStorageArchiveFrequency: "2048" - # Sets the strategy for handling historical chain data - # (Valid values: ARCHIVE, PRUNE) - dataStorageMode: PRUNE - # Store non-canonical blocks - dataStorageNonCanonicalBlocksEnabled: false # Rest API Settings restApi: From 8717ba0250f1516624af36aa9db5f75c5402e44e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Mon, 18 Dec 2023 13:33:33 +0100 Subject: [PATCH 10/34] Merge prod-stable additions (#74) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add lodestar (#56) * feat: add lodestar validator * refactor: remove hpa * fix: remove autoscaling * refactor: change bash to sh * feat: add http port and fix probes * feat: add rest for checks * chore: remove probes * Add envFrom to execution container (#67) * Add envFrom to execution container in case we have externalSecrets enabled * Set seq as part of the charts and not as a extraflag * clean up seq and mining extraData on nethermind execution charts * Remove space on charts * Fix peers not set propertly on statefulset for execution-beacon (#72) * Fix peers not set propertly on statefulset for execution-beacon * Switch to targetPeers * remove teku prune option * define upperbound peers for neku * add min targetPeers for teku lower bound --------- Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> Co-authored-by: Denis Policastro Co-authored-by: Marcos Antonio Maceo <35319980+stdevMac@users.noreply.github.com> --- .../templates/statefulset.yaml | 23 ++-- charts/execution-beacon/values.yaml | 7 +- charts/lodestar/.helmignore | 23 ++++ charts/lodestar/Chart.yaml | 18 ++++ charts/lodestar/templates/_helpers.tpl | 62 +++++++++++ charts/lodestar/templates/configmap.yaml | 34 ++++++ charts/lodestar/templates/ingress.yaml | 61 +++++++++++ charts/lodestar/templates/service.yaml | 24 +++++ charts/lodestar/templates/serviceaccount.yaml | 12 +++ charts/lodestar/templates/statefulset.yaml | 83 +++++++++++++++ charts/lodestar/values.yaml | 100 ++++++++++++++++++ 11 files changed, 432 insertions(+), 15 deletions(-) create mode 100644 charts/lodestar/.helmignore create mode 100644 charts/lodestar/Chart.yaml create mode 100644 charts/lodestar/templates/_helpers.tpl create mode 100644 charts/lodestar/templates/configmap.yaml create mode 100644 charts/lodestar/templates/ingress.yaml create mode 100644 charts/lodestar/templates/service.yaml create mode 100644 charts/lodestar/templates/serviceaccount.yaml create mode 100644 charts/lodestar/templates/statefulset.yaml create mode 100644 charts/lodestar/values.yaml diff --git a/charts/execution-beacon/templates/statefulset.yaml b/charts/execution-beacon/templates/statefulset.yaml index 710ab2268..a1b83fa27 100644 --- a/charts/execution-beacon/templates/statefulset.yaml +++ b/charts/execution-beacon/templates/statefulset.yaml @@ -121,6 +121,11 @@ spec: - name: execution image: "{{ (pluck .Values.execution.client .Values.global.image.execution | first ).repository }}:{{ (pluck .Values.execution.client .Values.global.image.execution | first ).tag }}" imagePullPolicy: {{ .Values.global.image.imagePullPolicy }} + {{- if .Values.global.externalSecrets.enabled }} + envFrom: + - secretRef: + name: eso-{{ include "common.names.fullname" . }} + {{- end }} command: - sh - -ac @@ -128,9 +133,7 @@ spec: {{- if .Values.global.p2pNodePort.enabled }} . /env/init-nodeport; {{- end }} - {{/* - Configuration for Nethermind execution client - */}} + {{- /* Configuration for Nethermind execution client */}} {{- if eq .Values.execution.client "nethermind" }} exec /nethermind/Nethermind.Runner --config={{ .Values.global.network }} @@ -179,9 +182,7 @@ spec: {{- if .Values.execution.terminalTotalDifficulty }} --Merge.TerminalTotalDifficulty={{ .Values.execution.terminalTotalDifficulty }} {{- end }} - {{/* - Configuration for Geth execution client - */}} + {{- /* Configuration for Geth execution client */}} {{- else if eq .Values.execution.client "geth" }} exec geth --syncmode=snap @@ -227,9 +228,7 @@ spec: --metrics.port={{ .Values.execution.metrics.port }} --metrics.addr={{ .Values.execution.metrics.host }} {{- end }} - {{/* - Configuration for Besu execution client - */}} + {{- /* Configuration for Besu execution client */}} {{- else if eq .Values.execution.client "besu" }} exec /opt/besu/bin/besu --network={{ .Values.global.network }} @@ -256,9 +255,7 @@ spec: --metrics.port={{ .Values.execution.metrics.port }} --metrics.addr={{ .Values.execution.metrics.host }} {{- end }} - {{/* - Configuration for Erigon execution client - */}} + {{- /* Configuration for Erigon execution client */}} {{- else if eq .Values.execution.client "erigon" }} exec erigon --datadir=/data/execution @@ -473,6 +470,8 @@ spec: --rest-api-cors-origins={{ .Values.beacon.restApi.corsOrigins | join "," }} {{- end }} --p2p-enabled=true + --p2p-peer-upper-bound={{ .Values.beacon.targetPeers }} + --p2p-peer-lower-bound={{ .Values.beacon.targetPeersMin }} {{- if .Values.global.p2pNodePort.enabled }} --p2p-advertised-port=$EXTERNAL_BEACON_PORT --p2p-advertised-ip=$EXTERNAL_IP diff --git a/charts/execution-beacon/values.yaml b/charts/execution-beacon/values.yaml index 4bb72f18a..b34f2455d 100644 --- a/charts/execution-beacon/values.yaml +++ b/charts/execution-beacon/values.yaml @@ -12,7 +12,7 @@ global: execution: nethermind: repository: "nethermind/nethermind" - tag: "1.22.0" + tag: "1.24.0" geth: repository: "ethereum/client-go" tag: "v1.13.5" @@ -28,7 +28,7 @@ global: tag: "v4.1.1" teku: repository: "consensys/teku" - tag: "23.11.0" + tag: "23.12.1" lighthouse: repository: "sigp/lighthouse" tag: "v4.5.0" @@ -37,7 +37,7 @@ global: tag: "multiarch-v23.11.0" lodestar: repository: "chainsafe/lodestar" - tag: "v1.12.0" + tag: "v1.12.1" ## JSON Web Token (JWT) authentication is used to secure the communication ## between the beacon node and execution client. You can generate a JWT using @@ -490,6 +490,7 @@ beacon: portName: "rpc" targetPeers: 50 + targetPeersMin: 40 ## Sets the total difficulty to manual overrides the default ## TERMINAL_TOTAL_DIFFICULTY value. WARNING: This flag should be used only if you diff --git a/charts/lodestar/.helmignore b/charts/lodestar/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/lodestar/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/lodestar/Chart.yaml b/charts/lodestar/Chart.yaml new file mode 100644 index 000000000..c503c124a --- /dev/null +++ b/charts/lodestar/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: lodestar +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 diff --git a/charts/lodestar/templates/_helpers.tpl b/charts/lodestar/templates/_helpers.tpl new file mode 100644 index 000000000..ba8f988be --- /dev/null +++ b/charts/lodestar/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "lodestar.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "lodestar.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "lodestar.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "lodestar.labels" -}} +helm.sh/chart: {{ include "lodestar.chart" . }} +{{ include "lodestar.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "lodestar.selectorLabels" -}} +app.kubernetes.io/name: {{ include "lodestar.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "lodestar.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "lodestar.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/lodestar/templates/configmap.yaml b/charts/lodestar/templates/configmap.yaml new file mode 100644 index 000000000..111c9e6cb --- /dev/null +++ b/charts/lodestar/templates/configmap.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: "{{ include "lodestar.fullname" . }}" + labels: + {{- include "lodestar.labels" . | nindent 4 }} +data: + run.sh: | + #!/bin/sh + + for f in /home/charon/validator_keys/keystore-*.json; do + echo "Importing key ${f}" + + # Import keystore with password. + node /usr/app/packages/cli/bin/lodestar validator import \ + --dataDir="/opt/data" \ + --network="$NETWORK" \ + --importKeystores="$f" \ + --importKeystoresPassword="${f//json/txt}" + done + + echo "Imported all keys" + + exec node /usr/app/packages/cli/bin/lodestar validator \ + --dataDir="/opt/data" \ + --network="$NETWORK" \ + --metrics=true \ + --metrics.address="0.0.0.0" \ + --metrics.port={{ .Values.service.ports.metrics }} \ + --beaconNodes="$BEACON_NODE_ADDRESS" \ + --builder="$BUILDER_API_ENABLED" \ + --builder.selection="$BUILDER_SELECTION" \ + --distributed diff --git a/charts/lodestar/templates/ingress.yaml b/charts/lodestar/templates/ingress.yaml new file mode 100644 index 000000000..720a7e104 --- /dev/null +++ b/charts/lodestar/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "lodestar.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "lodestar.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/lodestar/templates/service.yaml b/charts/lodestar/templates/service.yaml new file mode 100644 index 000000000..fb0755924 --- /dev/null +++ b/charts/lodestar/templates/service.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: "{{ include "lodestar.fullname" . }}" + labels: + {{- include "lodestar.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.ports.p2p }} + targetPort: p2p + protocol: TCP + name: p2p + - port: {{ .Values.service.ports.metrics }} + targetPort: metrics + protocol: TCP + name: metrics + - port: {{ .Values.service.ports.http }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "lodestar.selectorLabels" . | nindent 4 }} diff --git a/charts/lodestar/templates/serviceaccount.yaml b/charts/lodestar/templates/serviceaccount.yaml new file mode 100644 index 000000000..bb615f853 --- /dev/null +++ b/charts/lodestar/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "lodestar.serviceAccountName" . }} + labels: + {{- include "lodestar.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/lodestar/templates/statefulset.yaml b/charts/lodestar/templates/statefulset.yaml new file mode 100644 index 000000000..2f5f6dee6 --- /dev/null +++ b/charts/lodestar/templates/statefulset.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "lodestar.fullname" . }} + labels: + {{- include "lodestar.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "lodestar.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "lodestar.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "lodestar.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - "/bin/sh" + - "-c" + - "/opt/lodestar/run.sh" + ports: + - name: p2p + containerPort: {{ .Values.service.ports.p2p }} + protocol: TCP + - name: metrics + containerPort: {{ .Values.service.ports.metrics }} + protocol: TCP + - name: http + containerPort: {{ .Values.service.ports.http }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - mountPath: /opt/lodestar + name: data + - mountPath: /home/charon/validator_keys + name: validator-keys + env: + {{- toYaml .Values.env | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: data + configMap: + name: {{ include "lodestar.fullname" . }} + defaultMode: 0755 + - name: validator-keys + projected: + sources: + - secret: + name: {{ .Values.secrets.validatorKeys }} diff --git a/charts/lodestar/values.yaml b/charts/lodestar/values.yaml new file mode 100644 index 000000000..0d769dc5c --- /dev/null +++ b/charts/lodestar/values.yaml @@ -0,0 +1,100 @@ +# Default values for lodestar. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: chainsafe/lodestar + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "v1.11.3" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +env: + - name: "BEACON_NODE_ADDRESS" + value: "" + - name: "NETWORK" + value: "holesky" + - name: "BUILDER_API_ENABLED" + value: "true" + - name: "BUILDER_SELECTION" + value: "builderonly" + +# -- Kubernetes secrets names +secrets: + # -- validators keys + validatorKeys: "validator-keys" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: + {} + # fsGroup: 2000 + +securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + ports: + metrics: 5064 + p2p: 9000 + http: 9596 + +ingress: + enabled: false + className: "" + annotations: + {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: + {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +livnessProbe: null +readinessProbe: null From 60fcdb0c5c461836882a3dd5a2dbbdf2a64199b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:08:49 +0100 Subject: [PATCH 11/34] Features/quantstamp audit cleanup prep (#81) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * QS Audit, see PR for details/impacts * fix reference * fix configmap * revert chown on init due to init-nimbus issues * TWI-10 fixes * Added restrictions for dirk/vouch * Added more missing restrictions for containers * Taking updated configmap from cleanup-prep * Including changes from latest in cleanup-prep * changing to global * add staking-prefix to secrets * move security ctx to global in values, and move pod sctx to higher level * add defaults to init * fix values.yml * change defaults * move fsGroup to podSecurityContext level * move capabilities to the container level * remove root fs from pod level ctx * remove read only fs from w3s container * update mevboost stx * update stx for exec and beacon * add missing stx * change init.sh * check if chown is required * remove root fs from execution container * fix promrules * remove rootfs from beacon containers * add additonal default rules for nimbus and lodestar * use el flag instead * remove default sa from vals * fix sa * add stx to validators * fix values * use global * fixes * fix pubkeys * fix perms --------- Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> Co-authored-by: Mark Liu --- charts/dirk/templates/statefulset.yaml | 24 ++++++- .../execution-beacon/templates/configmap.yaml | 4 +- .../templates/prometheusrules.yaml | 46 ++++++++++---- .../templates/statefulset.yaml | 63 +++++++++---------- charts/execution-beacon/values.yaml | 20 ++++-- charts/mev-boost/templates/deployment.yaml | 4 +- charts/mev-boost/values.yaml | 28 +++++---- charts/validators/templates/_helpers.tpl | 2 +- .../validators/templates/external-secret.yaml | 2 +- .../validators/templates/serviceaccount.yaml | 2 +- charts/validators/templates/statefulset.yaml | 24 ++++--- charts/validators/values.yaml | 24 ++++--- charts/vouch/templates/statefulset.yaml | 16 ++++- .../web3signer/templates/external-secret.yaml | 10 +-- charts/web3signer/templates/statefulset.yaml | 19 +++--- charts/web3signer/values.yaml | 32 +++++----- 16 files changed, 200 insertions(+), 120 deletions(-) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index 20ae5a29d..823485bdf 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -80,7 +80,13 @@ spec: image: "{{ .Values.cliImage.repository }}:{{ .Values.cliImage.tag }}" imagePullPolicy: {{ .Values.cliImage.pullPolicy }} securityContext: - runAsUser: 0 + runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} + runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} + fsGroup: {{ .Values.global.securityContext.fsGroup | default 10000 }} + readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} + capabilities: + drop: + - ALL envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -105,7 +111,13 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: - runAsUser: 0 + runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} + runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} + fsGroup: {{ .Values.global.securityContext.fsGroup | default 10000 }} + readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} + capabilities: + drop: + - ALL envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -150,7 +162,13 @@ spec: image: "{{ .Values.cliImage.repository }}:{{ .Values.cliImage.tag }}" imagePullPolicy: {{ .Values.cliImage.pullPolicy }} securityContext: - runAsUser: 0 + runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} + runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} + fsGroup: {{ .Values.global.securityContext.fsGroup | default 10000 }} + readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} + capabilities: + drop: + - ALL envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} diff --git a/charts/execution-beacon/templates/configmap.yaml b/charts/execution-beacon/templates/configmap.yaml index 6e2baab2f..a58bd0b81 100644 --- a/charts/execution-beacon/templates/configmap.yaml +++ b/charts/execution-beacon/templates/configmap.yaml @@ -9,8 +9,8 @@ data: init.sh: | #!/bin/sh {{- if or (and .Values.execution.persistence.enabled .Values.execution.initChownData) (and .Values.beacon.persistence.enabled .Values.beacon.initChownData) (.Values.sharedPersistence.enabled) }} - mkdir -p /data && chown -R {{ .Values.global.securityContext.runAsUser }}:{{ .Values.global.securityContext.runAsUser }} /data; - mkdir -p /data/beacon && chown -R {{ .Values.global.securityContext.runAsUser }}:{{ .Values.global.securityContext.runAsUser }} /data/beacon; + # mkdir -p /data && chown -R {{ .Values.global.podSecurityContext.runAsUser }}:{{ .Values.global.podSecurityContext.runAsUser }} /data; + mkdir -p /data/beacon && chown -R {{ .Values.global.podSecurityContext.runAsUser }}:{{ .Values.global.podSecurityContext.runAsUser }} /data/beacon; {{- end }} echo "Namespace: ${POD_NAMESPACE} Pod: ${POD_NAME}"; {{- if .Values.global.p2pNodePort.enabled }} diff --git a/charts/execution-beacon/templates/prometheusrules.yaml b/charts/execution-beacon/templates/prometheusrules.yaml index 77b880355..d9afee3ad 100644 --- a/charts/execution-beacon/templates/prometheusrules.yaml +++ b/charts/execution-beacon/templates/prometheusrules.yaml @@ -4,23 +4,23 @@ apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: {{ include "common.names.fullname" . }}-execution - {{- if .Values.execution.metrics.prometheusRule.namespace }} - namespace: {{ .Values.execution.metrics.prometheusRule.namespace }} + {{- if .Values.global.metrics.prometheusRule.namespace }} + namespace: {{ .Values.global.metrics.prometheusRule.namespace }} {{- else }} namespace: {{ .Release.Namespace | quote }} {{- end }} labels: {{- include "common.labels.standard" . | nindent 4 }} - {{- if .Values.execution.metrics.prometheusRule.additionalLabels }} - {{- toYaml .Values.execution.metrics.prometheusRule.additionalLabels | nindent 4 }} + {{- if .Values.global.metrics.prometheusRule.additionalLabels }} + {{- toYaml .Values.global.metrics.prometheusRule.additionalLabels | nindent 4 }} {{- end }} spec: groups: - {{- with .Values.execution.metrics.prometheusRule.rules }} + {{- with .Values.global.metrics.prometheusRule.rules }} - name: {{ include "common.names.fullname" $ }} rules: {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - {{- if .Values.execution.metrics.prometheusRule.default }} + {{- if .Values.global.metrics.prometheusRule.default }} {{- if eq .Values.execution.client "geth" }} - name: {{ include "common.names.fullname" $ }}-default rules: @@ -50,23 +50,23 @@ apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: {{ include "common.names.fullname" . }}-beacon - {{- if .Values.beacon.metrics.prometheusRule.namespace }} - namespace: {{ .Values.beacon.metrics.prometheusRule.namespace }} + {{- if .Values.global.metrics.prometheusRule.namespace }} + namespace: {{ .Values.global.metrics.prometheusRule.namespace }} {{- else }} namespace: {{ .Release.Namespace | quote }} {{- end }} labels: {{- include "common.labels.standard" . | nindent 4 }} - {{- if .Values.beacon.metrics.prometheusRule.additionalLabels }} - {{- toYaml .Values.beacon.metrics.prometheusRule.additionalLabels | nindent 4 }} + {{- if .Values.global.metrics.prometheusRule.additionalLabels }} + {{- toYaml .Values.global.metrics.prometheusRule.additionalLabels | nindent 4 }} {{- end }} spec: groups: - {{- with .Values.beacon.metrics.prometheusRule.rules }} + {{- with .Values.global.metrics.prometheusRule.rules }} - name: {{ include "common.names.fullname" $ }} rules: {{- tpl (toYaml .) $ | nindent 8 }} {{- end }} - {{- if .Values.beacon.metrics.prometheusRule.default }} + {{- if .Values.global.metrics.prometheusRule.default }} {{- if eq .Values.beacon.client "lighthouse" }} - name: {{ include "common.names.fullname" $ }}-default rules: @@ -148,6 +148,28 @@ spec: annotations: summary: Teku beacon node is out of sync description: Check {{ printf "{{ $labels.pod }}" }} beacon node in namespace {{ printf "{{ $labels.namespace }}" }} + {{- else if eq .Values.beacon.client "nimbus" }} + - name: {{ include "common.names.fullname" $ }}-default + rules: + - alert: NimbusBeaconNodeDown + expr: up{job='{{ include "common.names.fullname" . }}'} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: Nimbus beacon node is down + description: Check {{ printf "{{ $labels.pod }}" }} beacon node in namespace {{ printf "{{ $labels.namespace }}" }} + {{- else if eq .Values.beacon.client "lodestar" }} + - name: {{ include "common.names.fullname" $ }}-default + rules: + - alert: LodestarBeaconNodeDown + expr: up{job='{{ include "common.names.fullname" . }}'} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: Lodestar beacon node is down + description: Check {{ printf "{{ $labels.pod }}" }} beacon node in namespace {{ printf "{{ $labels.namespace }}" }} {{- end }} {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/execution-beacon/templates/statefulset.yaml b/charts/execution-beacon/templates/statefulset.yaml index a1b83fa27..0120ba882 100644 --- a/charts/execution-beacon/templates/statefulset.yaml +++ b/charts/execution-beacon/templates/statefulset.yaml @@ -37,25 +37,20 @@ spec: nodeSelector: {{ toYaml . | nindent 8 | trim }} {{- end }} - {{- with .Values.global.securityContext }} securityContext: - {{ toYaml . | nindent 8 | trim }} - {{- end }} + {{- toYaml .Values.global.podSecurityContext | nindent 8 }} {{- if .Values.global.terminationGracePeriodSeconds }} terminationGracePeriodSeconds: {{ .Values.global.terminationGracePeriodSeconds }} {{- end }} serviceAccountName: {{ include "common.names.fullname" . }} priorityClassName: {{ .Values.global.priorityClassName | quote }} initContainers: - {{/* - Container used for initialising data directory and setting right values for ports and external_ip - */}} + {{/* Container used for initialising data directory and setting right values for ports and external_ip */}} - name: init image: "{{ .Values.global.initImage.repository }}:{{ .Values.global.initImage.tag }}" imagePullPolicy: {{ .Values.global.initImage.pullPolicy }} securityContext: - runAsNonRoot: false - runAsUser: 0 + {{- toYaml .Values.global.securityContext | nindent 12 }} env: - name: POD_IP valueFrom: @@ -88,13 +83,13 @@ spec: mountPath: /env - name: scripts-init mountPath: /scripts - {{/* - Container used for initialising nimbus checkpoint sync - */}} + {{/* Container used for initialising nimbus checkpoint sync */}} {{- if and (eq .Values.beacon.client "nimbus") .Values.beacon.checkPointSync.enabled }} - name: init-nimbus image: "{{ (pluck .Values.beacon.client .Values.global.image.beacon | first ).repository }}:{{ (pluck .Values.beacon.client .Values.global.image.beacon | first ).tag }}" imagePullPolicy: {{ .Values.beacon.imagePullPolicy }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 12 }} command: - sh - -ac @@ -115,12 +110,17 @@ spec: {{- end }} {{- end }} containers: - {{/* - Container used for Execution clients: Nethermind, Geth, Besu, Erigon - */}} + {{/* Container used for Execution clients: Nethermind, Geth, Besu, Erigon */}} - name: execution image: "{{ (pluck .Values.execution.client .Values.global.image.execution | first ).repository }}:{{ (pluck .Values.execution.client .Values.global.image.execution | first ).tag }}" imagePullPolicy: {{ .Values.global.image.imagePullPolicy }} + securityContext: + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL {{- if .Values.global.externalSecrets.enabled }} envFrom: - secretRef: @@ -372,15 +372,18 @@ spec: resources: {{ toYaml . | nindent 12 | trim }} {{- end }} - {{/* - Container used for Beacon clients: Prysm, Teku, Lighthouse, Nimbus, Lodestar - */}} + {{/* Container used for Beacon clients: Prysm, Teku, Lighthouse, Nimbus, Lodestar */}} - name: beacon image: "{{ (pluck .Values.beacon.client .Values.global.image.beacon | first ).repository }}:{{ (pluck .Values.beacon.client .Values.global.image.beacon | first ).tag }}" imagePullPolicy: {{ .Values.global.image.imagePullPolicy }} - {{/* - Configuration for Prysm beacon client - */}} + securityContext: + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL + {{/* Configuration for Prysm beacon client */}} {{- if eq .Values.beacon.client "prysm" }} args: - "--datadir=/data/beacon" @@ -432,9 +435,7 @@ spec: {{- range .Values.beacon.extraFlags }} - {{ . | quote }} {{- end }} - {{/* - Configuration for Teku beacon client - */}} + {{/* Configuration for Teku beacon client */}} {{- else if eq .Values.beacon.client "teku" }} command: - sh @@ -491,9 +492,7 @@ spec: {{- range .Values.beacon.extraFlags }} {{ . | quote }} {{- end }} - {{/* - Configuration for Lighthouse beacon client - */}} + {{/* Configuration for Lighthouse beacon client */}} {{- else if eq .Values.beacon.client "lighthouse" }} command: - sh @@ -548,9 +547,7 @@ spec: {{- range .Values.beacon.extraFlags }} {{ . }} {{- end }} - {{/* - Configuration for Nimbus beacon client - */}} + {{/* Configuration for Nimbus beacon client */}} {{- else if eq .Values.beacon.client "nimbus" }} command: - sh @@ -581,7 +578,7 @@ spec: {{- if .Values.beacon.suggestedFeeRecipient }} --suggested-fee-recipient={{ .Values.beacon.suggestedFeeRecipient }} {{- end }} - --web3-url=http://localhost:{{ .Values.execution.jsonrpc.engine.port }} + --el=http://localhost:{{ .Values.execution.jsonrpc.engine.port }} --max-peers={{ .Values.beacon.targetPeers }} --enr-auto-update=true {{- if .Values.global.p2pNodePort.enabled }} @@ -600,9 +597,7 @@ spec: {{- range .Values.beacon.extraFlags }} {{ . }} {{- end }} - {{/* - Configuration for Lodestar beacon client - */}} + {{/* Configuration for Lodestar beacon client */}} {{- else if eq .Values.beacon.client "lodestar" }} command: - sh @@ -719,6 +714,8 @@ spec: - name: ethsider image: "{{ .Values.global.ethsider.repository }}:{{ .Values.global.ethsider.tag }}" imagePullPolicy: {{ .Values.global.ethsider.pullPolicy }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 12 }} env: - name: EXECUTION_ENDPOINT value: "http://localhost:{{ .Values.execution.jsonrpc.http.port }}" diff --git a/charts/execution-beacon/values.yaml b/charts/execution-beacon/values.yaml index b34f2455d..ea05ae7c2 100644 --- a/charts/execution-beacon/values.yaml +++ b/charts/execution-beacon/values.yaml @@ -15,13 +15,13 @@ global: tag: "1.24.0" geth: repository: "ethereum/client-go" - tag: "v1.13.5" + tag: "v1.13.8" bseu: repository: "hyperledger/besu" tag: "23.10.2" erigon: repository: "thorax/erigon" - tag: "v2.54.0" + tag: "v2.55.1" beacon: prysm: repository: "gcr.io/prylabs-dev/prysm/beacon-chain" @@ -34,10 +34,10 @@ global: tag: "v4.5.0" nimbus: repository: "statusim/nimbus-eth2" - tag: "multiarch-v23.11.0" + tag: "multiarch-v24.1.0" lodestar: repository: "chainsafe/lodestar" - tag: "v1.12.1" + tag: "v1.13.0" ## JSON Web Token (JWT) authentication is used to secure the communication ## between the beacon node and execution client. You can generate a JWT using @@ -242,10 +242,18 @@ global: ## Pod Security Context ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ ## + podSecurityContext: + runAsNonRoot: true + runAsUser: 10000 + fsGroup: 10000 + securityContext: + readOnlyRootFilesystem: true runAsNonRoot: true - fsGroup: 1001 - runAsUser: 1001 + runAsUser: 10000 + capabilities: + drop: + - ALL execution: client: nethermind diff --git a/charts/mev-boost/templates/deployment.yaml b/charts/mev-boost/templates/deployment.yaml index 0a7d6ede8..30f2ba577 100644 --- a/charts/mev-boost/templates/deployment.yaml +++ b/charts/mev-boost/templates/deployment.yaml @@ -26,11 +26,11 @@ spec: {{- end }} serviceAccountName: {{ include "common.names.serviceAccountName" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml .Values.global.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} + {{- toYaml .Values.global.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} args: diff --git a/charts/mev-boost/values.yaml b/charts/mev-boost/values.yaml index 61a94600f..b84a85e35 100644 --- a/charts/mev-boost/values.yaml +++ b/charts/mev-boost/values.yaml @@ -13,7 +13,22 @@ global: serviceAccount: create: true - + ## Pod Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + podSecurityContext: + runAsNonRoot: true + runAsUser: 10000 + fsGroup: 10000 + + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL + # Relay URLs relays: mainnet: @@ -54,17 +69,6 @@ serviceAccount: podAnnotations: {} -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - service: type: ClusterIP port: 18550 diff --git a/charts/validators/templates/_helpers.tpl b/charts/validators/templates/_helpers.tpl index e379a2aea..c7eaa1c6f 100644 --- a/charts/validators/templates/_helpers.tpl +++ b/charts/validators/templates/_helpers.tpl @@ -128,7 +128,7 @@ Validator graffiti {{- end }} {{- define "web3signer" -}} -{{- if $.Values.web3signerEndpoint }}{{ $.Values.web3signerEndpoint }}{{- else }}http://{{ $.Values.global.project }}-web3signer-{{ $.Values.global.owner }}-web3signer:6174 +{{- if $.Values.web3signerEndpoint }}{{ $.Values.web3signerEndpoint }}{{- else }}http://{{ $.Values.global.project }}-{{ $.Values.global.label }}-web3signer:6174 {{- end }} {{- end }} diff --git a/charts/validators/templates/external-secret.yaml b/charts/validators/templates/external-secret.yaml index a94ec457a..20892cd43 100644 --- a/charts/validators/templates/external-secret.yaml +++ b/charts/validators/templates/external-secret.yaml @@ -17,7 +17,7 @@ spec: {{- if empty .Values.externalSecrets.data }} - secretKey: ESO_DB_KEYSTORE_URL remoteRef: - key: {{ .Values.global.owner }}-external-secrets + key: staking-{{ .Values.global.label }}-external-secrets property: dbKeystoreUrl {{- else }} {{- range .Values.externalSecrets.data }} diff --git a/charts/validators/templates/serviceaccount.yaml b/charts/validators/templates/serviceaccount.yaml index 241cc4a97..c083b02b5 100644 --- a/charts/validators/templates/serviceaccount.yaml +++ b/charts/validators/templates/serviceaccount.yaml @@ -10,4 +10,4 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/validators/templates/statefulset.yaml b/charts/validators/templates/statefulset.yaml index 7c795f528..cf9cea55b 100644 --- a/charts/validators/templates/statefulset.yaml +++ b/charts/validators/templates/statefulset.yaml @@ -64,10 +64,8 @@ spec: nodeSelector: {{ toYaml . | nindent 8 | trim }} {{- end }} - {{- with $root.Values.securityContext }} securityContext: - {{ toYaml . | nindent 8 | trim }} - {{- end }} + {{- toYaml $root.Values.global.podSecurityContext | nindent 8 }} serviceAccountName: {{ template "validators.serviceAccountName" $root }} priorityClassName: {{ $root.Values.priorityClassName | quote }} initContainers: @@ -75,7 +73,7 @@ spec: image: "{{ $root.Values.cliImage.repository }}:{{ $root.Values.cliImage.tag }}" imagePullPolicy: {{ $root.Values.cliImage.pullPolicy }} securityContext: - runAsUser: 0 + {{- toYaml $root.Values.global.securityContext | nindent 12 }} env: - name: WEB3SIGNER_URL value: {{ include "web3signer" $root }} @@ -111,12 +109,11 @@ spec: image: "{{ $root.Values.initImageBusybox.repository }}:{{ $root.Values.initImageBusybox.tag }}" imagePullPolicy: {{ $root.Values.initImageBusybox.pullPolicy }} securityContext: - runAsUser: 0 + {{- toYaml $root.Values.global.securityContext | nindent 12 }} command: - sh - -c - > - ls -lha /data; mkdir -p /data/lighthouse/validators; cp /data/validator_definitions.yml /data/lighthouse/validators/validator_definitions.yml; cat /data/signer_keys.yml > /data/config; @@ -125,9 +122,9 @@ spec: cat /data/proposerConfig.yaml; formatted_content=$(cat /data/signer_keys.yml | grep -o '".*"' | sed -e 's/"//g' | tr ',' '\n'); echo "$formatted_content" > /data/pubkeys.txt; - echo '{"externalSigner.pubkeys": ['"$(awk '{printf "%s\"%s\"", (NR==1 ? "" : ", "), $0}' pubkeys.txt)"']}' > /data/rcconfig.json; - cat /data/pubkeys.txt; - chown -R {{ $root.Values.securityContext.runAsUser }}:{{ $root.Values.securityContext.runAsUser }} /data; + echo '{"externalSigner.pubkeys": ['"$(awk '{printf "%s\"%s\"", (NR==1 ? "" : ", "), $0}' /data/pubkeys.txt)"']}' > /data/rcconfig.json; + cat /data/rcconfig.json; + ls -lha /data; {{- if eq $root.Values.type "nimbus" }} find /data/nimbus -type d -name '0x*' -exec chmod 0600 {}/remote_keystore.json \; {{- end }} @@ -139,6 +136,8 @@ spec: - name: watcher image: "{{ $root.Values.cliImage.repository }}:{{ $root.Values.cliImage.tag }}" imagePullPolicy: {{ $root.Values.cliImage.pullPolicy }} + securityContext: + {{- toYaml $root.Values.global.securityContext | nindent 12 }} env: - name: WEB3SIGNER_URL value: {{ include "web3signer" $root }} @@ -164,6 +163,13 @@ spec: - name: validator image: "{{ (pluck $root.Values.type $root.Values.image | first ).repository }}:{{ (pluck $root.Values.type $root.Values.image | first ).tag }}" imagePullPolicy: {{ $root.Values.image.pullPolicy }} + securityContext: + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL {{- if eq $root.Values.type "lodestar" }} command: - sh diff --git a/charts/validators/values.yaml b/charts/validators/values.yaml index 9cbc40a78..369c5aba4 100644 --- a/charts/validators/values.yaml +++ b/charts/validators/values.yaml @@ -7,6 +7,21 @@ global: project: "" imagePullSecrets: [] network: mainnet + ## Pod Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + podSecurityContext: + runAsNonRoot: true + runAsUser: 10000 + fsGroup: 10000 + + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL ## Provide a name in place of operator for `app:` labels ## @@ -16,13 +31,6 @@ nameOverride: "" ## fullnameOverride: "" -## Pod Security Context -## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ -## -securityContext: - fsGroup: 1001 - runAsUser: 1001 - ## RBAC configuration. ## ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/ ## @@ -144,7 +152,7 @@ terminationGracePeriodSeconds: 120 serviceAccount: # Specifies whether a service account should be created create: true - name: "validators" + name: "" # Annotations to add to the service account annotations: {} diff --git a/charts/vouch/templates/statefulset.yaml b/charts/vouch/templates/statefulset.yaml index 3c3844476..a0d6586c8 100644 --- a/charts/vouch/templates/statefulset.yaml +++ b/charts/vouch/templates/statefulset.yaml @@ -70,7 +70,13 @@ spec: image: "{{ .Values.cliImage.repository }}:{{ .Values.cliImage.tag }}" imagePullPolicy: {{ .Values.cliImage.pullPolicy }} securityContext: - runAsUser: 0 + runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} + runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} + fsGroup: {{ .Values.global.securityContext.fsGroup | default 10000 }} + readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} + capabilities: + drop: + - ALL envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -94,7 +100,13 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: - runAsUser: 0 + runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} + runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} + fsGroup: {{ .Values.global.securityContext.fsGroup | default 10000 }} + readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} + capabilities: + drop: + - ALL envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} diff --git a/charts/web3signer/templates/external-secret.yaml b/charts/web3signer/templates/external-secret.yaml index 8a2a19242..831ae99b9 100644 --- a/charts/web3signer/templates/external-secret.yaml +++ b/charts/web3signer/templates/external-secret.yaml @@ -17,23 +17,23 @@ spec: {{- if empty .Values.externalSecrets.data }} - secretKey: ESO_DECRYPTION_KEY remoteRef: - key: {{ .Values.global.owner }}-external-secrets + key: staking-{{ .Values.global.label }}-external-secrets property: decryptionKey - secretKey: ESO_DB_KEYSTORE_URL remoteRef: - key: {{ .Values.global.owner }}-external-secrets + key: staking-{{ .Values.global.label }}-external-secrets property: dbKeystoreUrl - secretKey: ESO_DB_URL remoteRef: - key: {{ .Values.global.owner }}-external-secrets + key: staking-{{ .Values.global.label }}-external-secrets property: dbUrl - secretKey: ESO_DB_USERNAME remoteRef: - key: {{ .Values.global.owner }}-external-secrets + key: staking-{{ .Values.global.label }}-external-secrets property: dbUsername - secretKey: ESO_DB_PASSWORD remoteRef: - key: {{ .Values.global.owner }}-external-secrets + key: staking-{{ .Values.global.label }}-external-secrets property: dbPassword {{- else }} {{- range .Values.externalSecrets.data }} diff --git a/charts/web3signer/templates/statefulset.yaml b/charts/web3signer/templates/statefulset.yaml index dde9afaa4..fb651564f 100644 --- a/charts/web3signer/templates/statefulset.yaml +++ b/charts/web3signer/templates/statefulset.yaml @@ -37,19 +37,19 @@ spec: {{- end }} serviceAccountName: {{ include "web3signer.serviceAccountName" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml .Values.global.podSecurityContext | nindent 8 }} initContainers: - name: init image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: - runAsUser: 0 + {{- toYaml .Values.global.securityContext | nindent 12 }} command: - sh - -ac - > mkdir -p /data/web3signer /data/keystore; - chown -R {{ .Values.podSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} /data/web3signer + chown -R {{ .Values.global.podSecurityContext.runAsUser }}:{{ .Values.global.podSecurityContext.fsGroup }} /data/web3signer envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -60,7 +60,7 @@ spec: image: "{{ .Values.flywayImage.repository }}:{{ .Values.flywayImage.tag }}" imagePullPolicy: {{ .Values.flywayImage.pullPolicy }} securityContext: - runAsUser: 0 + {{- toYaml .Values.global.securityContext | nindent 12 }} args: - -user=$(ESO_DB_USERNAME) - -password=$(ESO_DB_PASSWORD) @@ -76,7 +76,7 @@ spec: image: "{{ .Values.cliImage.repository }}:{{ .Values.cliImage.tag }}" imagePullPolicy: {{ .Values.cliImage.pullPolicy }} securityContext: - runAsUser: 0 + {{- toYaml .Values.global.securityContext | nindent 12 }} envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -87,7 +87,12 @@ spec: containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} args: @@ -141,7 +146,7 @@ spec: image: "{{ .Values.cliImage.repository }}:{{ .Values.cliImage.tag }}" imagePullPolicy: {{ .Values.cliImage.pullPolicy }} securityContext: - runAsUser: 0 + {{- toYaml .Values.global.securityContext | nindent 12 }} envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} diff --git a/charts/web3signer/values.yaml b/charts/web3signer/values.yaml index ab14d81c7..120cd068f 100644 --- a/charts/web3signer/values.yaml +++ b/charts/web3signer/values.yaml @@ -3,9 +3,24 @@ # Declare variables to be passed into your templates. global: - owner: "" + label: "" serviceAccount: create: true + ## Pod Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + podSecurityContext: + runAsNonRoot: true + runAsUser: 10000 + fsGroup: 10000 + + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL replicaCount: 3 @@ -116,21 +131,6 @@ serviceAccount: podAnnotations: {} -## Pod Security Context -## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ -## -podSecurityContext: - fsGroup: 1000 - runAsUser: 1000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - service: type: ClusterIP port: 80 From 71495311b11a83f91bfe0706bec0264641544b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 18 Jan 2024 09:31:00 +0100 Subject: [PATCH 12/34] Start using Nethermind chiseled image (#75) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * trying out chiseled nm * fix execution commands and args * fix ends * fix env-nodeport * try static envs * fix variable scope * fix extra brackets * Add ingress (#78) * Add ingress to execution-beacon * Set Service Type configurable * Add global.ingress values and test template generation * fix envFrom - redundant * Fix init container when p2p type is LoadBalancer (#79) * Fix init container when p2p type is LoadBalancer * Remove extra dash in POD_PREFIX_NAME variable * Fix syntax errors (#80) * Add support for externalTrafficPolicy on execution-beacon svc (#82) * Fix erigon's metrics flags indentation * Generate non-nethermind extra flags when nethermind is not chosen * Remove quotes from Nethermind's extra flags --------- Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> Signed-off-by: Miguel Tenorio <46824157+AntiD2ta@users.noreply.github.com> Co-authored-by: Miguel Tenorio <46824157+AntiD2ta@users.noreply.github.com> Co-authored-by: AntiD2ta --- .../execution-beacon/templates/configmap.yaml | 43 +++--- .../execution-beacon/templates/ingress.yaml | 56 ++++++++ .../execution-beacon/templates/service.yaml | 8 +- .../templates/statefulset.yaml | 134 +++++++++--------- charts/execution-beacon/values.yaml | 41 ++++++ 5 files changed, 193 insertions(+), 89 deletions(-) create mode 100644 charts/execution-beacon/templates/ingress.yaml diff --git a/charts/execution-beacon/templates/configmap.yaml b/charts/execution-beacon/templates/configmap.yaml index a58bd0b81..e0e501f58 100644 --- a/charts/execution-beacon/templates/configmap.yaml +++ b/charts/execution-beacon/templates/configmap.yaml @@ -15,25 +15,32 @@ data: echo "Namespace: ${POD_NAMESPACE} Pod: ${POD_NAME}"; {{- if .Values.global.p2pNodePort.enabled }} {{- if eq .Values.global.p2pNodePort.type "LoadBalancer" }} - until [ -n "$(kubectl -n ${POD_NAMESPACE} get svc/${POD_NAME} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do echo "Waiting for load balancer to get an IP" && sleep 10; done; - export EXTERNAL_EXECUTION_PORT=$(kubectl -n ${POD_NAMESPACE} get services -l "client=execution,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}'); - export EXTERNAL_BEACON_PORT=$(kubectl -n ${POD_NAMESPACE} get services -l "client=beacon,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}'); - export EXTERNAL_IP=$(kubectl -n ${POD_NAMESPACE} get svc/${POD_NAME} -o jsonpath='{.status.loadBalancer.ingress[0].ip}'); + export POD_NUMBER_SUFFIX=$(echo "${POD_NAME}" | grep -o -E '[0-9]+$') + export POD_PREFIX_NAME={{ include "common.names.fullname" . }} + for SERVICE_TYPE in beacon execution; do + SERVICE_NAME="${POD_PREFIX_NAME}-${SERVICE_TYPE}-${POD_NUMBER_SUFFIX}" + until [ -n "$(kubectl -n ${POD_NAMESPACE} get svc/${SERVICE_NAME} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do + echo "Waiting for load balancer to get an IP for ${SERVICE_NAME}" && sleep 10 + done + done + export EXTERNAL_EXECUTION_PORT=$(kubectl -n ${POD_NAMESPACE} get services -l "client=execution,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}') + export EXTERNAL_BEACON_PORT=$(kubectl -n ${POD_NAMESPACE} get services -l "client=beacon,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}') + export EXTERNAL_IP=$(kubectl -n ${POD_NAMESPACE} get svc/${POD_PREFIX_NAME}-beacon-${POD_NUMBER_SUFFIX} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') {{- else }} - export EXTERNAL_EXECUTION_PORT=$(kubectl get services -l "client=execution,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}'); - export EXTERNAL_BEACON_PORT=$(kubectl get services -l "client=beacon,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}'); - export EXTERNAL_IP=$(kubectl get nodes "${NODE_NAME}" -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}'); + export EXTERNAL_EXECUTION_PORT=$(kubectl get services -l "client=execution,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}') + export EXTERNAL_BEACON_PORT=$(kubectl get services -l "client=beacon,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}') + export EXTERNAL_IP=$(kubectl get nodes "${NODE_NAME}" -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}') {{- end }} - echo "EXTERNAL_EXECUTION_PORT=$EXTERNAL_EXECUTION_PORT" > /env/init-nodeport; - echo "EXTERNAL_BEACON_PORT=$EXTERNAL_BEACON_PORT" >> /env/init-nodeport; - echo "EXTERNAL_IP=$EXTERNAL_IP" >> /env/init-nodeport; - cat /env/init-nodeport; + echo "EXTERNAL_EXECUTION_PORT=$EXTERNAL_EXECUTION_PORT" > /env/init-nodeport + echo "EXTERNAL_BEACON_PORT=$EXTERNAL_BEACON_PORT" >> /env/init-nodeport + echo "EXTERNAL_IP=$EXTERNAL_IP" >> /env/init-nodeport + cat /env/init-nodeport {{- if eq .Values.beacon.client "prysm" }} - touch /data/beacon/config.yaml; - echo "p2p-host-ip: $EXTERNAL_IP" > /data/beacon/config.yaml; - echo "p2p-tcp-port: $EXTERNAL_BEACON_PORT" >> /data/beacon/config.yaml; - echo "p2p-udp-port: $EXTERNAL_BEACON_PORT" >> /data/beacon/config.yaml; - echo "Updated Prysm config.yaml file at /data/beacon/config.yaml"; + touch /data/beacon/config.yaml + echo "p2p-host-ip: $EXTERNAL_IP" > /data/beacon/config.yaml + echo "p2p-tcp-port: $EXTERNAL_BEACON_PORT" >> /data/beacon/config.yaml + echo "p2p-udp-port: $EXTERNAL_BEACON_PORT" >> /data/beacon/config.yaml + echo "Updated Prysm config.yaml file at /data/beacon/config.yaml" {{- end }} - if [ -f "/data/beacon/config.yaml" ]; then echo "Beacon client config:" && cat "/data/beacon/config.yaml"; fi; -{{- end }} \ No newline at end of file + if [ -f "/data/beacon/config.yaml" ]; then echo "Beacon client config:" && cat "/data/beacon/config.yaml"; fi +{{- end }} diff --git a/charts/execution-beacon/templates/ingress.yaml b/charts/execution-beacon/templates/ingress.yaml new file mode 100644 index 000000000..c545b1610 --- /dev/null +++ b/charts/execution-beacon/templates/ingress.yaml @@ -0,0 +1,56 @@ +{{- if .Values.global.ingress.enabled -}} + {{- $serviceName := include "common.names.fullname" . -}} + {{- $routePrefix := .Values.global.ingress.routePrefix | default "/" -}} + {{- $defaultPath := list (dict "path" $routePrefix "port" 8545 "pathType" "ImplementationSpecific") -}} + {{- $paths := .Values.global.ingress.paths | default $defaultPath -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: +{{- if .Values.global.ingress.annotations }} + annotations: +{{ toYaml .Values.global.ingress.annotations | indent 4 }} +{{- end }} + name: {{ include "common.names.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} +{{- if .Values.global.ingress.labels }} +{{ toYaml .Values.global.ingress.labels | indent 4 }} +{{- end }} +spec: + {{- if .Values.global.ingress.ingressClassName }} + ingressClassName: {{ .Values.global.ingress.ingressClassName }} + {{- end }} + rules: + {{- if .Values.global.ingress.hosts }} + {{- range $host := .Values.global.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ $p.path }} + pathType: {{ $p.pathType }} + backend: + service: + name: {{ $serviceName }} + port: + number: {{ $p.port }} + {{- end }} + {{- end }} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ $p.path }} + pathType: {{ $p.pathType }} + backend: + service: + name: {{ $serviceName }} + port: + number: {{ $p.port }} + {{- end }} + {{- end }} + {{- if .Values.global.ingress.tls }} + tls: +{{ tpl (toYaml .Values.global.ingress.tls | indent 4) . }} + {{- end }} +{{- end }} diff --git a/charts/execution-beacon/templates/service.yaml b/charts/execution-beacon/templates/service.yaml index 9a06281be..ec7dceac7 100644 --- a/charts/execution-beacon/templates/service.yaml +++ b/charts/execution-beacon/templates/service.yaml @@ -23,8 +23,11 @@ spec: clientIP: timeoutSeconds: {{ .Values.global.sessionAffinity.timeoutSeconds }} {{- end }} - type: ClusterIP -{{- if .Values.global.service.svcHeadless }} + type: {{ .Values.global.service.type }} # ClusterIP, NodePort, LoadBalancer, or ExternalName +{{- if .Values.global.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.global.service.externalTrafficPolicy }} +{{- end }} +{{- if .Values.global.service.svcHeadless}} clusterIP: None {{- end }} ports: @@ -71,4 +74,3 @@ spec: {{- end }} selector: {{- include "common.labels.matchLabels" . | nindent 4 }} - diff --git a/charts/execution-beacon/templates/statefulset.yaml b/charts/execution-beacon/templates/statefulset.yaml index 0120ba882..07a0b3776 100644 --- a/charts/execution-beacon/templates/statefulset.yaml +++ b/charts/execution-beacon/templates/statefulset.yaml @@ -114,18 +114,55 @@ spec: - name: execution image: "{{ (pluck .Values.execution.client .Values.global.image.execution | first ).repository }}:{{ (pluck .Values.execution.client .Values.global.image.execution | first ).tag }}" imagePullPolicy: {{ .Values.global.image.imagePullPolicy }} - securityContext: - readOnlyRootFilesystem: false - runAsNonRoot: true - runAsUser: 10000 - capabilities: - drop: - - ALL + {{/* Configuration for Nethermind execution client */}} + {{- if eq .Values.execution.client "nethermind" }} + args: + - --config={{ .Values.global.network }} + - --datadir=/data/execution + {{- if .Values.execution.jsonrpc.enabled }} + - --JsonRpc.Enabled={{ .Values.execution.jsonrpc.enabled }} + - --JsonRpc.EnabledModules={{ .Values.execution.jsonrpc.namespaces.nethermind | join "," }} + - --JsonRpc.Host={{ .Values.execution.jsonrpc.host }} + - --JsonRpc.Port={{ .Values.execution.jsonrpc.http.port }} + {{- if .Values.execution.healthchecks.enabled }} + - --HealthChecks.Enabled={{ .Values.execution.healthchecks.enabled }} + - --HealthChecks.Slug={{ .Values.execution.healthchecks.slug }} + - --HealthChecks.PollingInterval={{ .Values.execution.healthchecks.pollingInterval }} + - --HealthChecks.LowStorageSpaceShutdownThreshold={{ .Values.execution.healthchecks.lowStorageSpaceShutdownThreshold }} + - --HealthChecks.LowStorageSpaceWarningThreshold={{ .Values.execution.healthchecks.lowStorageSpaceWarningThreshold }} + {{- if .Values.execution.jsonrpc.websocket.enabled }} + - --JsonRpc.WebSocketsPort={{ .Values.execution.jsonrpc.websocket.port }} + {{- end }} + {{- end }} + {{- end }} + {{- if or .Values.global.JWTSecret .Values.global.externalSecrets.enabled }} {{- if .Values.global.externalSecrets.enabled }} - envFrom: - - secretRef: - name: eso-{{ include "common.names.fullname" . }} + - --JsonRpc.JwtSecretFile=/external-secrets/JWT_SECRET + {{- else }} + - --JsonRpc.JwtSecretFile=/secret/jwtsecret + {{- end }} + - --JsonRpc.EnginePort={{ .Values.execution.jsonrpc.engine.port }} + - --JsonRpc.EngineHost={{ .Values.execution.jsonrpc.host }} + - --JsonRpc.EngineEnabledModules={{ .Values.execution.jsonrpc.namespaces.nethermind | join "," }} + {{- end }} + {{- if and .Values.global.metrics.enabled .Values.execution.metrics.enabled }} + - --Metrics.Enabled={{ .Values.execution.metrics.enabled }} + - --Metrics.ExposePort={{ .Values.execution.metrics.port }} + - --Metrics.NodeName=$POD_NAME + {{- end }} + - --Network.MaxActivePeers={{ .Values.execution.targetPeers }} + {{- if not .Values.global.p2pNodePort.enabled }} + - --Network.ExternalIp=$POD_IP + - --Network.P2PPort={{ include "execution.p2pPort" . }} + - --Network.DiscoveryPort={{ include "execution.p2pPort" . }} + {{- end }} + {{- if .Values.execution.terminalTotalDifficulty }} + - --Merge.TerminalTotalDifficulty={{ .Values.execution.terminalTotalDifficulty }} {{- end }} + {{- range .Values.execution.extraFlags }} + - {{ . }} + {{- end }} + {{- else }} command: - sh - -ac @@ -133,57 +170,8 @@ spec: {{- if .Values.global.p2pNodePort.enabled }} . /env/init-nodeport; {{- end }} - {{- /* Configuration for Nethermind execution client */}} - {{- if eq .Values.execution.client "nethermind" }} - exec /nethermind/Nethermind.Runner - --config={{ .Values.global.network }} - --datadir=/data/execution - {{- if .Values.execution.jsonrpc.enabled }} - --JsonRpc.Enabled={{ .Values.execution.jsonrpc.enabled }} - --JsonRpc.EnabledModules={{ .Values.execution.jsonrpc.namespaces.nethermind | join "," }} - --JsonRpc.Host={{ .Values.execution.jsonrpc.host }} - --JsonRpc.Port={{ .Values.execution.jsonrpc.http.port }} - {{- if .Values.execution.healthchecks.enabled }} - --HealthChecks.Enabled={{ .Values.execution.healthchecks.enabled }} - --HealthChecks.Slug={{ .Values.execution.healthchecks.slug }} - --HealthChecks.PollingInterval={{ .Values.execution.healthchecks.pollingInterval }} - --HealthChecks.LowStorageSpaceShutdownThreshold={{ .Values.execution.healthchecks.lowStorageSpaceShutdownThreshold }} - --HealthChecks.LowStorageSpaceWarningThreshold={{ .Values.execution.healthchecks.lowStorageSpaceWarningThreshold }} - {{- if .Values.execution.jsonrpc.websocket.enabled }} - --JsonRpc.WebSocketsPort={{ .Values.execution.jsonrpc.websocket.port }} - {{- end }} - {{- end }} - {{- end }} - {{- if or .Values.global.JWTSecret .Values.global.externalSecrets.enabled }} - {{- if .Values.global.externalSecrets.enabled }} - --JsonRpc.JwtSecretFile=/external-secrets/JWT_SECRET - {{- else }} - --JsonRpc.JwtSecretFile=/secret/jwtsecret - {{- end }} - --JsonRpc.EnginePort={{ .Values.execution.jsonrpc.engine.port }} - --JsonRpc.EngineHost={{ .Values.execution.jsonrpc.host }} - --JsonRpc.EngineEnabledModules={{ .Values.execution.jsonrpc.namespaces.nethermind | join "," }} - {{- end }} - {{- if and .Values.global.metrics.enabled .Values.execution.metrics.enabled }} - --Metrics.Enabled={{ .Values.execution.metrics.enabled }} - --Metrics.ExposePort={{ .Values.execution.metrics.port }} - --Metrics.NodeName=$POD_NAME - {{- end }} - --Network.MaxActivePeers={{ .Values.execution.targetPeers }} - {{- if .Values.global.p2pNodePort.enabled }} - --Network.ExternalIp=$EXTERNAL_IP - --Network.P2PPort=$EXTERNAL_EXECUTION_PORT - --Network.DiscoveryPort=$EXTERNAL_EXECUTION_PORT - {{- else }} - --Network.ExternalIp=$POD_IP - --Network.P2PPort={{ include "execution.p2pPort" . }} - --Network.DiscoveryPort={{ include "execution.p2pPort" . }} - {{- end }} - {{- if .Values.execution.terminalTotalDifficulty }} - --Merge.TerminalTotalDifficulty={{ .Values.execution.terminalTotalDifficulty }} - {{- end }} - {{- /* Configuration for Geth execution client */}} - {{- else if eq .Values.execution.client "geth" }} + {{/* Configuration for Geth execution client */}} + {{- if eq .Values.execution.client "geth" }} exec geth --syncmode=snap --datadir=/data/execution @@ -290,18 +278,16 @@ spec: --ws {{- end }} {{- end }} - {{- range .Values.extraFlags }} - {{ . }} - {{- end }} {{- if .Values.global.metrics.enabled }} - --metrics - --metrics.addr={{ .Values.execution.metrics.host }} - --metrics.port={{ .Values.execution.metrics.port }} + --metrics + --metrics.addr={{ .Values.execution.metrics.host }} + --metrics.port={{ .Values.execution.metrics.port }} {{- end }} {{- end }} {{- range .Values.execution.extraFlags }} - {{ . }} + {{ . }} {{- end }} + {{- end}} env: - name: POD_IP valueFrom: @@ -311,6 +297,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + {{- if and (eq .Values.execution.client "nethermind") (.Values.global.p2pNodePort.enabled) }} + {{- range $i, $e := until (int .Values.global.replicaCount) }} + {{- $portExecution := add $.Values.global.p2pNodePort.startAtExecution $i }} + {{- if hasKey $.Values.global.p2pNodePort.replicaToNodePort ($i | toString) }} + {{- $portExecution = index $.Values.global.p2pNodePort.replicaToNodePort ($i | toString) }} + {{- end }} + - name: NETHERMIND_NETWORKCONFIG_DISCOVERYPORT + value: "{{ $portExecution }}" + - name: NETHERMIND_NETWORKCONFIG_P2PPORT + value: "{{ $portExecution }}" + {{- end }} + {{- end }} {{- if .Values.execution.javaOpts.enabled }} - name: BESU_OPTS value: {{ .Values.execution.javaOpts.maxHeapSize }} diff --git a/charts/execution-beacon/values.yaml b/charts/execution-beacon/values.yaml index ea05ae7c2..adf39ef1e 100644 --- a/charts/execution-beacon/values.yaml +++ b/charts/execution-beacon/values.yaml @@ -136,6 +136,7 @@ global: service: svcHeadless: true + type: ClusterIP sessionAffinity: # Whether to enable session affinity or not @@ -255,6 +256,46 @@ global: drop: - ALL + ingress: + enabled: false + ingressClassName: nginx + + ## Route Prefix. Can skip it if any item of path has the path defined. + ## + routePrefix: / + + annotations: {} + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules + ## If not defined the following default object will be used: + ## - path: "/" + ## port: 8545 + ## pathType: "ImplementationSpecific" + paths: [] + ## paths: + ## - path: "/execution" + ## port: 8545 + ## pathType: "Prefix" + ## - path: "/beacon" + ## port: 8080 + ## pathType: "Exact" + + ## TLS configuration for Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: execution-beacon-general-tls + # hosts: + # - nethermind.example.com + execution: client: nethermind From 0343615400b50affe86aa7a567bd2928319a76a4 Mon Sep 17 00:00:00 2001 From: Aivaras Ko Date: Thu, 18 Jan 2024 11:03:40 +0200 Subject: [PATCH 13/34] Revert "Start using Nethermind chiseled image (#75)" This reverts commit 71495311b11a83f91bfe0706bec0264641544b1f. --- .../execution-beacon/templates/configmap.yaml | 43 +++--- .../execution-beacon/templates/ingress.yaml | 56 -------- .../execution-beacon/templates/service.yaml | 8 +- .../templates/statefulset.yaml | 134 +++++++++--------- charts/execution-beacon/values.yaml | 41 ------ 5 files changed, 89 insertions(+), 193 deletions(-) delete mode 100644 charts/execution-beacon/templates/ingress.yaml diff --git a/charts/execution-beacon/templates/configmap.yaml b/charts/execution-beacon/templates/configmap.yaml index e0e501f58..a58bd0b81 100644 --- a/charts/execution-beacon/templates/configmap.yaml +++ b/charts/execution-beacon/templates/configmap.yaml @@ -15,32 +15,25 @@ data: echo "Namespace: ${POD_NAMESPACE} Pod: ${POD_NAME}"; {{- if .Values.global.p2pNodePort.enabled }} {{- if eq .Values.global.p2pNodePort.type "LoadBalancer" }} - export POD_NUMBER_SUFFIX=$(echo "${POD_NAME}" | grep -o -E '[0-9]+$') - export POD_PREFIX_NAME={{ include "common.names.fullname" . }} - for SERVICE_TYPE in beacon execution; do - SERVICE_NAME="${POD_PREFIX_NAME}-${SERVICE_TYPE}-${POD_NUMBER_SUFFIX}" - until [ -n "$(kubectl -n ${POD_NAMESPACE} get svc/${SERVICE_NAME} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do - echo "Waiting for load balancer to get an IP for ${SERVICE_NAME}" && sleep 10 - done - done - export EXTERNAL_EXECUTION_PORT=$(kubectl -n ${POD_NAMESPACE} get services -l "client=execution,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}') - export EXTERNAL_BEACON_PORT=$(kubectl -n ${POD_NAMESPACE} get services -l "client=beacon,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}') - export EXTERNAL_IP=$(kubectl -n ${POD_NAMESPACE} get svc/${POD_PREFIX_NAME}-beacon-${POD_NUMBER_SUFFIX} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + until [ -n "$(kubectl -n ${POD_NAMESPACE} get svc/${POD_NAME} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do echo "Waiting for load balancer to get an IP" && sleep 10; done; + export EXTERNAL_EXECUTION_PORT=$(kubectl -n ${POD_NAMESPACE} get services -l "client=execution,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}'); + export EXTERNAL_BEACON_PORT=$(kubectl -n ${POD_NAMESPACE} get services -l "client=beacon,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}'); + export EXTERNAL_IP=$(kubectl -n ${POD_NAMESPACE} get svc/${POD_NAME} -o jsonpath='{.status.loadBalancer.ingress[0].ip}'); {{- else }} - export EXTERNAL_EXECUTION_PORT=$(kubectl get services -l "client=execution,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}') - export EXTERNAL_BEACON_PORT=$(kubectl get services -l "client=beacon,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}') - export EXTERNAL_IP=$(kubectl get nodes "${NODE_NAME}" -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}') + export EXTERNAL_EXECUTION_PORT=$(kubectl get services -l "client=execution,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}'); + export EXTERNAL_BEACON_PORT=$(kubectl get services -l "client=beacon,pod in (${POD_NAME}), type in (p2p)" -o jsonpath='{.items[0].spec.ports[0].nodePort}'); + export EXTERNAL_IP=$(kubectl get nodes "${NODE_NAME}" -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}'); {{- end }} - echo "EXTERNAL_EXECUTION_PORT=$EXTERNAL_EXECUTION_PORT" > /env/init-nodeport - echo "EXTERNAL_BEACON_PORT=$EXTERNAL_BEACON_PORT" >> /env/init-nodeport - echo "EXTERNAL_IP=$EXTERNAL_IP" >> /env/init-nodeport - cat /env/init-nodeport + echo "EXTERNAL_EXECUTION_PORT=$EXTERNAL_EXECUTION_PORT" > /env/init-nodeport; + echo "EXTERNAL_BEACON_PORT=$EXTERNAL_BEACON_PORT" >> /env/init-nodeport; + echo "EXTERNAL_IP=$EXTERNAL_IP" >> /env/init-nodeport; + cat /env/init-nodeport; {{- if eq .Values.beacon.client "prysm" }} - touch /data/beacon/config.yaml - echo "p2p-host-ip: $EXTERNAL_IP" > /data/beacon/config.yaml - echo "p2p-tcp-port: $EXTERNAL_BEACON_PORT" >> /data/beacon/config.yaml - echo "p2p-udp-port: $EXTERNAL_BEACON_PORT" >> /data/beacon/config.yaml - echo "Updated Prysm config.yaml file at /data/beacon/config.yaml" + touch /data/beacon/config.yaml; + echo "p2p-host-ip: $EXTERNAL_IP" > /data/beacon/config.yaml; + echo "p2p-tcp-port: $EXTERNAL_BEACON_PORT" >> /data/beacon/config.yaml; + echo "p2p-udp-port: $EXTERNAL_BEACON_PORT" >> /data/beacon/config.yaml; + echo "Updated Prysm config.yaml file at /data/beacon/config.yaml"; {{- end }} - if [ -f "/data/beacon/config.yaml" ]; then echo "Beacon client config:" && cat "/data/beacon/config.yaml"; fi -{{- end }} + if [ -f "/data/beacon/config.yaml" ]; then echo "Beacon client config:" && cat "/data/beacon/config.yaml"; fi; +{{- end }} \ No newline at end of file diff --git a/charts/execution-beacon/templates/ingress.yaml b/charts/execution-beacon/templates/ingress.yaml deleted file mode 100644 index c545b1610..000000000 --- a/charts/execution-beacon/templates/ingress.yaml +++ /dev/null @@ -1,56 +0,0 @@ -{{- if .Values.global.ingress.enabled -}} - {{- $serviceName := include "common.names.fullname" . -}} - {{- $routePrefix := .Values.global.ingress.routePrefix | default "/" -}} - {{- $defaultPath := list (dict "path" $routePrefix "port" 8545 "pathType" "ImplementationSpecific") -}} - {{- $paths := .Values.global.ingress.paths | default $defaultPath -}} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: -{{- if .Values.global.ingress.annotations }} - annotations: -{{ toYaml .Values.global.ingress.annotations | indent 4 }} -{{- end }} - name: {{ include "common.names.fullname" . }} - labels: - {{- include "common.labels.standard" . | nindent 4 }} -{{- if .Values.global.ingress.labels }} -{{ toYaml .Values.global.ingress.labels | indent 4 }} -{{- end }} -spec: - {{- if .Values.global.ingress.ingressClassName }} - ingressClassName: {{ .Values.global.ingress.ingressClassName }} - {{- end }} - rules: - {{- if .Values.global.ingress.hosts }} - {{- range $host := .Values.global.ingress.hosts }} - - host: {{ tpl $host $ }} - http: - paths: - {{- range $p := $paths }} - - path: {{ $p.path }} - pathType: {{ $p.pathType }} - backend: - service: - name: {{ $serviceName }} - port: - number: {{ $p.port }} - {{- end }} - {{- end }} - {{- else }} - - http: - paths: - {{- range $p := $paths }} - - path: {{ $p.path }} - pathType: {{ $p.pathType }} - backend: - service: - name: {{ $serviceName }} - port: - number: {{ $p.port }} - {{- end }} - {{- end }} - {{- if .Values.global.ingress.tls }} - tls: -{{ tpl (toYaml .Values.global.ingress.tls | indent 4) . }} - {{- end }} -{{- end }} diff --git a/charts/execution-beacon/templates/service.yaml b/charts/execution-beacon/templates/service.yaml index ec7dceac7..9a06281be 100644 --- a/charts/execution-beacon/templates/service.yaml +++ b/charts/execution-beacon/templates/service.yaml @@ -23,11 +23,8 @@ spec: clientIP: timeoutSeconds: {{ .Values.global.sessionAffinity.timeoutSeconds }} {{- end }} - type: {{ .Values.global.service.type }} # ClusterIP, NodePort, LoadBalancer, or ExternalName -{{- if .Values.global.service.externalTrafficPolicy }} - externalTrafficPolicy: {{ .Values.global.service.externalTrafficPolicy }} -{{- end }} -{{- if .Values.global.service.svcHeadless}} + type: ClusterIP +{{- if .Values.global.service.svcHeadless }} clusterIP: None {{- end }} ports: @@ -74,3 +71,4 @@ spec: {{- end }} selector: {{- include "common.labels.matchLabels" . | nindent 4 }} + diff --git a/charts/execution-beacon/templates/statefulset.yaml b/charts/execution-beacon/templates/statefulset.yaml index 07a0b3776..0120ba882 100644 --- a/charts/execution-beacon/templates/statefulset.yaml +++ b/charts/execution-beacon/templates/statefulset.yaml @@ -114,55 +114,18 @@ spec: - name: execution image: "{{ (pluck .Values.execution.client .Values.global.image.execution | first ).repository }}:{{ (pluck .Values.execution.client .Values.global.image.execution | first ).tag }}" imagePullPolicy: {{ .Values.global.image.imagePullPolicy }} - {{/* Configuration for Nethermind execution client */}} - {{- if eq .Values.execution.client "nethermind" }} - args: - - --config={{ .Values.global.network }} - - --datadir=/data/execution - {{- if .Values.execution.jsonrpc.enabled }} - - --JsonRpc.Enabled={{ .Values.execution.jsonrpc.enabled }} - - --JsonRpc.EnabledModules={{ .Values.execution.jsonrpc.namespaces.nethermind | join "," }} - - --JsonRpc.Host={{ .Values.execution.jsonrpc.host }} - - --JsonRpc.Port={{ .Values.execution.jsonrpc.http.port }} - {{- if .Values.execution.healthchecks.enabled }} - - --HealthChecks.Enabled={{ .Values.execution.healthchecks.enabled }} - - --HealthChecks.Slug={{ .Values.execution.healthchecks.slug }} - - --HealthChecks.PollingInterval={{ .Values.execution.healthchecks.pollingInterval }} - - --HealthChecks.LowStorageSpaceShutdownThreshold={{ .Values.execution.healthchecks.lowStorageSpaceShutdownThreshold }} - - --HealthChecks.LowStorageSpaceWarningThreshold={{ .Values.execution.healthchecks.lowStorageSpaceWarningThreshold }} - {{- if .Values.execution.jsonrpc.websocket.enabled }} - - --JsonRpc.WebSocketsPort={{ .Values.execution.jsonrpc.websocket.port }} - {{- end }} - {{- end }} - {{- end }} - {{- if or .Values.global.JWTSecret .Values.global.externalSecrets.enabled }} + securityContext: + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL {{- if .Values.global.externalSecrets.enabled }} - - --JsonRpc.JwtSecretFile=/external-secrets/JWT_SECRET - {{- else }} - - --JsonRpc.JwtSecretFile=/secret/jwtsecret - {{- end }} - - --JsonRpc.EnginePort={{ .Values.execution.jsonrpc.engine.port }} - - --JsonRpc.EngineHost={{ .Values.execution.jsonrpc.host }} - - --JsonRpc.EngineEnabledModules={{ .Values.execution.jsonrpc.namespaces.nethermind | join "," }} - {{- end }} - {{- if and .Values.global.metrics.enabled .Values.execution.metrics.enabled }} - - --Metrics.Enabled={{ .Values.execution.metrics.enabled }} - - --Metrics.ExposePort={{ .Values.execution.metrics.port }} - - --Metrics.NodeName=$POD_NAME - {{- end }} - - --Network.MaxActivePeers={{ .Values.execution.targetPeers }} - {{- if not .Values.global.p2pNodePort.enabled }} - - --Network.ExternalIp=$POD_IP - - --Network.P2PPort={{ include "execution.p2pPort" . }} - - --Network.DiscoveryPort={{ include "execution.p2pPort" . }} - {{- end }} - {{- if .Values.execution.terminalTotalDifficulty }} - - --Merge.TerminalTotalDifficulty={{ .Values.execution.terminalTotalDifficulty }} - {{- end }} - {{- range .Values.execution.extraFlags }} - - {{ . }} + envFrom: + - secretRef: + name: eso-{{ include "common.names.fullname" . }} {{- end }} - {{- else }} command: - sh - -ac @@ -170,8 +133,57 @@ spec: {{- if .Values.global.p2pNodePort.enabled }} . /env/init-nodeport; {{- end }} - {{/* Configuration for Geth execution client */}} - {{- if eq .Values.execution.client "geth" }} + {{- /* Configuration for Nethermind execution client */}} + {{- if eq .Values.execution.client "nethermind" }} + exec /nethermind/Nethermind.Runner + --config={{ .Values.global.network }} + --datadir=/data/execution + {{- if .Values.execution.jsonrpc.enabled }} + --JsonRpc.Enabled={{ .Values.execution.jsonrpc.enabled }} + --JsonRpc.EnabledModules={{ .Values.execution.jsonrpc.namespaces.nethermind | join "," }} + --JsonRpc.Host={{ .Values.execution.jsonrpc.host }} + --JsonRpc.Port={{ .Values.execution.jsonrpc.http.port }} + {{- if .Values.execution.healthchecks.enabled }} + --HealthChecks.Enabled={{ .Values.execution.healthchecks.enabled }} + --HealthChecks.Slug={{ .Values.execution.healthchecks.slug }} + --HealthChecks.PollingInterval={{ .Values.execution.healthchecks.pollingInterval }} + --HealthChecks.LowStorageSpaceShutdownThreshold={{ .Values.execution.healthchecks.lowStorageSpaceShutdownThreshold }} + --HealthChecks.LowStorageSpaceWarningThreshold={{ .Values.execution.healthchecks.lowStorageSpaceWarningThreshold }} + {{- if .Values.execution.jsonrpc.websocket.enabled }} + --JsonRpc.WebSocketsPort={{ .Values.execution.jsonrpc.websocket.port }} + {{- end }} + {{- end }} + {{- end }} + {{- if or .Values.global.JWTSecret .Values.global.externalSecrets.enabled }} + {{- if .Values.global.externalSecrets.enabled }} + --JsonRpc.JwtSecretFile=/external-secrets/JWT_SECRET + {{- else }} + --JsonRpc.JwtSecretFile=/secret/jwtsecret + {{- end }} + --JsonRpc.EnginePort={{ .Values.execution.jsonrpc.engine.port }} + --JsonRpc.EngineHost={{ .Values.execution.jsonrpc.host }} + --JsonRpc.EngineEnabledModules={{ .Values.execution.jsonrpc.namespaces.nethermind | join "," }} + {{- end }} + {{- if and .Values.global.metrics.enabled .Values.execution.metrics.enabled }} + --Metrics.Enabled={{ .Values.execution.metrics.enabled }} + --Metrics.ExposePort={{ .Values.execution.metrics.port }} + --Metrics.NodeName=$POD_NAME + {{- end }} + --Network.MaxActivePeers={{ .Values.execution.targetPeers }} + {{- if .Values.global.p2pNodePort.enabled }} + --Network.ExternalIp=$EXTERNAL_IP + --Network.P2PPort=$EXTERNAL_EXECUTION_PORT + --Network.DiscoveryPort=$EXTERNAL_EXECUTION_PORT + {{- else }} + --Network.ExternalIp=$POD_IP + --Network.P2PPort={{ include "execution.p2pPort" . }} + --Network.DiscoveryPort={{ include "execution.p2pPort" . }} + {{- end }} + {{- if .Values.execution.terminalTotalDifficulty }} + --Merge.TerminalTotalDifficulty={{ .Values.execution.terminalTotalDifficulty }} + {{- end }} + {{- /* Configuration for Geth execution client */}} + {{- else if eq .Values.execution.client "geth" }} exec geth --syncmode=snap --datadir=/data/execution @@ -278,16 +290,18 @@ spec: --ws {{- end }} {{- end }} + {{- range .Values.extraFlags }} + {{ . }} + {{- end }} {{- if .Values.global.metrics.enabled }} - --metrics - --metrics.addr={{ .Values.execution.metrics.host }} - --metrics.port={{ .Values.execution.metrics.port }} + --metrics + --metrics.addr={{ .Values.execution.metrics.host }} + --metrics.port={{ .Values.execution.metrics.port }} {{- end }} {{- end }} {{- range .Values.execution.extraFlags }} - {{ . }} + {{ . }} {{- end }} - {{- end}} env: - name: POD_IP valueFrom: @@ -297,18 +311,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - {{- if and (eq .Values.execution.client "nethermind") (.Values.global.p2pNodePort.enabled) }} - {{- range $i, $e := until (int .Values.global.replicaCount) }} - {{- $portExecution := add $.Values.global.p2pNodePort.startAtExecution $i }} - {{- if hasKey $.Values.global.p2pNodePort.replicaToNodePort ($i | toString) }} - {{- $portExecution = index $.Values.global.p2pNodePort.replicaToNodePort ($i | toString) }} - {{- end }} - - name: NETHERMIND_NETWORKCONFIG_DISCOVERYPORT - value: "{{ $portExecution }}" - - name: NETHERMIND_NETWORKCONFIG_P2PPORT - value: "{{ $portExecution }}" - {{- end }} - {{- end }} {{- if .Values.execution.javaOpts.enabled }} - name: BESU_OPTS value: {{ .Values.execution.javaOpts.maxHeapSize }} diff --git a/charts/execution-beacon/values.yaml b/charts/execution-beacon/values.yaml index adf39ef1e..ea05ae7c2 100644 --- a/charts/execution-beacon/values.yaml +++ b/charts/execution-beacon/values.yaml @@ -136,7 +136,6 @@ global: service: svcHeadless: true - type: ClusterIP sessionAffinity: # Whether to enable session affinity or not @@ -256,46 +255,6 @@ global: drop: - ALL - ingress: - enabled: false - ingressClassName: nginx - - ## Route Prefix. Can skip it if any item of path has the path defined. - ## - routePrefix: / - - annotations: {} - labels: {} - - ## Hostnames. - ## Must be provided if Ingress is enabled. - ## - # hosts: - # - prometheus.domain.com - hosts: [] - - ## Paths to use for ingress rules - ## If not defined the following default object will be used: - ## - path: "/" - ## port: 8545 - ## pathType: "ImplementationSpecific" - paths: [] - ## paths: - ## - path: "/execution" - ## port: 8545 - ## pathType: "Prefix" - ## - path: "/beacon" - ## port: 8080 - ## pathType: "Exact" - - ## TLS configuration for Ingress - ## Secret must be manually created in the namespace - ## - tls: [] - # - secretName: execution-beacon-general-tls - # hosts: - # - nethermind.example.com - execution: client: nethermind From b5f8e104e368d1aed0032369c7e52fd87f781c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:21:50 +0100 Subject: [PATCH 14/34] Update statefulset.yaml (#101) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update statefulset.yaml Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> * Update values.yaml Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --------- Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/dirk/templates/statefulset.yaml | 30 +++++------------------- charts/dirk/values.yaml | 32 +++++++++++++------------- 2 files changed, 22 insertions(+), 40 deletions(-) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index 823485bdf..bdbaa84ee 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -45,7 +45,7 @@ spec: {{- end }} serviceAccountName: {{ include "dirk.serviceAccountName" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml .Values.global.podSecurityContext | nindent 8 }} initContainers: - name: init image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" @@ -65,7 +65,7 @@ spec: printenv dirk-ca.crt | base64 -d > /data/dirk/certs/ca.crt; printenv dirk-ca.key | base64 -d > /data/dirk/certs/ca.key; bash /scripts/generate_cert.sh /data/dirk/certs/ ca ${HOSTNAME}.{{ include "dirk.fullname" . }}; - chown -R {{ .Values.podSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} /data/dirk; + chown -R {{ .Values.global.podSecurityContext.runAsUser }}:{{ .Values.global.podSecurityContext.fsGroup }} /data/dirk; envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -80,13 +80,7 @@ spec: image: "{{ .Values.cliImage.repository }}:{{ .Values.cliImage.tag }}" imagePullPolicy: {{ .Values.cliImage.pullPolicy }} securityContext: - runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} - runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} - fsGroup: {{ .Values.global.securityContext.fsGroup | default 10000 }} - readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} - capabilities: - drop: - - ALL + {{- toYaml .Values.global.securityContext | nindent 12 }} envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -111,13 +105,7 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: - runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} - runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} - fsGroup: {{ .Values.global.securityContext.fsGroup | default 10000 }} - readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} - capabilities: - drop: - - ALL + {{- toYaml .Values.global.securityContext | nindent 12 }} envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -162,13 +150,7 @@ spec: image: "{{ .Values.cliImage.repository }}:{{ .Values.cliImage.tag }}" imagePullPolicy: {{ .Values.cliImage.pullPolicy }} securityContext: - runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} - runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} - fsGroup: {{ .Values.global.securityContext.fsGroup | default 10000 }} - readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} - capabilities: - drop: - - ALL + {{- toYaml .Values.global.securityContext | nindent 12 }} envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -218,4 +200,4 @@ spec: storageClassName: {{ .Values.persistence.storageClassName }} resources: requests: - storage: {{ .Values.persistence.size | quote }} \ No newline at end of file + storage: {{ .Values.persistence.size | quote }} diff --git a/charts/dirk/values.yaml b/charts/dirk/values.yaml index 534429519..d6479f518 100644 --- a/charts/dirk/values.yaml +++ b/charts/dirk/values.yaml @@ -5,7 +5,22 @@ global: serviceAccount: create: true - + ## Pod Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + podSecurityContext: + runAsNonRoot: true + runAsUser: 10000 + fsGroup: 10000 + + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL + replicaCount: 3 image: @@ -106,21 +121,6 @@ serviceAccount: podAnnotations: {} -## Pod Security Context -## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ -## -podSecurityContext: - fsGroup: 1000 - runAsUser: 1000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - service: type: ClusterIP httpPort: 8881 From 55f99fd09157ab857cc11c80f39906ffd4e1fa46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:24:33 +0100 Subject: [PATCH 15/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/dirk/templates/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index bdbaa84ee..640ea904c 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -51,7 +51,7 @@ spec: image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: - runAsUser: 0 + {{- toYaml .Values.global.securityContext | nindent 12 }} command: - bash - -xc From 9ed6c0dee42f696ba2c60db3ee839fe36b4ee912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:27:59 +0100 Subject: [PATCH 16/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/dirk/templates/statefulset.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index 640ea904c..318bd81af 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -51,7 +51,12 @@ spec: image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: - {{- toYaml .Values.global.securityContext | nindent 12 }} + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL command: - bash - -xc From 8d0458eab283bcf5ea87d45d149de73a50095a97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:30:33 +0100 Subject: [PATCH 17/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/dirk/templates/statefulset.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index 318bd81af..836d0685f 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -52,11 +52,7 @@ spec: imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: readOnlyRootFilesystem: false - runAsNonRoot: true - runAsUser: 10000 - capabilities: - drop: - - ALL + runAsNonRoot: false command: - bash - -xc From 812717b241347dcc269779b19d09bba77baa4238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:36:24 +0100 Subject: [PATCH 18/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/dirk/templates/statefulset.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index 836d0685f..850b5e0ca 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -51,8 +51,8 @@ spec: image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: - readOnlyRootFilesystem: false - runAsNonRoot: false + runAsUser: 0 + allowPrivilegeEscalation: true command: - bash - -xc From 19549011a79522e05f3693abf797078125640441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 29 Feb 2024 18:34:58 +0100 Subject: [PATCH 19/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/dirk/templates/statefulset.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index 850b5e0ca..851ce1c54 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -52,12 +52,10 @@ spec: imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: runAsUser: 0 - allowPrivilegeEscalation: true command: - bash - -xc - > - apk add openssl envsubst; rm -f /data/dirk/dirty; mkdir -p /data/dirk/certs; export INDEX=$((${HOSTNAME##*-}+1)); From 45740d84440a5345e6ec31378e352b751f612981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 29 Feb 2024 18:36:22 +0100 Subject: [PATCH 20/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/dirk/templates/statefulset.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index 851ce1c54..50a0b735c 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -51,7 +51,12 @@ spec: image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: - runAsUser: 0 + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL command: - bash - -xc From 152b819a7ba8171453353b8342e10fb5c5f4c660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 29 Feb 2024 18:45:21 +0100 Subject: [PATCH 21/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/dirk/templates/statefulset.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index 50a0b735c..318bd81af 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -61,6 +61,7 @@ spec: - bash - -xc - > + apk add openssl envsubst; rm -f /data/dirk/dirty; mkdir -p /data/dirk/certs; export INDEX=$((${HOSTNAME##*-}+1)); From 1518f8b769ffacc6fd840290feb73f028b850c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 29 Feb 2024 18:55:26 +0100 Subject: [PATCH 22/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/dirk/templates/statefulset.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index 318bd81af..eae6363cb 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -51,12 +51,8 @@ spec: image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: - readOnlyRootFilesystem: false - runAsNonRoot: true - runAsUser: 10000 - capabilities: - drop: - - ALL + allowPrivilegeEscalation: false + runAsUser: 0 command: - bash - -xc From 11a93ec1246234859322f468d31acd0ba547849e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:06:15 +0100 Subject: [PATCH 23/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/dirk/templates/statefulset.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index eae6363cb..640ea904c 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -51,8 +51,7 @@ spec: image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: - allowPrivilegeEscalation: false - runAsUser: 0 + {{- toYaml .Values.global.securityContext | nindent 12 }} command: - bash - -xc From 1c4beb922fd24654506e1a80dfeea61ac6bd7963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Thu, 29 Feb 2024 19:07:40 +0100 Subject: [PATCH 24/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/dirk/templates/statefulset.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index 640ea904c..64474ce0b 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -56,7 +56,6 @@ spec: - bash - -xc - > - apk add openssl envsubst; rm -f /data/dirk/dirty; mkdir -p /data/dirk/certs; export INDEX=$((${HOSTNAME##*-}+1)); From f5058da118d6e1b1b238b9fa1d300dda91c66c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:05:43 +0100 Subject: [PATCH 25/34] Ethereum/dirk vouch unify (#102) * Rename env vars * Add admin-ips * Test new format * Update admin IPs --------- Co-authored-by: Aivaras Ko --- charts/dirk/templates/configmap.yaml | 3 +++ charts/dirk/templates/statefulset.yaml | 4 ++-- charts/dirk/values.yaml | 3 +++ charts/vouch/templates/statefulset.yaml | 4 ++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/charts/dirk/templates/configmap.yaml b/charts/dirk/templates/configmap.yaml index 03fc561fe..32463a033 100644 --- a/charts/dirk/templates/configmap.yaml +++ b/charts/dirk/templates/configmap.yaml @@ -21,6 +21,9 @@ data: id: $INDEX name: $HOSTNAME.{{ include "dirk.fullname" . }} listen-address: 0.0.0.0:{{ .Values.service.httpPort }} + rules: + # admin-ips is a list of IP addresses from which requests for voluntary exists will be accepted. + admin-ips: {{ .Values.dirk.server.rules.adminIPs }} certificates: ca-cert: file:///data/dirk/certs/ca.crt server-cert: file:///data/dirk/certs/$HOSTNAME.{{ include "dirk.fullname" . }}.crt diff --git a/charts/dirk/templates/statefulset.yaml b/charts/dirk/templates/statefulset.yaml index 64474ce0b..d9e8f12a7 100644 --- a/charts/dirk/templates/statefulset.yaml +++ b/charts/dirk/templates/statefulset.yaml @@ -61,8 +61,8 @@ spec: export INDEX=$((${HOSTNAME##*-}+1)); echo "$INDEX" > /data/dirk/index; envsubst < /config/dirk.yaml.tml > /data/dirk/dirk.yaml; - printenv dirk-ca.crt | base64 -d > /data/dirk/certs/ca.crt; - printenv dirk-ca.key | base64 -d > /data/dirk/certs/ca.key; + printenv CA_CRT | base64 -d > /data/dirk/certs/ca.crt; + printenv CA_KEY | base64 -d > /data/dirk/certs/ca.key; bash /scripts/generate_cert.sh /data/dirk/certs/ ca ${HOSTNAME}.{{ include "dirk.fullname" . }}; chown -R {{ .Values.global.podSecurityContext.runAsUser }}:{{ .Values.global.podSecurityContext.fsGroup }} /data/dirk; envFrom: diff --git a/charts/dirk/values.yaml b/charts/dirk/values.yaml index d6479f518..35a781801 100644 --- a/charts/dirk/values.yaml +++ b/charts/dirk/values.yaml @@ -52,6 +52,9 @@ externalSecrets: key: dirk dirk: + server: + rules: + adminIPs: [127.0.0.1] loglevel: Debug clientName: client1 tracing: diff --git a/charts/vouch/templates/statefulset.yaml b/charts/vouch/templates/statefulset.yaml index a0d6586c8..522ec85f0 100644 --- a/charts/vouch/templates/statefulset.yaml +++ b/charts/vouch/templates/statefulset.yaml @@ -52,8 +52,8 @@ spec: apk add openssl; mkdir -p {{ .Values.vouchDataDir }}/{{ include "common.names.fullname" . }}/certs/; cp /config/vouch.yaml {{ .Values.vouchDataDir }}/; - printenv vouch-ca.crt | base64 -d > {{ .Values.vouchDataDir }}/{{ include "common.names.fullname" . }}/certs/ca.crt; - printenv vouch-ca.key | base64 -d > {{ .Values.vouchDataDir }}/{{ include "common.names.fullname" . }}/certs/ca.key; + printenv CA_CRT | base64 -d > {{ .Values.vouchDataDir }}/{{ include "common.names.fullname" . }}/certs/ca.crt; + printenv CA_KEY | base64 -d > {{ .Values.vouchDataDir }}/{{ include "common.names.fullname" . }}/certs/ca.key; bash /scripts/generate_cert.sh {{ .Values.vouchDataDir }}/{{ include "common.names.fullname" . }}/certs/ ca {{ include "common.names.fullname" . }}; chown -R {{ .Values.podSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} {{ .Values.vouchDataDir }}; envFrom: From 08c9ae3a97c5768aec2fe862069f1c19ecf3d120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:23:54 +0100 Subject: [PATCH 26/34] Vouch patch (#103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update statefulset.yaml Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> * Update values.yaml Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --------- Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/vouch/templates/statefulset.yaml | 25 ++++------------ charts/vouch/values.yaml | 38 ++++++++++++------------- 2 files changed, 25 insertions(+), 38 deletions(-) diff --git a/charts/vouch/templates/statefulset.yaml b/charts/vouch/templates/statefulset.yaml index 522ec85f0..15cc7d437 100644 --- a/charts/vouch/templates/statefulset.yaml +++ b/charts/vouch/templates/statefulset.yaml @@ -38,24 +38,23 @@ spec: {{- end }} serviceAccountName: {{ include "vouch.serviceAccountName" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml .Values.global.securityContext | nindent 12 }} initContainers: - name: init image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: - runAsUser: 0 + {{- toYaml .Values.global.securityContext | nindent 12 }} command: - sh - -ac - > - apk add openssl; mkdir -p {{ .Values.vouchDataDir }}/{{ include "common.names.fullname" . }}/certs/; cp /config/vouch.yaml {{ .Values.vouchDataDir }}/; printenv CA_CRT | base64 -d > {{ .Values.vouchDataDir }}/{{ include "common.names.fullname" . }}/certs/ca.crt; printenv CA_KEY | base64 -d > {{ .Values.vouchDataDir }}/{{ include "common.names.fullname" . }}/certs/ca.key; bash /scripts/generate_cert.sh {{ .Values.vouchDataDir }}/{{ include "common.names.fullname" . }}/certs/ ca {{ include "common.names.fullname" . }}; - chown -R {{ .Values.podSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} {{ .Values.vouchDataDir }}; + chown -R {{ .Values.global.podSecurityContext.runAsUser }}:{{ .Values.global.podSecurityContext.fsGroup }} {{ .Values.vouchDataDir }}; envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -70,13 +69,7 @@ spec: image: "{{ .Values.cliImage.repository }}:{{ .Values.cliImage.tag }}" imagePullPolicy: {{ .Values.cliImage.pullPolicy }} securityContext: - runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} - runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} - fsGroup: {{ .Values.global.securityContext.fsGroup | default 10000 }} - readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} - capabilities: - drop: - - ALL + {{- toYaml .Values.global.securityContext | nindent 12 }} envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -100,13 +93,7 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: - runAsNonRoot: {{ .Values.global.securityContext.runAsNonRoot | default true }} - runAsUser: {{ .Values.global.securityContext.runAsUser | default 10000 }} - fsGroup: {{ .Values.global.securityContext.fsGroup | default 10000 }} - readOnlyRootFilesystem: {{ .Values.global.securityContext.readOnlyRootFilesystem | default true }} - capabilities: - drop: - - ALL + {{- toYaml .Values.global.securityContext | nindent 12 }} envFrom: - secretRef: name: eso-{{ include "common.names.fullname" . }} @@ -157,4 +144,4 @@ spec: - name: data emptyDir: medium: Memory - sizeLimit: 128Mi \ No newline at end of file + sizeLimit: 128Mi diff --git a/charts/vouch/values.yaml b/charts/vouch/values.yaml index 95caa339e..3c15da94a 100644 --- a/charts/vouch/values.yaml +++ b/charts/vouch/values.yaml @@ -5,18 +5,33 @@ global: serviceAccount: create: true - + ## Pod Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + podSecurityContext: + runAsNonRoot: true + runAsUser: 10000 + fsGroup: 10000 + + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL + image: repository: attestant/vouch pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "1.7.6" + tag: "1.8.0" ## Init image is used to chown data volume, etc. ## initImage: - repository: bash - tag: "5.2" + repository: nethermindeth/bash + tag: "5.2-alpine3.19" pullPolicy: IfNotPresent cliImage: @@ -126,21 +141,6 @@ serviceAccount: podAnnotations: {} -## Pod Security Context -## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ -## -podSecurityContext: - fsGroup: 1000 - runAsUser: 1000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - service: type: ClusterIP httpPort: 8881 From 2b8540b3d4cc8cd8ae97bd185bac401cee5a0e4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:26:52 +0100 Subject: [PATCH 27/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/vouch/templates/statefulset.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/charts/vouch/templates/statefulset.yaml b/charts/vouch/templates/statefulset.yaml index 15cc7d437..2586fc5ec 100644 --- a/charts/vouch/templates/statefulset.yaml +++ b/charts/vouch/templates/statefulset.yaml @@ -44,7 +44,12 @@ spec: image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: - {{- toYaml .Values.global.securityContext | nindent 12 }} + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL command: - sh - -ac From 272805b9dd5db1967b7c5aff6dcd45f64e7b77c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:32:40 +0100 Subject: [PATCH 28/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/vouch/templates/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/vouch/templates/statefulset.yaml b/charts/vouch/templates/statefulset.yaml index 2586fc5ec..f4c26dd5e 100644 --- a/charts/vouch/templates/statefulset.yaml +++ b/charts/vouch/templates/statefulset.yaml @@ -45,7 +45,7 @@ spec: imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: readOnlyRootFilesystem: false - runAsNonRoot: true + runAsNonRoot: false runAsUser: 10000 capabilities: drop: From 0f7810b2de2509b9e15dd22703eaa4f60041cef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:34:12 +0100 Subject: [PATCH 29/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/vouch/templates/statefulset.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/charts/vouch/templates/statefulset.yaml b/charts/vouch/templates/statefulset.yaml index f4c26dd5e..636235dab 100644 --- a/charts/vouch/templates/statefulset.yaml +++ b/charts/vouch/templates/statefulset.yaml @@ -46,10 +46,6 @@ spec: securityContext: readOnlyRootFilesystem: false runAsNonRoot: false - runAsUser: 10000 - capabilities: - drop: - - ALL command: - sh - -ac From fe06304c166a14c64e67f5472d97dbbef25027c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Fri, 1 Mar 2024 14:37:10 +0100 Subject: [PATCH 30/34] Update statefulset.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mateusz Jędrzejewski <33068017+matilote@users.noreply.github.com> --- charts/vouch/templates/statefulset.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/charts/vouch/templates/statefulset.yaml b/charts/vouch/templates/statefulset.yaml index 636235dab..6a63df0db 100644 --- a/charts/vouch/templates/statefulset.yaml +++ b/charts/vouch/templates/statefulset.yaml @@ -38,14 +38,13 @@ spec: {{- end }} serviceAccountName: {{ include "vouch.serviceAccountName" . }} securityContext: - {{- toYaml .Values.global.securityContext | nindent 12 }} + {{- toYaml .Values.global.podSecurityContext | nindent 8 }} initContainers: - name: init image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} securityContext: - readOnlyRootFilesystem: false - runAsNonRoot: false + {{- toYaml .Values.global.securityContext | nindent 12 }} command: - sh - -ac From 5fb2b2613df3204a7c1a5a7e3dfec3927c7596a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Sat, 16 Mar 2024 17:37:54 +0000 Subject: [PATCH 31/34] add blockrelay http service (#106) --- charts/vouch/templates/service.yaml | 4 ++++ charts/vouch/templates/statefulset.yaml | 3 +++ charts/vouch/values.yaml | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/charts/vouch/templates/service.yaml b/charts/vouch/templates/service.yaml index 1631d174e..4ae945194 100644 --- a/charts/vouch/templates/service.yaml +++ b/charts/vouch/templates/service.yaml @@ -15,6 +15,10 @@ spec: targetPort: http protocol: TCP name: http + - port: {{ .Values.blockRelayPort }} + targetPort: blockrelay + protocol: TCP + name: blockrelay - port: {{ .Values.metricsPort }} targetPort: metrics protocol: TCP diff --git a/charts/vouch/templates/statefulset.yaml b/charts/vouch/templates/statefulset.yaml index 6a63df0db..1bcc4f518 100644 --- a/charts/vouch/templates/statefulset.yaml +++ b/charts/vouch/templates/statefulset.yaml @@ -102,6 +102,9 @@ spec: - name: http containerPort: {{ .Values.service.httpPort }} protocol: TCP + - name: blockrelay + containerPort: {{ .Values.blockRelayPort }} + protocol: TCP - name: metrics containerPort: {{ .Values.metricsPort }} protocol: TCP diff --git a/charts/vouch/values.yaml b/charts/vouch/values.yaml index 3c15da94a..0be86e03a 100644 --- a/charts/vouch/values.yaml +++ b/charts/vouch/values.yaml @@ -77,6 +77,7 @@ vouch: loglevel: 'trace' listenaddress: '0.0.0.0:8081' blockrelay: + listen-address: '0.0.0.0:18550' fallbackfeerecipient: '0x0000000000000000000000000000000000000001' tracing: strategies: @@ -98,6 +99,9 @@ fullnameOverride: "" ## Log levels are OFF, FATAL, WARN, INFO, DEBUG, TRACE, ALL. loggingLevel: "INFO" +## By default, Vouch's MEV-boost service listens on port 18550. Beacon nodes used by Vouch should be configured to talk directly to this, rather than other MEV-boost services or directly to relays. +blockRelayPort: 18550 + ## Port on which vouch HTTP listens. ## httpPort: 8881 From 4a07ea2fe4f9edb11a0cbdfe459647e820539b28 Mon Sep 17 00:00:00 2001 From: matilote Date: Mon, 18 Mar 2024 13:25:15 +0100 Subject: [PATCH 32/34] add clickhouse from bitnami to our charts repo --- charts/clickhouse/.helmignore | 23 + charts/clickhouse/Chart.lock | 9 + charts/clickhouse/Chart.yaml | 38 + charts/clickhouse/README.md | 614 +++++++++ charts/clickhouse/templates/NOTES.txt | 59 + charts/clickhouse/templates/_helpers.tpl | 219 +++ .../clickhouse/templates/configmap-extra.yaml | 20 + .../templates/configmap-users-extra.yaml | 20 + charts/clickhouse/templates/configmap.yaml | 20 + charts/clickhouse/templates/extra-list.yaml | 9 + .../templates/ingress-tls-secrets.yaml | 44 + charts/clickhouse/templates/ingress.yaml | 59 + .../templates/init-scripts-secret.yaml | 19 + .../clickhouse/templates/networkpolicy.yaml | 136 ++ .../clickhouse/templates/prometheusrule.yaml | 24 + .../templates/scripts-configmap.yaml | 34 + charts/clickhouse/templates/secret.yaml | 20 + .../clickhouse/templates/service-account.yaml | 19 + .../templates/service-external-access.yaml | 155 +++ .../templates/service-headless.yaml | 69 + charts/clickhouse/templates/service.yaml | 152 +++ .../clickhouse/templates/servicemonitor.yaml | 47 + .../templates/start-scripts-secret.yaml | 19 + charts/clickhouse/templates/statefulset.yaml | 462 +++++++ charts/clickhouse/templates/tls-secret.yaml | 29 + charts/clickhouse/values.yaml | 1200 +++++++++++++++++ 26 files changed, 3519 insertions(+) create mode 100644 charts/clickhouse/.helmignore create mode 100644 charts/clickhouse/Chart.lock create mode 100644 charts/clickhouse/Chart.yaml create mode 100644 charts/clickhouse/README.md create mode 100644 charts/clickhouse/templates/NOTES.txt create mode 100644 charts/clickhouse/templates/_helpers.tpl create mode 100644 charts/clickhouse/templates/configmap-extra.yaml create mode 100644 charts/clickhouse/templates/configmap-users-extra.yaml create mode 100644 charts/clickhouse/templates/configmap.yaml create mode 100644 charts/clickhouse/templates/extra-list.yaml create mode 100644 charts/clickhouse/templates/ingress-tls-secrets.yaml create mode 100644 charts/clickhouse/templates/ingress.yaml create mode 100644 charts/clickhouse/templates/init-scripts-secret.yaml create mode 100644 charts/clickhouse/templates/networkpolicy.yaml create mode 100644 charts/clickhouse/templates/prometheusrule.yaml create mode 100644 charts/clickhouse/templates/scripts-configmap.yaml create mode 100644 charts/clickhouse/templates/secret.yaml create mode 100644 charts/clickhouse/templates/service-account.yaml create mode 100644 charts/clickhouse/templates/service-external-access.yaml create mode 100644 charts/clickhouse/templates/service-headless.yaml create mode 100644 charts/clickhouse/templates/service.yaml create mode 100644 charts/clickhouse/templates/servicemonitor.yaml create mode 100644 charts/clickhouse/templates/start-scripts-secret.yaml create mode 100644 charts/clickhouse/templates/statefulset.yaml create mode 100644 charts/clickhouse/templates/tls-secret.yaml create mode 100644 charts/clickhouse/values.yaml diff --git a/charts/clickhouse/.helmignore b/charts/clickhouse/.helmignore new file mode 100644 index 000000000..fb56657ab --- /dev/null +++ b/charts/clickhouse/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# img folder +img/ diff --git a/charts/clickhouse/Chart.lock b/charts/clickhouse/Chart.lock new file mode 100644 index 000000000..617994105 --- /dev/null +++ b/charts/clickhouse/Chart.lock @@ -0,0 +1,9 @@ +dependencies: +- name: zookeeper + repository: oci://registry-1.docker.io/bitnamicharts + version: 12.12.1 +- name: common + repository: oci://registry-1.docker.io/bitnamicharts + version: 2.19.0 +digest: sha256:92742d7cab086ee69ec5087e9ed723cc5387b8f434b6e503f04e57bd1c3e8040 +generated: "2024-03-15T10:52:50.842593241Z" diff --git a/charts/clickhouse/Chart.yaml b/charts/clickhouse/Chart.yaml new file mode 100644 index 000000000..28443608d --- /dev/null +++ b/charts/clickhouse/Chart.yaml @@ -0,0 +1,38 @@ +# Copyright VMware, Inc. +# SPDX-License-Identifier: APACHE-2.0 + +annotations: + category: Database + licenses: Apache-2.0 + images: | + - name: clickhouse + image: docker.io/bitnami/clickhouse:24.2.2-debian-12-r0 + - name: os-shell + image: docker.io/bitnami/os-shell:12-debian-12-r16 + - name: zookeeper + image: docker.io/bitnami/zookeeper:3.8.4-debian-12-r0 +apiVersion: v2 +appVersion: 24.2.2 +dependencies: +- condition: zookeeper.enabled + name: zookeeper + repository: oci://registry-1.docker.io/bitnamicharts + version: 12.x.x +- name: common + repository: oci://registry-1.docker.io/bitnamicharts + tags: + - bitnami-common + version: 2.x.x +description: ClickHouse is an open-source column-oriented OLAP database management system. Use it to boost your database performance while providing linear scalability and hardware efficiency. +home: https://bitnami.com +icon: https://bitnami.com/assets/stacks/clickhouse/img/clickhouse-stack-220x234.png +keywords: +- database +- sharding +maintainers: +- name: VMware, Inc. + url: https://github.com/bitnami/charts +name: clickhouse +sources: +- https://github.com/bitnami/charts/tree/main/bitnami/clickhouse +version: 5.3.1 diff --git a/charts/clickhouse/README.md b/charts/clickhouse/README.md new file mode 100644 index 000000000..caecae708 --- /dev/null +++ b/charts/clickhouse/README.md @@ -0,0 +1,614 @@ + + +# Bitnami package for ClickHouse + +ClickHouse is an open-source column-oriented OLAP database management system. Use it to boost your database performance while providing linear scalability and hardware efficiency. + +[Overview of ClickHouse](https://clickhouse.com/) + +Trademarks: This software listing is packaged by Bitnami. The respective trademarks mentioned in the offering are owned by the respective companies, and use of them does not imply any affiliation or endorsement. + +## TL;DR + +```console +helm install my-release oci://registry-1.docker.io/bitnamicharts/clickhouse +``` + +Looking to use ClickHouse in production? Try [VMware Tanzu Application Catalog](https://bitnami.com/enterprise), the enterprise edition of Bitnami Application Catalog. + +## Introduction + +Bitnami charts for Helm are carefully engineered, actively maintained and are the quickest and easiest way to deploy containers on a Kubernetes cluster that are ready to handle production workloads. + +This chart bootstraps a [ClickHouse](https://github.com/clickhouse/clickhouse) Deployment in a [Kubernetes](https://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.dev/) for deployment and management of Helm Charts in clusters. + +## Prerequisites + +- Kubernetes 1.23+ +- Helm 3.8.0+ +- PV provisioner support in the underlying infrastructure +- ReadWriteMany volumes for deployment scaling + +> If you are using Kubernetes 1.18, the following code needs to be commented out. +> seccompProfile: +> type: "RuntimeDefault" + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install my-release oci://REGISTRY_NAME/REPOSITORY_NAME/clickhouse +``` + +> Note: You need to substitute the placeholders `REGISTRY_NAME` and `REPOSITORY_NAME` with a reference to your Helm chart registry and repository. For example, in the case of Bitnami, you need to use `REGISTRY_NAME=registry-1.docker.io` and `REPOSITORY_NAME=bitnamicharts`. + +The command deploys ClickHouse on the Kubernetes cluster in the default configuration. The [Parameters](#parameters) section lists the parameters that can be configured during installation. + +> **Tip**: List all releases using `helm list` + +## Configuration and installation details + +### Resource requests and limits + +Bitnami charts allow setting resource requests and limits for all containers inside the chart deployment. These are inside the `resources` value (check parameter table). Setting requests is essential for production workloads and these should be adapted to your specific use case. + +To make this process easier, the chart contains the `resourcesPreset` values, which automatically sets the `resources` section according to different presets. Check these presets in [the bitnami/common chart](https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15). However, in production workloads using `resourcePreset` is discouraged as it may not fully adapt to your specific needs. Find more information on container resource management in the [official Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/). + +### [Rolling VS Immutable tags](https://docs.bitnami.com/tutorials/understand-rolling-tags-containers) + +It is strongly recommended to use immutable tags in a production environment. This ensures your deployment does not change automatically if the same tag is updated with a different image. + +Bitnami will release a new chart updating its containers if a new version of the main container, significant changes, or critical vulnerabilities exist. + +### ClickHouse keeper support + +You can set `keeper.enabled` to use ClickHouse keeper. If `keeper.enabled=true`, Zookeeper settings will not be ignore. + +### External Zookeeper support + +You may want to have ClickHouse connect to an external zookeeper rather than installing one inside your cluster. Typical reasons for this are to use a managed database service, or to share a common database server for all your applications. To achieve this, the chart allows you to specify credentials for an external database with the [`externalZookeeper` parameter](#parameters). You should also disable the Zookeeper installation with the `zookeeper.enabled` option. Here is an example: + +```console +zookeper.enabled=false +externalZookeeper.host=myexternalhost +externalZookeeper.user=myuser +externalZookeeper.password=mypassword +externalZookeeper.database=mydatabase +externalZookeeper.port=3306 +``` + +### Ingress without TLS + +For using ingress (example without TLS): + +```yaml +ingress: + ## If true, ClickHouse server Ingress will be created + ## + enabled: true + + ## ClickHouse server Ingress annotations + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: 'true' + + ## ClickHouse server Ingress hostnames + ## Must be provided if Ingress is enabled + ## + hosts: + - clickhouse.domain.com +``` + +### Ingress TLS + +If your cluster allows automatic creation/retrieval of TLS certificates (e.g. [kube-lego](https://github.com/jetstack/kube-lego)), please refer to the documentation for that mechanism. + +To manually configure TLS, first create/retrieve a key & certificate pair for the address(es) you wish to protect. Then create a TLS secret (named `clickhouse-server-tls` in this example) in the namespace. Include the secret's name, along with the desired hostnames, in the Ingress TLS section of your custom `values.yaml` file: + +```yaml +ingress: + ## If true, ClickHouse server Ingress will be created + ## + enabled: true + + ## ClickHouse server Ingress annotations + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: 'true' + + ## ClickHouse server Ingress hostnames + ## Must be provided if Ingress is enabled + ## + hosts: + - clickhouse.domain.com + + ## ClickHouse server Ingress TLS configuration + ## Secrets must be manually created in the namespace + ## + tls: + - secretName: clickhouse-server-tls + hosts: + - clickhouse.domain.com +``` + +### TLS secrets + +This chart facilitates the creation of TLS secrets for use with the Ingress controller (although this is not mandatory). There are several common use cases: + +- Generate certificate secrets based on chart parameters. +- Enable externally generated certificates. +- Manage application certificates via an external service (like [cert-manager](https://github.com/jetstack/cert-manager/)). +- Create self-signed certificates within the chart (if supported). + +In the first two cases, a certificate and a key are needed. Files are expected in `.pem` format. + +Here is an example of a certificate file: + +> NOTE: There may be more than one certificate if there is a certificate chain. + +```text +-----BEGIN CERTIFICATE----- +MIID6TCCAtGgAwIBAgIJAIaCwivkeB5EMA0GCSqGSIb3DQEBCwUAMFYxCzAJBgNV +... +jScrvkiBO65F46KioCL9h5tDvomdU1aqpI/CBzhvZn1c0ZTf87tGQR8NK7v7 +-----END CERTIFICATE----- +``` + +Here is an example of a certificate key: + +```text +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAvLYcyu8f3skuRyUgeeNpeDvYBCDcgq+LsWap6zbX5f8oLqp4 +... +wrj2wDbCDCFmfqnSJ+dKI3vFLlEz44sAV8jX/kd4Y6ZTQhlLbYc= +-----END RSA PRIVATE KEY----- +``` + +- If using Helm to manage the certificates based on the parameters, copy these values into the `certificate` and `key` values for a given `*.ingress.secrets` entry. +- If managing TLS secrets separately, it is necessary to create a TLS secret with name `INGRESS_HOSTNAME-tls` (where INGRESS_HOSTNAME is a placeholder to be replaced with the hostname you set using the `*.ingress.hostname` parameter). +- If your cluster has a [cert-manager](https://github.com/jetstack/cert-manager) add-on to automate the management and issuance of TLS certificates, add to `*.ingress.annotations` the [corresponding ones](https://cert-manager.io/docs/usage/ingress/#supported-annotations) for cert-manager. +- If using self-signed certificates created by Helm, set both `*.ingress.tls` and `*.ingress.selfSigned` to `true`. + +### Additional environment variables + +In case you want to add extra environment variables (useful for advanced operations like custom init scripts), you can use the `extraEnvVars` property. + +```yaml +clickhouse: + extraEnvVars: + - name: LOG_LEVEL + value: error +``` + +Alternatively, you can use a ConfigMap or a Secret with the environment variables. To do so, use the `extraEnvVarsCM` or the `extraEnvVarsSecret` values. + +### Sidecars + +If additional containers are needed in the same pod as ClickHouse (such as additional metrics or logging exporters), they can be defined using the `sidecars` parameter. + +```yaml +sidecars: +- name: your-image-name + image: your-image + imagePullPolicy: Always + ports: + - name: portname + containerPort: 1234 +``` + +If these sidecars export extra ports, extra port definitions can be added using the `service.extraPorts` parameter (where available), as shown in the example below: + +```yaml +service: + extraPorts: + - name: extraPort + port: 11311 + targetPort: 11311 +``` + +> NOTE: This Helm chart already includes sidecar containers for the Prometheus exporters (where applicable). These can be activated by adding the `--enable-metrics=true` parameter at deployment time. The `sidecars` parameter should therefore only be used for any extra sidecar containers. + +If additional init containers are needed in the same pod, they can be defined using the `initContainers` parameter. Here is an example: + +```yaml +initContainers: + - name: your-image-name + image: your-image + imagePullPolicy: Always + ports: + - name: portname + containerPort: 1234 +``` + +Learn more about [sidecar containers](https://kubernetes.io/docs/concepts/workloads/pods/) and [init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/). + +### Using custom scripts + +For advanced operations, the Bitnami ClickHouse chart allows using custom init and start scripts that will be mounted in `/docker-entrypoint.initdb.d` and `/docker-entrypoint.startdb.d` . The `init` scripts will be run on the first boot whereas the `start` scripts will be run on every container start. For adding the scripts directly as values use the `initdbScripts` and `startdbScripts` values. For using Secrets use the `initdbScriptsSecret` and `startdbScriptsSecret`. + +```yaml +initdbScriptsSecret: init-scripts-secret +startdbScriptsSecret: start-scripts-secret +``` + +### Pod affinity + +This chart allows you to set your custom affinity using the `affinity` parameter. Find more information about Pod affinity in the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). + +As an alternative, use one of the preset configurations for pod affinity, pod anti-affinity, and node affinity available at the [bitnami/common](https://github.com/bitnami/charts/tree/main/bitnami/common#affinities) chart. To do so, set the `podAffinityPreset`, `podAntiAffinityPreset`, or `nodeAffinityPreset` parameters. + +## Persistence + +The [Bitnami ClickHouse](https://github.com/bitnami/containers/tree/main/bitnami/clickhouse) image stores the ClickHouse data and configurations at the `/bitnami` path of the container. Persistent Volume Claims are used to keep the data across deployments. This is known to work in GCE, AWS, and minikube. + +## Parameters + +### Global parameters + +| Name | Description | Value | +| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | +| `global.imageRegistry` | Global Docker image registry | `""` | +| `global.imagePullSecrets` | Global Docker registry secret names as an array | `[]` | +| `global.storageClass` | Global StorageClass for Persistent Volume(s) | `""` | +| `global.compatibility.openshift.adaptSecurityContext` | Adapt the securityContext sections of the deployment to make them compatible with Openshift restricted-v2 SCC: remove runAsUser, runAsGroup and fsGroup and let the platform use their allowed default IDs. Possible values: auto (apply if the detected running cluster is Openshift), force (perform the adaptation always), disabled (do not perform adaptation) | `disabled` | + +### Common parameters + +| Name | Description | Value | +| ------------------------ | --------------------------------------------------------------------------------------- | --------------- | +| `kubeVersion` | Override Kubernetes version | `""` | +| `nameOverride` | String to partially override common.names.name | `""` | +| `fullnameOverride` | String to fully override common.names.fullname | `""` | +| `namespaceOverride` | String to fully override common.names.namespace | `""` | +| `commonLabels` | Labels to add to all deployed objects | `{}` | +| `commonAnnotations` | Annotations to add to all deployed objects | `{}` | +| `clusterDomain` | Kubernetes cluster domain name | `cluster.local` | +| `extraDeploy` | Array of extra objects to deploy with the release | `[]` | +| `diagnosticMode.enabled` | Enable diagnostic mode (all probes will be disabled and the command will be overridden) | `false` | +| `diagnosticMode.command` | Command to override all containers in the deployment | `["sleep"]` | +| `diagnosticMode.args` | Args to override all containers in the deployment | `["infinity"]` | + +### ClickHouse Parameters + +| Name | Description | Value | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | +| `image.registry` | ClickHouse image registry | `REGISTRY_NAME` | +| `image.repository` | ClickHouse image repository | `REPOSITORY_NAME/clickhouse` | +| `image.digest` | ClickHouse image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | `""` | +| `image.pullPolicy` | ClickHouse image pull policy | `IfNotPresent` | +| `image.pullSecrets` | ClickHouse image pull secrets | `[]` | +| `image.debug` | Enable ClickHouse image debug mode | `false` | +| `shards` | Number of ClickHouse shards to deploy | `2` | +| `replicaCount` | Number of ClickHouse replicas per shard to deploy | `3` | +| `distributeReplicasByZone` | Schedules replicas of the same shard to different availability zones | `false` | +| `containerPorts.http` | ClickHouse HTTP container port | `8123` | +| `containerPorts.https` | ClickHouse HTTPS container port | `8443` | +| `containerPorts.tcp` | ClickHouse TCP container port | `9000` | +| `containerPorts.tcpSecure` | ClickHouse TCP (secure) container port | `9440` | +| `containerPorts.keeper` | ClickHouse keeper TCP container port | `2181` | +| `containerPorts.keeperSecure` | ClickHouse keeper TCP (secure) container port | `3181` | +| `containerPorts.keeperInter` | ClickHouse keeper interserver TCP container port | `9444` | +| `containerPorts.mysql` | ClickHouse MySQL container port | `9004` | +| `containerPorts.postgresql` | ClickHouse PostgreSQL container port | `9005` | +| `containerPorts.interserver` | ClickHouse Interserver container port | `9009` | +| `containerPorts.metrics` | ClickHouse metrics container port | `8001` | +| `livenessProbe.enabled` | Enable livenessProbe on ClickHouse containers | `true` | +| `livenessProbe.initialDelaySeconds` | Initial delay seconds for livenessProbe | `10` | +| `livenessProbe.periodSeconds` | Period seconds for livenessProbe | `10` | +| `livenessProbe.timeoutSeconds` | Timeout seconds for livenessProbe | `1` | +| `livenessProbe.failureThreshold` | Failure threshold for livenessProbe | `3` | +| `livenessProbe.successThreshold` | Success threshold for livenessProbe | `1` | +| `readinessProbe.enabled` | Enable readinessProbe on ClickHouse containers | `true` | +| `readinessProbe.initialDelaySeconds` | Initial delay seconds for readinessProbe | `10` | +| `readinessProbe.periodSeconds` | Period seconds for readinessProbe | `10` | +| `readinessProbe.timeoutSeconds` | Timeout seconds for readinessProbe | `1` | +| `readinessProbe.failureThreshold` | Failure threshold for readinessProbe | `3` | +| `readinessProbe.successThreshold` | Success threshold for readinessProbe | `1` | +| `startupProbe.enabled` | Enable startupProbe on ClickHouse containers | `false` | +| `startupProbe.initialDelaySeconds` | Initial delay seconds for startupProbe | `10` | +| `startupProbe.periodSeconds` | Period seconds for startupProbe | `10` | +| `startupProbe.timeoutSeconds` | Timeout seconds for startupProbe | `1` | +| `startupProbe.failureThreshold` | Failure threshold for startupProbe | `3` | +| `startupProbe.successThreshold` | Success threshold for startupProbe | `1` | +| `customLivenessProbe` | Custom livenessProbe that overrides the default one | `{}` | +| `customReadinessProbe` | Custom readinessProbe that overrides the default one | `{}` | +| `customStartupProbe` | Custom startupProbe that overrides the default one | `{}` | +| `resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). | `none` | +| `resources` | Set container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | +| `podSecurityContext.enabled` | Enabled ClickHouse pods' Security Context | `true` | +| `podSecurityContext.fsGroupChangePolicy` | Set filesystem group change policy | `Always` | +| `podSecurityContext.sysctls` | Set kernel settings using the sysctl interface | `[]` | +| `podSecurityContext.supplementalGroups` | Set filesystem extra groups | `[]` | +| `podSecurityContext.fsGroup` | Set ClickHouse pod's Security Context fsGroup | `1001` | +| `containerSecurityContext.enabled` | Enable containers' Security Context | `true` | +| `containerSecurityContext.seLinuxOptions` | Set SELinux options in container | `nil` | +| `containerSecurityContext.runAsUser` | Set containers' Security Context runAsUser | `1001` | +| `containerSecurityContext.runAsGroup` | Set containers' Security Context runAsGroup | `0` | +| `containerSecurityContext.runAsNonRoot` | Set containers' Security Context runAsNonRoot | `true` | +| `containerSecurityContext.readOnlyRootFilesystem` | Set read only root file system pod's | `false` | +| `containerSecurityContext.privileged` | Set contraller container's Security Context privileged | `false` | +| `containerSecurityContext.allowPrivilegeEscalation` | Set contraller container's Security Context allowPrivilegeEscalation | `false` | +| `containerSecurityContext.capabilities.drop` | List of capabilities to be droppedn | `["ALL"]` | +| `containerSecurityContext.seccompProfile.type` | Set container's Security Context seccomp profile | `RuntimeDefault` | +| `auth.username` | ClickHouse Admin username | `default` | +| `auth.password` | ClickHouse Admin password | `""` | +| `auth.existingSecret` | Name of a secret containing the Admin password | `""` | +| `auth.existingSecretKey` | Name of the key inside the existing secret | `""` | +| `logLevel` | Logging level | `information` | + +### ClickHouse keeper configuration parameters + +| Name | Description | Value | +| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ----------------------- | +| `keeper.enabled` | Deploy ClickHouse keeper. Support is experimental. | `false` | +| `defaultConfigurationOverrides` | Default configuration overrides (evaluated as a template) | `""` | +| `existingOverridesConfigmap` | The name of an existing ConfigMap with your custom configuration for ClickHouse | `""` | +| `extraOverrides` | Extra configuration overrides (evaluated as a template) apart from the default | `""` | +| `extraOverridesConfigmap` | The name of an existing ConfigMap with extra configuration for ClickHouse | `""` | +| `extraOverridesSecret` | The name of an existing ConfigMap with your custom configuration for ClickHouse | `""` | +| `usersExtraOverrides` | Users extra configuration overrides (evaluated as a template) apart from the default | `""` | +| `usersExtraOverridesConfigmap` | The name of an existing ConfigMap with users extra configuration for ClickHouse | `""` | +| `usersExtraOverridesSecret` | The name of an existing ConfigMap with your custom users configuration for ClickHouse | `""` | +| `initdbScripts` | Dictionary of initdb scripts | `{}` | +| `initdbScriptsSecret` | ConfigMap with the initdb scripts (Note: Overrides `initdbScripts`) | `""` | +| `startdbScripts` | Dictionary of startdb scripts | `{}` | +| `startdbScriptsSecret` | ConfigMap with the startdb scripts (Note: Overrides `startdbScripts`) | `""` | +| `command` | Override default container command (useful when using custom images) | `["/scripts/setup.sh"]` | +| `args` | Override default container args (useful when using custom images) | `[]` | +| `automountServiceAccountToken` | Mount Service Account token in pod | `false` | +| `hostAliases` | ClickHouse pods host aliases | `[]` | +| `podLabels` | Extra labels for ClickHouse pods | `{}` | +| `podAnnotations` | Annotations for ClickHouse pods | `{}` | +| `podAffinityPreset` | Pod affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard` | `""` | +| `podAntiAffinityPreset` | Pod anti-affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard` | `soft` | +| `nodeAffinityPreset.type` | Node affinity preset type. Ignored if `affinity` is set. Allowed values: `soft` or `hard` | `""` | +| `nodeAffinityPreset.key` | Node label key to match. Ignored if `affinity` is set | `""` | +| `nodeAffinityPreset.values` | Node label values to match. Ignored if `affinity` is set | `[]` | +| `affinity` | Affinity for ClickHouse pods assignment | `{}` | +| `nodeSelector` | Node labels for ClickHouse pods assignment | `{}` | +| `tolerations` | Tolerations for ClickHouse pods assignment | `[]` | +| `updateStrategy.type` | ClickHouse statefulset strategy type | `RollingUpdate` | +| `podManagementPolicy` | Statefulset Pod management policy, it needs to be Parallel to be able to complete the cluster join | `Parallel` | +| `priorityClassName` | ClickHouse pods' priorityClassName | `""` | +| `topologySpreadConstraints` | Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template | `[]` | +| `schedulerName` | Name of the k8s scheduler (other than default) for ClickHouse pods | `""` | +| `terminationGracePeriodSeconds` | Seconds Redmine pod needs to terminate gracefully | `""` | +| `lifecycleHooks` | for the ClickHouse container(s) to automate configuration before or after startup | `{}` | +| `extraEnvVars` | Array with extra environment variables to add to ClickHouse nodes | `[]` | +| `extraEnvVarsCM` | Name of existing ConfigMap containing extra env vars for ClickHouse nodes | `""` | +| `extraEnvVarsSecret` | Name of existing Secret containing extra env vars for ClickHouse nodes | `""` | +| `extraVolumes` | Optionally specify extra list of additional volumes for the ClickHouse pod(s) | `[]` | +| `extraVolumeMounts` | Optionally specify extra list of additional volumeMounts for the ClickHouse container(s) | `[]` | +| `extraVolumeClaimTemplates` | Optionally specify extra list of additional volumeClaimTemplates for the ClickHouse container(s) | `[]` | +| `sidecars` | Add additional sidecar containers to the ClickHouse pod(s) | `[]` | +| `initContainers` | Add additional init containers to the ClickHouse pod(s) | `[]` | +| `tls.enabled` | Enable TLS traffic support | `false` | +| `tls.autoGenerated` | Generate automatically self-signed TLS certificates | `false` | +| `tls.certificatesSecret` | Name of an existing secret that contains the certificates | `""` | +| `tls.certFilename` | Certificate filename | `""` | +| `tls.certKeyFilename` | Certificate key filename | `""` | +| `tls.certCAFilename` | CA Certificate filename | `""` | + +### Traffic Exposure Parameters + +| Name | Description | Value | +| ------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `service.type` | ClickHouse service type | `ClusterIP` | +| `service.ports.http` | ClickHouse service HTTP port | `8123` | +| `service.ports.https` | ClickHouse service HTTPS port | `443` | +| `service.ports.tcp` | ClickHouse service TCP port | `9000` | +| `service.ports.tcpSecure` | ClickHouse service TCP (secure) port | `9440` | +| `service.ports.keeper` | ClickHouse keeper TCP container port | `2181` | +| `service.ports.keeperSecure` | ClickHouse keeper TCP (secure) container port | `3181` | +| `service.ports.keeperInter` | ClickHouse keeper interserver TCP container port | `9444` | +| `service.ports.mysql` | ClickHouse service MySQL port | `9004` | +| `service.ports.postgresql` | ClickHouse service PostgreSQL port | `9005` | +| `service.ports.interserver` | ClickHouse service Interserver port | `9009` | +| `service.ports.metrics` | ClickHouse service metrics port | `8001` | +| `service.nodePorts.http` | Node port for HTTP | `""` | +| `service.nodePorts.https` | Node port for HTTPS | `""` | +| `service.nodePorts.tcp` | Node port for TCP | `""` | +| `service.nodePorts.tcpSecure` | Node port for TCP (with TLS) | `""` | +| `service.nodePorts.keeper` | ClickHouse keeper TCP container port | `""` | +| `service.nodePorts.keeperSecure` | ClickHouse keeper TCP (secure) container port | `""` | +| `service.nodePorts.keeperInter` | ClickHouse keeper interserver TCP container port | `""` | +| `service.nodePorts.mysql` | Node port for MySQL | `""` | +| `service.nodePorts.postgresql` | Node port for PostgreSQL | `""` | +| `service.nodePorts.interserver` | Node port for Interserver | `""` | +| `service.nodePorts.metrics` | Node port for metrics | `""` | +| `service.clusterIP` | ClickHouse service Cluster IP | `""` | +| `service.loadBalancerIP` | ClickHouse service Load Balancer IP | `""` | +| `service.loadBalancerSourceRanges` | ClickHouse service Load Balancer sources | `[]` | +| `service.externalTrafficPolicy` | ClickHouse service external traffic policy | `Cluster` | +| `service.annotations` | Additional custom annotations for ClickHouse service | `{}` | +| `service.extraPorts` | Extra ports to expose in ClickHouse service (normally used with the `sidecars` value) | `[]` | +| `service.sessionAffinity` | Control where client requests go, to the same pod or round-robin | `None` | +| `service.sessionAffinityConfig` | Additional settings for the sessionAffinity | `{}` | +| `service.headless.annotations` | Annotations for the headless service. | `{}` | +| `externalAccess.enabled` | Enable Kubernetes external cluster access to ClickHouse | `false` | +| `externalAccess.service.type` | Kubernetes Service type for external access. It can be NodePort, LoadBalancer or ClusterIP | `LoadBalancer` | +| `externalAccess.service.ports.http` | ClickHouse service HTTP port | `80` | +| `externalAccess.service.ports.https` | ClickHouse service HTTPS port | `443` | +| `externalAccess.service.ports.tcp` | ClickHouse service TCP port | `9000` | +| `externalAccess.service.ports.tcpSecure` | ClickHouse service TCP (secure) port | `9440` | +| `externalAccess.service.ports.keeper` | ClickHouse keeper TCP container port | `2181` | +| `externalAccess.service.ports.keeperSecure` | ClickHouse keeper TCP (secure) container port | `3181` | +| `externalAccess.service.ports.keeperInter` | ClickHouse keeper interserver TCP container port | `9444` | +| `externalAccess.service.ports.mysql` | ClickHouse service MySQL port | `9004` | +| `externalAccess.service.ports.postgresql` | ClickHouse service PostgreSQL port | `9005` | +| `externalAccess.service.ports.interserver` | ClickHouse service Interserver port | `9009` | +| `externalAccess.service.ports.metrics` | ClickHouse service metrics port | `8001` | +| `externalAccess.service.loadBalancerIPs` | Array of load balancer IPs for each ClickHouse . Length must be the same as replicaCount | `[]` | +| `externalAccess.service.loadBalancerAnnotations` | Array of load balancer annotations for each ClickHouse . Length must be the same as shards multiplied by replicaCount | `[]` | +| `externalAccess.service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | `[]` | +| `externalAccess.service.nodePorts.http` | Node port for HTTP | `[]` | +| `externalAccess.service.nodePorts.https` | Node port for HTTPS | `[]` | +| `externalAccess.service.nodePorts.tcp` | Node port for TCP | `[]` | +| `externalAccess.service.nodePorts.tcpSecure` | Node port for TCP (with TLS) | `[]` | +| `externalAccess.service.nodePorts.keeper` | ClickHouse keeper TCP container port | `[]` | +| `externalAccess.service.nodePorts.keeperSecure` | ClickHouse keeper TCP container port (with TLS) | `[]` | +| `externalAccess.service.nodePorts.keeperInter` | ClickHouse keeper interserver TCP container port | `[]` | +| `externalAccess.service.nodePorts.mysql` | Node port for MySQL | `[]` | +| `externalAccess.service.nodePorts.postgresql` | Node port for PostgreSQL | `[]` | +| `externalAccess.service.nodePorts.interserver` | Node port for Interserver | `[]` | +| `externalAccess.service.nodePorts.metrics` | Node port for metrics | `[]` | +| `externalAccess.service.labels` | Service labels for external access | `{}` | +| `externalAccess.service.annotations` | Service annotations for external access | `{}` | +| `externalAccess.service.extraPorts` | Extra ports to expose in the ClickHouse external service | `[]` | +| `ingress.enabled` | Enable ingress record generation for ClickHouse | `false` | +| `ingress.pathType` | Ingress path type | `ImplementationSpecific` | +| `ingress.apiVersion` | Force Ingress API version (automatically detected if not set) | `""` | +| `ingress.hostname` | Default host for the ingress record | `clickhouse.local` | +| `ingress.ingressClassName` | IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+) | `""` | +| `ingress.path` | Default path for the ingress record | `/` | +| `ingress.annotations` | Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations. | `{}` | +| `ingress.tls` | Enable TLS configuration for the host defined at `ingress.hostname` parameter | `false` | +| `ingress.selfSigned` | Create a TLS secret for this ingress record using self-signed certificates generated by Helm | `false` | +| `ingress.extraHosts` | An array with additional hostname(s) to be covered with the ingress record | `[]` | +| `ingress.extraPaths` | An array with additional arbitrary paths that may need to be added to the ingress under the main host | `[]` | +| `ingress.extraTls` | TLS configuration for additional hostname(s) to be covered with this ingress record | `[]` | +| `ingress.secrets` | Custom TLS certificates as secrets | `[]` | +| `ingress.extraRules` | Additional rules to be covered with this ingress record | `[]` | + +### Persistence Parameters + +| Name | Description | Value | +| --------------------------- | ----------------------------------------------------------------------- | ------------------- | +| `persistence.enabled` | Enable persistence using Persistent Volume Claims | `true` | +| `persistence.existingClaim` | Name of an existing PVC to use | `""` | +| `persistence.storageClass` | Storage class of backing PVC | `""` | +| `persistence.labels` | Persistent Volume Claim labels | `{}` | +| `persistence.annotations` | Persistent Volume Claim annotations | `{}` | +| `persistence.accessModes` | Persistent Volume Access Modes | `["ReadWriteOnce"]` | +| `persistence.size` | Size of data volume | `8Gi` | +| `persistence.selector` | Selector to match an existing Persistent Volume for ClickHouse data PVC | `{}` | +| `persistence.dataSource` | Custom PVC data source | `{}` | + +### Init Container Parameters + +| Name | Description | Value | +| ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | +| `volumePermissions.enabled` | Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` | `false` | +| `volumePermissions.image.registry` | OS Shell + Utility image registry | `REGISTRY_NAME` | +| `volumePermissions.image.repository` | OS Shell + Utility image repository | `REPOSITORY_NAME/os-shell` | +| `volumePermissions.image.pullPolicy` | OS Shell + Utility image pull policy | `IfNotPresent` | +| `volumePermissions.image.pullSecrets` | OS Shell + Utility image pull secrets | `[]` | +| `volumePermissions.resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). | `none` | +| `volumePermissions.resources` | Set container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | +| `volumePermissions.containerSecurityContext.seLinuxOptions` | Set SELinux options in container | `nil` | +| `volumePermissions.containerSecurityContext.runAsUser` | Set init container's Security Context runAsUser | `0` | + +### Other Parameters + +| Name | Description | Value | +| --------------------------------------------- | ------------------------------------------------------------------------------------------------------ | ------- | +| `serviceAccount.create` | Specifies whether a ServiceAccount should be created | `true` | +| `serviceAccount.name` | The name of the ServiceAccount to use. | `""` | +| `serviceAccount.annotations` | Additional Service Account annotations (evaluated as a template) | `{}` | +| `serviceAccount.automountServiceAccountToken` | Automount service account token for the server service account | `false` | +| `metrics.enabled` | Enable the export of Prometheus metrics | `false` | +| `metrics.podAnnotations` | Annotations for metrics scraping | `{}` | +| `metrics.serviceMonitor.enabled` | if `true`, creates a Prometheus Operator ServiceMonitor (also requires `metrics.enabled` to be `true`) | `false` | +| `metrics.serviceMonitor.namespace` | Namespace in which Prometheus is running | `""` | +| `metrics.serviceMonitor.annotations` | Additional custom annotations for the ServiceMonitor | `{}` | +| `metrics.serviceMonitor.labels` | Extra labels for the ServiceMonitor | `{}` | +| `metrics.serviceMonitor.jobLabel` | The name of the label on the target service to use as the job name in Prometheus | `""` | +| `metrics.serviceMonitor.honorLabels` | honorLabels chooses the metric's labels on collisions with target labels | `false` | +| `metrics.serviceMonitor.interval` | Interval at which metrics should be scraped. | `""` | +| `metrics.serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `""` | +| `metrics.serviceMonitor.metricRelabelings` | Specify additional relabeling of metrics | `[]` | +| `metrics.serviceMonitor.relabelings` | Specify general relabeling | `[]` | +| `metrics.serviceMonitor.selector` | Prometheus instance selector labels | `{}` | +| `metrics.prometheusRule.enabled` | Create a PrometheusRule for Prometheus Operator | `false` | +| `metrics.prometheusRule.namespace` | Namespace for the PrometheusRule Resource (defaults to the Release Namespace) | `""` | +| `metrics.prometheusRule.additionalLabels` | Additional labels that can be used so PrometheusRule will be discovered by Prometheus | `{}` | +| `metrics.prometheusRule.rules` | PrometheusRule definitions | `[]` | + +### External Zookeeper paramaters + +| Name | Description | Value | +| --------------------------- | ----------------------------------------- | ------ | +| `externalZookeeper.servers` | List of external zookeeper servers to use | `[]` | +| `externalZookeeper.port` | Port of the Zookeeper servers | `2888` | + +### Zookeeper subchart parameters + +| Name | Description | Value | +| -------------------------------- | ----------------------------- | --------------------------- | +| `zookeeper.enabled` | Deploy Zookeeper subchart | `true` | +| `zookeeper.replicaCount` | Number of Zookeeper instances | `3` | +| `zookeeper.service.ports.client` | Zookeeper client port | `2181` | +| `zookeeper.image.registry` | Zookeeper image registry | `REGISTRY_NAME` | +| `zookeeper.image.repository` | Zookeeper image repository | `REPOSITORY_NAME/zookeeper` | +| `zookeeper.image.pullPolicy` | Zookeeper image pull policy | `IfNotPresent` | + +### Network Policies + +| Name | Description | Value | +| --------------------------------------- | --------------------------------------------------------------- | ------ | +| `networkPolicy.enabled` | Specifies whether a NetworkPolicy should be created | `true` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.allowExternalEgress` | Allow the pod to access any range of port and all destinations. | `true` | +| `networkPolicy.extraIngress` | Add extra ingress rules to the NetworkPolice | `[]` | +| `networkPolicy.extraEgress` | Add extra ingress rules to the NetworkPolicy | `[]` | +| `networkPolicy.ingressNSMatchLabels` | Labels to match to allow traffic from other namespaces | `{}` | +| `networkPolicy.ingressNSPodMatchLabels` | Pod labels to match to allow traffic from other namespaces | `{}` | + +See to create the table. + +The above parameters map to the env variables defined in [bitnami/clickhouse](https://github.com/bitnami/containers/tree/main/bitnami/clickhouse). For more information please refer to the [bitnami/clickhouse](https://github.com/bitnami/containers/tree/main/bitnami/clickhouse) image documentation. + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```console +helm install my-release \ + --set auth.username=admin \ + --set auth.password=password \ + oci://REGISTRY_NAME/REPOSITORY_NAME/clickhouse +``` + +> Note: You need to substitute the placeholders `REGISTRY_NAME` and `REPOSITORY_NAME` with a reference to your Helm chart registry and repository. For example, in the case of Bitnami, you need to use `REGISTRY_NAME=registry-1.docker.io` and `REPOSITORY_NAME=bitnamicharts`. + +The above command sets the ClickHouse administrator account username and password to `admin` and `password` respectively. + +> NOTE: Once this chart is deployed, it is not possible to change the application's access credentials, such as usernames or passwords, using Helm. To change these application credentials after deployment, delete any persistent volumes (PVs) used by the chart and re-deploy it, or use the application's built-in administrative tools if available. + +Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example, + +```console +helm install my-release -f values.yaml oci://REGISTRY_NAME/REPOSITORY_NAME/clickhouse +``` + +> Note: You need to substitute the placeholders `REGISTRY_NAME` and `REPOSITORY_NAME` with a reference to your Helm chart registry and repository. For example, in the case of Bitnami, you need to use `REGISTRY_NAME=registry-1.docker.io` and `REPOSITORY_NAME=bitnamicharts`. +> **Tip**: You can use the default [values.yaml](https://github.com/bitnami/charts/tree/main/bitnami/clickhouse/values.yaml) + +## Troubleshooting + +Find more information about how to deal with common errors related to Bitnami's Helm charts in [this troubleshooting guide](https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues). + +## Upgrading + +### To 2.0.0 + +This major updates the Zookeeper subchart to it newest major, 11.0.0. For more information on this subchart's major, please refer to [zookeeper upgrade notes](https://github.com/bitnami/charts/tree/main/bitnami/zookeeper#to-1100). + +## License + +Copyright © 2024 Broadcom. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/charts/clickhouse/templates/NOTES.txt b/charts/clickhouse/templates/NOTES.txt new file mode 100644 index 000000000..5bbc1f431 --- /dev/null +++ b/charts/clickhouse/templates/NOTES.txt @@ -0,0 +1,59 @@ +CHART NAME: {{ .Chart.Name }} +CHART VERSION: {{ .Chart.Version }} +APP VERSION: {{ .Chart.AppVersion }} + +** Please be patient while the chart is being deployed ** + +{{- if .Values.diagnosticMode.enabled }} +The chart has been deployed in diagnostic mode. All probes have been disabled and the command has been overwritten with: + + command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 4 }} + args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 4 }} + +Get the list of pods by executing: + + kubectl get pods --namespace {{ include "common.names.namespace" . | quote }} -l app.kubernetes.io/instance={{ .Release.Name }} + +Access the pod you want to debug by executing + + kubectl exec --namespace {{ include "common.names.namespace" . | quote }} -ti -- bash + +In order to replicate the container startup scripts execute this command: + + /opt/bitnami/scripts/clickhouse/entrypoint.sh /opt/bitnami/scripts/clickhouse/run.sh + +{{- else }} + +ClickHouse is available in the following address: + +{{- if .Values.externalAccess.enabled }} + +NOTE: It may take a few minutes for the LoadBalancer IP to be available. + + kubectl get svc --namespace {{ template "common.names.namespace" . }} -l "app.kubernetes.io/name={{ template "common.names.fullname" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=clickhouse" -w + +{{- else if (eq "LoadBalancer" .Values.service.type) }} + + export SERVICE_IP=$(kubectl get svc --namespace {{ template "common.names.namespace" . }} {{ template "common.names.fullname" . }} --template "{{ "{{ range (index .status.loadBalancer.ingress 0) }}{{ . }}{{ end }}" }}") + +{{- else if (eq "NodePort" .Values.service.type)}} + + export NODE_IP=$(kubectl get nodes --namespace {{ template "common.names.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + export NODE_PORT=$(kubectl get --namespace {{ template "common.names.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "common.names.fullname" . }}) + +{{- else if (eq "ClusterIP" .Values.service.type)}} + + kubectl port-forward --namespace {{ template "common.names.namespace" . }} svc/{{ template "common.names.fullname" . }} {{ .Values.service.ports.tcp }}:9000 & + +{{- end }} + +Credentials: + + echo "Username : {{ .Values.auth.username }}" + echo "Password : $(kubectl get secret --namespace {{ .Release.Namespace }} {{ include "clickhouse.secretName" . }} -o jsonpath="{.data.{{ include "clickhouse.secretKey" .}}}" | base64 -d)" + +{{- end }} + +{{- include "common.warnings.rollingTag" .Values.image }} +{{- include "clickhouse.validateValues" . }} +{{- include "common.warnings.resources" (dict "sections" (list "" "volumePermissions") "context" $) }} diff --git a/charts/clickhouse/templates/_helpers.tpl b/charts/clickhouse/templates/_helpers.tpl new file mode 100644 index 000000000..b52435267 --- /dev/null +++ b/charts/clickhouse/templates/_helpers.tpl @@ -0,0 +1,219 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* +Return the proper ClickHouse image name +*/}} +{{- define "clickhouse.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }} +{{- end -}} + +{{/* +Return the proper image name (for the init container volume-permissions image) +*/}} +{{- define "clickhouse.volumePermissions.image" -}} +{{- include "common.images.image" ( dict "imageRoot" .Values.volumePermissions.image "global" .Values.global ) -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "clickhouse.imagePullSecrets" -}} +{{- include "common.images.pullSecrets" (dict "images" (list .Values.image .Values.volumePermissions.image) "global" .Values.global) -}} +{{- end -}} + +{{/* +Return true if a TLS credentials secret object should be created +*/}} +{{- define "clickhouse.createTlsSecret" -}} +{{- if and .Values.tls.autoGenerated (not .Values.tls.certificatesSecret) }} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Return the path to the CA cert file. +*/}} +{{- define "clickhouse.tlsSecretName" -}} +{{- if .Values.tls.autoGenerated }} + {{- printf "%s-crt" (include "common.names.fullname" .) -}} +{{- else -}} + {{ required "A secret containing TLS certificates is required when TLS is enabled" .Values.tls.certificatesSecret }} +{{- end -}} +{{- end -}} + +{{/* +Return the path to the cert file. +*/}} +{{- define "clickhouse.tlsCert" -}} +{{- if .Values.tls.autoGenerated }} + {{- printf "/opt/bitnami/clickhouse/certs/tls.crt" -}} +{{- else -}} + {{- required "Certificate filename is required when TLS in enabled" .Values.tls.certFilename | printf "/opt/bitnami/clickhouse/certs/%s" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the path to the cert key file. +*/}} +{{- define "clickhouse.tlsCertKey" -}} +{{- if .Values.tls.autoGenerated }} + {{- printf "/opt/bitnami/clickhouse/certs/tls.key" -}} +{{- else -}} +{{- required "Certificate Key filename is required when TLS in enabled" .Values.tls.certKeyFilename | printf "/opt/bitnami/clickhouse/certs/%s" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the path to the CA cert file. +*/}} +{{- define "clickhouse.tlsCACert" -}} +{{- if .Values.tls.autoGenerated }} + {{- printf "/opt/bitnami/clickhouse/certs/ca.crt" -}} +{{- else -}} + {{- printf "/opt/bitnami/clickhouse/certs/%s" .Values.tls.certCAFilename -}} +{{- end -}} +{{- end -}} + +{{/* +Get the ClickHouse configuration configmap. +*/}} +{{- define "clickhouse.configmapName" -}} +{{- if .Values.existingOverridesConfigmap -}} + {{- .Values.existingOverridesConfigmap -}} +{{- else }} + {{- printf "%s" (include "common.names.fullname" . ) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the ClickHouse configuration configmap. +*/}} +{{- define "clickhouse.extraConfigmapName" -}} +{{- if .Values.extraOverridesConfigmap -}} + {{- .Values.extraOverridesConfigmap -}} +{{- else }} + {{- printf "%s-extra" (include "common.names.fullname" . ) -}} +{{- end -}} +{{- end -}} + + +{{/* +Get the ClickHouse configuration users configmap. +*/}} +{{- define "clickhouse.usersExtraConfigmapName" -}} +{{- if .Values.usersExtraOverridesConfigmap -}} + {{- .Values.usersExtraOverridesConfigmap -}} +{{- else }} + {{- printf "%s-users-extra" (include "common.names.fullname" . ) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the Clickhouse password secret name +*/}} +{{- define "clickhouse.secretName" -}} +{{- if .Values.auth.existingSecret -}} + {{- .Values.auth.existingSecret -}} +{{- else }} + {{- printf "%s" (include "common.names.fullname" . ) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the ClickHouse password key inside the secret +*/}} +{{- define "clickhouse.secretKey" -}} +{{- if .Values.auth.existingSecret -}} + {{- .Values.auth.existingSecretKey -}} +{{- else }} + {{- print "admin-password" -}} +{{- end -}} +{{- end -}} + +{{/* +Get the startialization scripts Secret name. +*/}} +{{- define "clickhouse.startdbScriptsSecret" -}} +{{- if .Values.startdbScriptsSecret -}} + {{- printf "%s" (tpl .Values.startdbScriptsSecret $) -}} +{{- else -}} + {{- printf "%s-start-scripts" (include "common.names.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Get the initialization scripts Secret name. +*/}} +{{- define "clickhouse.initdbScriptsSecret" -}} +{{- if .Values.initdbScriptsSecret -}} + {{- printf "%s" (tpl .Values.initdbScriptsSecret $) -}} +{{- else -}} + {{- printf "%s-init-scripts" (include "common.names.fullname" .) -}} +{{- end -}} +{{- end -}} + +{{/* +Return the path to the CA cert file. +*/}} +{{- define "clickhouse.headlessServiceName" -}} +{{- printf "%s-headless" (include "common.names.fullname" .) -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "clickhouse.zookeeper.fullname" -}} +{{- include "common.names.dependency.fullname" (dict "chartName" "zookeeper" "chartValues" .Values.zookeeper "context" $) -}} +{{- end -}} + +{{/* +Return the path to the CA cert file. +*/}} +{{- define "clickhouse.zookeeper.headlessServiceName" -}} +{{- printf "%s-headless" (include "clickhouse.zookeeper.fullname" .) -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "clickhouse.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "common.names.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Compile all warnings into a single message. +*/}} +{{- define "clickhouse.validateValues" -}} +{{- $messages := list -}} +{{- $messages := append $messages (include "clickhouse.validateValues.zookeeper" .) -}} +{{- $messages := without $messages "" -}} +{{- $message := join "\n" $messages -}} + +{{- if $message -}} +{{- printf "\nVALUES VALIDATION:\n%s" $message -}} +{{- end -}} +{{- end -}} + +{{/* Validate values of ClickHouse - [Zoo]keeper */}} +{{- define "clickhouse.validateValues.zookeeper" -}} +{{- if or (and .Values.keeper.enabled .Values.zookeeper.enabled) (and .Values.keeper.enabled .Values.externalZookeeper.servers) (and .Values.zookeeper.enabled .Values.externalZookeeper.servers) -}} +clickhouse: Multiple [Zoo]keeper + You can only use one [zoo]keeper + Please choose use ClickHouse keeper or + installing a Zookeeper chart (--set zookeeper.enabled=true) or + using an external instance (--set zookeeper.servers ) +{{- end -}} +{{- if and (not .Values.keeper.enabled) (not .Values.zookeeper.enabled) (not .Values.externalZookeeper.servers) (ne (int .Values.shards) 1) (ne (int .Values.replicaCount) 1) -}} +clickhouse: No [Zoo]keeper + If you are deploying more than one ClickHouse instance, you need to enable [Zoo]keeper. Please choose installing a [Zoo]keeper (--set keeper.enabled=true) or (--set zookeeper.enabled=true) or + using an external instance (--set zookeeper.servers ) +{{- end -}} +{{- end -}} diff --git a/charts/clickhouse/templates/configmap-extra.yaml b/charts/clickhouse/templates/configmap-extra.yaml new file mode 100644 index 000000000..153cf4d50 --- /dev/null +++ b/charts/clickhouse/templates/configmap-extra.yaml @@ -0,0 +1,20 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.extraOverrides (not .Values.extraOverridesConfigmap) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-extra" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + 01_extra_overrides.xml: | + {{- include "common.tplvalues.render" (dict "value" .Values.extraOverrides "context" $) | nindent 4 }} +{{- end }} diff --git a/charts/clickhouse/templates/configmap-users-extra.yaml b/charts/clickhouse/templates/configmap-users-extra.yaml new file mode 100644 index 000000000..056d2d026 --- /dev/null +++ b/charts/clickhouse/templates/configmap-users-extra.yaml @@ -0,0 +1,20 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.usersExtraOverrides (not .Values.usersExtraOverridesConfigmap) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-users-extra" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + 01_users_extra_overrides.xml: | + {{- include "common.tplvalues.render" (dict "value" .Values.usersExtraOverrides "context" $) | nindent 4 }} +{{- end }} diff --git a/charts/clickhouse/templates/configmap.yaml b/charts/clickhouse/templates/configmap.yaml new file mode 100644 index 000000000..2462712b5 --- /dev/null +++ b/charts/clickhouse/templates/configmap.yaml @@ -0,0 +1,20 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if not .Values.existingOverridesConfigmap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + 00_default_overrides.xml: | + {{- include "common.tplvalues.render" (dict "value" .Values.defaultConfigurationOverrides "context" $) | nindent 4 }} +{{- end }} diff --git a/charts/clickhouse/templates/extra-list.yaml b/charts/clickhouse/templates/extra-list.yaml new file mode 100644 index 000000000..2d35a580e --- /dev/null +++ b/charts/clickhouse/templates/extra-list.yaml @@ -0,0 +1,9 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- range .Values.extraDeploy }} +--- +{{ include "common.tplvalues.render" (dict "value" . "context" $) }} +{{- end }} diff --git a/charts/clickhouse/templates/ingress-tls-secrets.yaml b/charts/clickhouse/templates/ingress-tls-secrets.yaml new file mode 100644 index 000000000..6ef20e36a --- /dev/null +++ b/charts/clickhouse/templates/ingress-tls-secrets.yaml @@ -0,0 +1,44 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if .Values.ingress.enabled }} +{{- if .Values.ingress.secrets }} +{{- range .Values.ingress.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .name }} + namespace: {{ $.Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $.Values.commonLabels "context" $ ) | nindent 4 }} + {{- if $.Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $.Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + tls.crt: {{ .certificate | b64enc }} + tls.key: {{ .key | b64enc }} +--- +{{- end }} +{{- end }} +{{- if and .Values.ingress.tls .Values.ingress.selfSigned }} +{{- $secretName := printf "%s-tls" .Values.ingress.hostname }} +{{- $ca := genCA "clickhouse-ca" 365 }} +{{- $cert := genSignedCert .Values.ingress.hostname nil (list .Values.ingress.hostname) 365 $ca }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + tls.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.crt" "defaultValue" $cert.Cert "context" $) }} + tls.key: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.key" "defaultValue" $cert.Key "context" $) }} + ca.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "ca.crt" "defaultValue" $ca.Cert "context" $) }} +{{- end }} +{{- end }} diff --git a/charts/clickhouse/templates/ingress.yaml b/charts/clickhouse/templates/ingress.yaml new file mode 100644 index 000000000..7000ecebf --- /dev/null +++ b/charts/clickhouse/templates/ingress.yaml @@ -0,0 +1,59 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if .Values.ingress.enabled }} +apiVersion: {{ include "common.capabilities.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if or .Values.ingress.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.ingress.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.ingressClassName (eq "true" (include "common.ingress.supportsIngressClassname" .)) }} + ingressClassName: {{ .Values.ingress.ingressClassName | quote }} + {{- end }} + rules: + {{- if .Values.ingress.hostname }} + - host: {{ .Values.ingress.hostname | quote }} + http: + paths: + {{- if .Values.ingress.extraPaths }} + {{- toYaml .Values.ingress.extraPaths | nindent 10 }} + {{- end }} + - path: {{ .Values.ingress.path }} + {{- if eq "true" (include "common.ingress.supportsPathType" .) }} + pathType: {{ .Values.ingress.pathType }} + {{- end }} + backend: {{- include "common.ingress.backend" (dict "serviceName" (include "common.names.fullname" .) "servicePort" "http" "context" $) | nindent 14 }} + {{- end }} + {{- range .Values.ingress.extraHosts }} + - host: {{ .name | quote }} + http: + paths: + - path: {{ default "/" .path }} + {{- if eq "true" (include "common.ingress.supportsPathType" $) }} + pathType: {{ default "ImplementationSpecific" .pathType }} + {{- end }} + backend: {{- include "common.ingress.backend" (dict "serviceName" (include "common.names.fullname" $) "servicePort" "http" "context" $) | nindent 14 }} + {{- end }} + {{- if .Values.ingress.extraRules }} + {{- include "common.tplvalues.render" (dict "value" .Values.ingress.extraRules "context" $) | nindent 4 }} + {{- end }} + {{- if or (and .Values.ingress.tls (or (include "common.ingress.certManagerRequest" ( dict "annotations" .Values.ingress.annotations )) .Values.ingress.selfSigned)) .Values.ingress.extraTls }} + tls: + {{- if and .Values.ingress.tls (or (include "common.ingress.certManagerRequest" ( dict "annotations" .Values.ingress.annotations )) .Values.ingress.selfSigned) }} + - hosts: + - {{ .Values.ingress.hostname | quote }} + secretName: {{ printf "%s-tls" .Values.ingress.hostname }} + {{- end }} + {{- if .Values.ingress.extraTls }} + {{- include "common.tplvalues.render" (dict "value" .Values.ingress.extraTls "context" $) | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/clickhouse/templates/init-scripts-secret.yaml b/charts/clickhouse/templates/init-scripts-secret.yaml new file mode 100644 index 000000000..323670930 --- /dev/null +++ b/charts/clickhouse/templates/init-scripts-secret.yaml @@ -0,0 +1,19 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.initdbScripts (not .Values.initdbScriptsSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-init-scripts" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +stringData: +{{- include "common.tplvalues.render" (dict "value" .Values.initdbScripts "context" .) | nindent 2 }} +{{- end }} diff --git a/charts/clickhouse/templates/networkpolicy.yaml b/charts/clickhouse/templates/networkpolicy.yaml new file mode 100644 index 000000000..68635e094 --- /dev/null +++ b/charts/clickhouse/templates/networkpolicy.yaml @@ -0,0 +1,136 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if .Values.networkPolicy.enabled }} +kind: NetworkPolicy +apiVersion: {{ include "common.capabilities.networkPolicy.apiVersion" . }} +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.podLabels .Values.commonLabels ) "context" . ) }} + podSelector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + app.kubernetes.io/component: clickhouse + policyTypes: + - Ingress + - Egress + {{- if .Values.networkPolicy.allowExternalEgress }} + egress: + - {} + {{- else }} + egress: + # Allow dns resolution + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # Allow outbound connections to other cluster pods + - ports: + - port: {{ .Values.service.ports.http }} + {{- if .Values.tls.enabled }} + - port: {{ .Values.service.ports.https }} + {{- end }} + - port: {{ .Values.service.ports.tcp }} + {{- if .Values.tls.enabled }} + - port: {{ .Values.service.ports.tcpSecure }} + {{- end }} + {{- if .Values.keeper.enabled }} + - port: {{ .Values.service.ports.keeper }} + - port: {{ .Values.service.ports.keeperInter }} + {{- if .Values.tls.enabled }} + - port: {{ .Values.service.ports.keeperSecure }} + {{- end }} + {{- end }} + - port: {{ .Values.service.ports.mysql }} + - port: {{ .Values.service.ports.postgresql }} + - port: {{ .Values.service.ports.interserver }} + {{- if .Values.metrics.enabled }} + - port: {{ .Values.service.ports.metrics }} + {{- end }} + {{- if $.Values.externalAccess.enabled }} + - port: {{ $.Values.externalAccess.service.ports.http }} + {{- if $.Values.tls.enabled }} + - port: {{ $.Values.externalAccess.service.ports.https }} + {{- end }} + {{- if $.Values.metrics.enabled }} + - port: {{ $.Values.externalAccess.service.ports.metrics }} + {{- end }} + - port: {{ $.Values.externalAccess.service.ports.tcp }} + {{- if $.Values.tls.enabled }} + - port: {{ $.Values.externalAccess.service.ports.tcpSecure }} + {{- end }} + {{- if $.Values.keeper.enabled }} + - port: {{ $.Values.externalAccess.service.ports.keeper }} + - port: {{ $.Values.externalAccess.service.ports.keeperInter }} + {{- if $.Values.tls.enabled }} + - port: {{ $.Values.externalAccess.service.ports.keeperSecure }} + {{- end }} + {{- end }} + - port: {{ $.Values.externalAccess.service.ports.mysql }} + - port: {{ $.Values.externalAccess.service.ports.postgresql }} + - port: {{ $.Values.externalAccess.service.ports.interserver }} + {{- end }} + to: + - podSelector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 14 }} + {{- if .Values.networkPolicy.extraEgress }} + {{- include "common.tplvalues.render" ( dict "value" .Values.networkPolicy.extraEgress "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} + ingress: + - ports: + - port: {{ $.Values.containerPorts.http }} + - port: {{ $.Values.containerPorts.tcp }} + - port: {{ $.Values.containerPorts.mysql }} + - port: {{ $.Values.containerPorts.postgresql }} + - port: {{ $.Values.containerPorts.interserver }} + {{- if $.Values.tls.enabled }} + - port: {{ $.Values.containerPorts.tcpSecure }} + - port: {{ $.Values.containerPorts.https }} + {{- end }} + {{- if $.Values.keeper.enabled }} + - port: {{ $.Values.containerPorts.keeper }} + - port: {{ $.Values.containerPorts.keeperInter }} + {{- if $.Values.tls.enabled }} + - port : {{ $.Values.containerPorts.keeperSecure }} + {{- end }} + {{- end }} + {{- if $.Values.metrics.enabled }} + - port: {{ $.Values.containerPorts.metrics }} + {{- end }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 14 }} + app.kubernetes.io/component: clickhouse + - podSelector: + matchLabels: + {{ include "common.names.fullname" . }}-client: "true" + {{- if .Values.networkPolicy.ingressNSMatchLabels }} + - namespaceSelector: + matchLabels: + {{- range $key, $value := .Values.networkPolicy.ingressNSMatchLabels }} + {{ $key | quote }}: {{ $value | quote }} + {{- end }} + {{- if .Values.networkPolicy.ingressNSPodMatchLabels }} + podSelector: + matchLabels: + {{- range $key, $value := .Values.networkPolicy.ingressNSPodMatchLabels }} + {{ $key | quote }}: {{ $value | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.networkPolicy.extraIngress }} + {{- include "common.tplvalues.render" ( dict "value" .Values.networkPolicy.extraIngress "context" $ ) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/clickhouse/templates/prometheusrule.yaml b/charts/clickhouse/templates/prometheusrule.yaml new file mode 100644 index 000000000..dc2d05d3a --- /dev/null +++ b/charts/clickhouse/templates/prometheusrule.yaml @@ -0,0 +1,24 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.metrics.enabled .Values.metrics.prometheusRule.enabled .Values.metrics.prometheusRule.rules }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ default .Release.Namespace .Values.metrics.prometheusRule.namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: metrics + {{- if .Values.metrics.prometheusRule.additionalLabels }} + {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.prometheusRule.additionalLabels "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + groups: + - name: {{ include "common.names.fullname" . }} + rules: {{- toYaml .Values.metrics.prometheusRule.rules | nindent 8 }} +{{- end }} diff --git a/charts/clickhouse/templates/scripts-configmap.yaml b/charts/clickhouse/templates/scripts-configmap.yaml new file mode 100644 index 000000000..86aa34dcd --- /dev/null +++ b/charts/clickhouse/templates/scripts-configmap.yaml @@ -0,0 +1,34 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-scripts" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + setup.sh: |- + #!/bin/bash + + # Execute entrypoint as usual after obtaining KEEPER_SERVER_ID + # check KEEPER_SERVER_ID in persistent volume via myid + # if not present, set based on POD hostname + if [[ -f "/bitnami/clickhouse/keeper/data/myid" ]]; then + export KEEPER_SERVER_ID="$(cat /bitnami/clickhouse/keeper/data/myid)" + else + HOSTNAME="$(hostname -s)" + if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then + export KEEPER_SERVER_ID=${BASH_REMATCH[2]} + else + echo "Failed to get index from hostname $HOST" + exit 1 + fi + fi + exec /opt/bitnami/scripts/clickhouse/entrypoint.sh /opt/bitnami/scripts/clickhouse/run.sh -- --listen_host=0.0.0.0 diff --git a/charts/clickhouse/templates/secret.yaml b/charts/clickhouse/templates/secret.yaml new file mode 100644 index 000000000..16068010a --- /dev/null +++ b/charts/clickhouse/templates/secret.yaml @@ -0,0 +1,20 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if not .Values.auth.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: Opaque +data: + admin-password: {{ include "common.secrets.passwords.manage" (dict "secret" (include "common.names.fullname" .) "key" "admin-password" "providedValues" (list "auth.password") "context" $) }} +{{- end }} diff --git a/charts/clickhouse/templates/service-account.yaml b/charts/clickhouse/templates/service-account.yaml new file mode 100644 index 000000000..649086dac --- /dev/null +++ b/charts/clickhouse/templates/service-account.yaml @@ -0,0 +1,19 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "clickhouse.serviceAccountName" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if or .Values.serviceAccount.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.serviceAccount.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} +{{- end }} diff --git a/charts/clickhouse/templates/service-external-access.yaml b/charts/clickhouse/templates/service-external-access.yaml new file mode 100644 index 000000000..f50baa214 --- /dev/null +++ b/charts/clickhouse/templates/service-external-access.yaml @@ -0,0 +1,155 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if $.Values.externalAccess.enabled }} +{{- $shards := $.Values.shards | int }} +{{- $replicas := $.Values.replicaCount | int }} +{{- $totalNodes := mul $shards $replicas }} +{{- range $shard, $e := until $shards }} +{{- range $i, $_e := until $replicas }} +{{- $loadBalancerAnnotationPosOffset := mul $shard $replicas }} +{{- $loadBalancerAnnotationPosition := add $loadBalancerAnnotationPosOffset $i }} +{{- $targetPod := printf "%s-shard%d-%d" (include "common.names.fullname" $) $shard $i }} +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-external" $targetPod | trunc 63 | trimSuffix "-" }} + namespace: {{ $.Release.Namespace | quote }} + {{- $labels := include "common.tplvalues.merge" ( dict "values" ( list $.Values.externalAccess.service.labels $.Values.commonLabels ) "context" $ ) }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $labels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + pod: {{ $targetPod }} + {{- if or $.Values.externalAccess.service.annotations $.Values.commonAnnotations $.Values.externalAccess.service.loadBalancerAnnotations }} + annotations: + {{- if and (not (empty $.Values.externalAccess.service.loadBalancerAnnotations)) (eq (len $.Values.externalAccess.service.loadBalancerAnnotations) $totalNodes) }} + {{ include "common.tplvalues.render" ( dict "value" (index $.Values.externalAccess.service.loadBalancerAnnotations $loadBalancerAnnotationPosition) "context" $) | nindent 4 }} + {{- end }} + {{- if $.Values.externalAccess.service.annotations }} + {{- include "common.tplvalues.render" ( dict "value" $.Values.externalAccess.service.annotations "context" $) | nindent 4 }} + {{- end }} + {{- if $.Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" $.Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} +spec: + type: {{ $.Values.externalAccess.service.type }} + {{- if eq $.Values.externalAccess.service.type "LoadBalancer" }} + {{- if and (not (empty $.Values.externalAccess.service.loadBalancerIPs)) (eq (len $.Values.externalAccess.service.loadBalancerIPs) $totalNodes) }} + loadBalancerIP: {{ index $.Values.externalAccess.service.loadBalancerIPs $i }} + {{- end }} + {{- if $.Values.externalAccess.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{- toYaml $.Values.externalAccess.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- end }} + ports: + - name: http + port: {{ $.Values.externalAccess.service.ports.http }} + targetPort: http + {{- if not (empty $.Values.externalAccess.service.nodePorts.http) }} + nodePort: {{ index $.Values.externalAccess.service.nodePorts.http $i }} + {{- else }} + nodePort: null + {{- end }} + {{- if $.Values.tls.enabled }} + - name: https + port: {{ $.Values.externalAccess.service.ports.https }} + targetPort: https + {{- if not (empty $.Values.externalAccess.service.nodePorts.https) }} + nodePort: {{ index $.Values.externalAccess.service.nodePorts.https $i }} + {{- else }} + nodePort: null + {{- end }} + {{- end }} + {{- if $.Values.metrics.enabled }} + - name: http-metrics + port: {{ $.Values.externalAccess.service.ports.metrics }} + targetPort: http-metrics + {{- if not (empty $.Values.externalAccess.service.nodePorts.metrics) }} + nodePort: {{ index $.Values.externalAccess.service.nodePorts.metrics $i }} + {{- else }} + nodePort: null + {{- end }} + {{- end }} + - name: tcp + port: {{ $.Values.externalAccess.service.ports.tcp }} + targetPort: tcp + {{- if not (empty $.Values.externalAccess.service.nodePorts.tcp) }} + nodePort: {{ index $.Values.externalAccess.service.nodePorts.tcp $i }} + {{- else }} + nodePort: null + {{- end }} + {{- if $.Values.tls.enabled }} + - name: tcp-secure + port: {{ $.Values.externalAccess.service.ports.tcpSecure }} + targetPort: tcp-secure + {{- if not (empty $.Values.externalAccess.service.nodePorts.tcpSecure) }} + nodePort: {{ index $.Values.externalAccess.service.nodePorts.tcpSecure $i }} + {{- else }} + nodePort: null + {{- end }} + {{- end }} + {{- if $.Values.keeper.enabled }} + - name: tcp-keeper + port: {{ $.Values.externalAccess.service.ports.keeper }} + targetPort: tcp-keeper + {{- if not (empty $.Values.externalAccess.service.nodePorts.keeper) }} + nodePort: {{ index $.Values.externalAccess.service.nodePorts.keeper $i }} + {{- else }} + nodePort: null + {{- end }} + - name: tcp-keeperinter + port: {{ $.Values.externalAccess.service.ports.keeperInter }} + targetPort: tcp-keeperinter + {{- if not (empty $.Values.externalAccess.service.nodePorts.keeperInter) }} + nodePort: {{ index $.Values.externalAccess.service.nodePorts.keeperInter $i }} + {{- else }} + nodePort: null + {{- end }} + {{- if $.Values.tls.enabled }} + - name: tcp-keepertls + port: {{ $.Values.externalAccess.service.ports.keeperSecure }} + targetPort: tcp-keepertls + {{- if not (empty $.Values.externalAccess.service.nodePorts.keeperSecure) }} + nodePort: {{ index $.Values.externalAccess.service.nodePorts.keeperSecure $i }} + {{- else }} + nodePort: null + {{- end }} + {{- end }} + {{- end }} + - name: tcp-mysql + port: {{ $.Values.externalAccess.service.ports.mysql }} + targetPort: tcp-mysql + {{- if not (empty $.Values.externalAccess.service.nodePorts.mysql) }} + nodePort: {{ index $.Values.externalAccess.service.nodePorts.mysql $i }} + {{- else }} + nodePort: null + {{- end }} + - name: tcp-postgresql + port: {{ $.Values.externalAccess.service.ports.postgresql }} + targetPort: tcp-postgresql + {{- if not (empty $.Values.externalAccess.service.nodePorts.postgresql) }} + nodePort: {{ index $.Values.externalAccess.service.nodePorts.postgresql $i }} + {{- else }} + nodePort: null + {{- end }} + - name: tcp-intersrv + port: {{ $.Values.externalAccess.service.ports.interserver }} + targetPort: tcp-intersrv + {{- if not (empty $.Values.externalAccess.service.nodePorts.interserver) }} + nodePort: {{ index $.Values.externalAccess.service.nodePorts.interserver $i }} + {{- else }} + nodePort: null + {{- end }} + {{- if $.Values.externalAccess.service.extraPorts }} + {{- include "common.tplvalues.render" (dict "value" $.Values.externalAccess.service.extraPorts "context" $) | nindent 4 }} + {{- end }} + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list $.Values.podLabels $.Values.commonLabels ) "context" $ ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + statefulset.kubernetes.io/pod-name: {{ $targetPod }} +--- +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/clickhouse/templates/service-headless.yaml b/charts/clickhouse/templates/service-headless.yaml new file mode 100644 index 000000000..f989841bd --- /dev/null +++ b/charts/clickhouse/templates/service-headless.yaml @@ -0,0 +1,69 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "clickhouse.headlessServiceName" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if or .Values.service.headless.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.service.headless.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: http + targetPort: http + port: {{ .Values.service.ports.http }} + protocol: TCP + - name: tcp + targetPort: tcp + port: {{ .Values.service.ports.tcp }} + protocol: TCP + {{- if .Values.tls.enabled }} + - name: tcp-secure + targetPort: tcp-secure + port: {{ .Values.service.ports.tcpSecure }} + protocol: TCP + {{- end }} + {{- if .Values.keeper.enabled }} + - name: tcp-keeper + targetPort: tcp-keeper + port: {{ .Values.service.ports.keeper }} + protocol: TCP + - name: tcp-keeperinter + targetPort: tcp-keeperinter + port: {{ .Values.service.ports.keeperInter }} + protocol: TCP + {{- if .Values.tls.enabled }} + - name: tcp-keepertls + targetPort: tcp-keepertls + port: {{ .Values.service.ports.keeperSecure }} + protocol: TCP + {{- end }} + {{- end }} + - name: tcp-mysql + targetPort: tcp-mysql + port: {{ .Values.service.ports.mysql }} + protocol: TCP + - name: tcp-postgresql + targetPort: tcp-postgresql + port: {{ .Values.service.ports.postgresql }} + protocol: TCP + - name: http-intersrv + targetPort: http-intersrv + port: {{ .Values.service.ports.interserver }} + protocol: TCP + {{- if .Values.service.extraPorts }} + {{- include "common.tplvalues.render" (dict "value" .Values.service.extraPorts "context" $) | nindent 4 }} + {{- end }} + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse diff --git a/charts/clickhouse/templates/service.yaml b/charts/clickhouse/templates/service.yaml new file mode 100644 index 000000000..f54e2268a --- /dev/null +++ b/charts/clickhouse/templates/service.yaml @@ -0,0 +1,152 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if or .Values.service.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.service.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if and .Values.service.clusterIP (eq .Values.service.type "ClusterIP") }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if .Values.service.sessionAffinity }} + sessionAffinity: {{ .Values.service.sessionAffinity }} + {{- end }} + {{- if .Values.service.sessionAffinityConfig }} + sessionAffinityConfig: {{- include "common.tplvalues.render" (dict "value" .Values.service.sessionAffinityConfig "context" $) | nindent 4 }} + {{- end }} + {{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort") }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | quote }} + {{- end }} + {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerSourceRanges)) }} + loadBalancerSourceRanges: {{- toYaml .Values.service.loadBalancerSourceRanges | nindent 4 }} + {{- end }} + {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP)) }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + ports: + - name: http + targetPort: http + port: {{ .Values.service.ports.http }} + protocol: TCP + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.http)) }} + nodePort: {{ .Values.service.nodePorts.http }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- if .Values.tls.enabled }} + - name: https + targetPort: https + port: {{ .Values.service.ports.https }} + protocol: TCP + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.https)) }} + nodePort: {{ .Values.service.nodePorts.https }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- end }} + - name: tcp + targetPort: tcp + port: {{ .Values.service.ports.tcp }} + protocol: TCP + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.tcp)) }} + nodePort: {{ .Values.service.nodePorts.tcp }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- if .Values.tls.enabled }} + - name: tcp-secure + targetPort: tcp-secure + port: {{ .Values.service.ports.tcpSecure }} + protocol: TCP + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.tcpSecure)) }} + nodePort: {{ .Values.service.nodePorts.tcpSecure }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- end }} + {{- if .Values.keeper.enabled }} + - name: tcp-keeper + targetPort: tcp-keeper + port: {{ .Values.service.ports.keeper }} + protocol: TCP + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.tcp)) }} + nodePort: {{ .Values.service.nodePorts.keeper }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + - name: tcp-keeperinter + targetPort: tcp-keeperinter + port: {{ .Values.service.ports.keeperInter }} + protocol: TCP + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.tcp)) }} + nodePort: {{ .Values.service.nodePorts.keeperInter }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- if .Values.tls.enabled }} + - name: tcp-keepertls + targetPort: tcp-keepertls + port: {{ .Values.service.ports.keeperSecure }} + protocol: TCP + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.tcpSecure)) }} + nodePort: {{ .Values.service.nodePorts.keeperSecure }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- end }} + {{- end }} + - name: tcp-mysql + targetPort: tcp-mysql + port: {{ .Values.service.ports.mysql }} + protocol: TCP + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.mysql)) }} + nodePort: {{ .Values.service.nodePorts.mysql }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + - name: tcp-postgresql + targetPort: tcp-postgresql + port: {{ .Values.service.ports.postgresql }} + protocol: TCP + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.postgresql)) }} + nodePort: {{ .Values.service.nodePorts.postgresql }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + - name: http-intersrv + targetPort: http-intersrv + port: {{ .Values.service.ports.interserver }} + protocol: TCP + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.interserver)) }} + nodePort: {{ .Values.service.nodePorts.interserver }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- if .Values.metrics.enabled }} + - name: http-metrics + targetPort: http-metrics + port: {{ .Values.service.ports.metrics }} + protocol: TCP + {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.metrics)) }} + nodePort: {{ .Values.service.nodePorts.metrics }} + {{- else if eq .Values.service.type "ClusterIP" }} + nodePort: null + {{- end }} + {{- end }} + {{- if .Values.service.extraPorts }} + {{- include "common.tplvalues.render" (dict "value" .Values.service.extraPorts "context" $) | nindent 4 }} + {{- end }} + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse diff --git a/charts/clickhouse/templates/servicemonitor.yaml b/charts/clickhouse/templates/servicemonitor.yaml new file mode 100644 index 000000000..2148b375d --- /dev/null +++ b/charts/clickhouse/templates/servicemonitor.yaml @@ -0,0 +1,47 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ default (include "common.names.namespace" .) .Values.metrics.serviceMonitor.namespace | quote }} + {{- $labels := include "common.tplvalues.merge" ( dict "values" ( list .Values.metrics.serviceMonitor.labels .Values.commonLabels ) "context" . ) }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $labels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if or .Values.metrics.serviceMonitor.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.metrics.serviceMonitor.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ .Values.metrics.serviceMonitor.jobLabel | quote }} + selector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 6 }} + {{- if .Values.metrics.serviceMonitor.selector }} + {{- include "common.tplvalues.render" (dict "value" .Values.metrics.serviceMonitor.selector "context" $) | nindent 6 }} + {{- end }} + endpoints: + - port: http-metrics + path: "/metrics" + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.honorLabels }} + honorLabels: {{ .Values.metrics.serviceMonitor.honorLabels }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.serviceMonitor.metricRelabelings "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.relabelings }} + relabelings: {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.serviceMonitor.relabelings "context" $) | nindent 8 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "common.names.namespace" . | quote }} +{{- end }} diff --git a/charts/clickhouse/templates/start-scripts-secret.yaml b/charts/clickhouse/templates/start-scripts-secret.yaml new file mode 100644 index 000000000..c579f2e46 --- /dev/null +++ b/charts/clickhouse/templates/start-scripts-secret.yaml @@ -0,0 +1,19 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if and .Values.startdbScripts (not .Values.startdbScriptsSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-start-scripts" (include "common.names.fullname" .) }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +stringData: +{{- include "common.tplvalues.render" (dict "value" .Values.startdbScripts "context" .) | nindent 2 }} +{{- end }} diff --git a/charts/clickhouse/templates/statefulset.yaml b/charts/clickhouse/templates/statefulset.yaml new file mode 100644 index 000000000..9357d4287 --- /dev/null +++ b/charts/clickhouse/templates/statefulset.yaml @@ -0,0 +1,462 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- $shards := .Values.shards | int }} +{{- range $i, $e := until $shards }} +apiVersion: {{ include "common.capabilities.statefulset.apiVersion" $ }} +kind: StatefulSet +metadata: + name: {{ printf "%s-shard%d" (include "common.names.fullname" $ ) $i }} + namespace: {{ include "common.names.namespace" $ | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $.Values.commonLabels "context" $ ) | nindent 4 }} + app.kubernetes.io/component: clickhouse + {{- if $.Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $.Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + replicas: {{ $.Values.replicaCount }} + podManagementPolicy: {{ $.Values.podManagementPolicy | quote }} + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list $.Values.podLabels $.Values.commonLabels ) "context" $ ) }} + selector: + matchLabels: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 6 }} + app.kubernetes.io/component: clickhouse + serviceName: {{ printf "%s-headless" (include "common.names.fullname" $) }} + {{- if $.Values.updateStrategy }} + updateStrategy: {{- toYaml $.Values.updateStrategy | nindent 4 }} + {{- end }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") $ | sha256sum }} + checksum/config-extra: {{ include (print $.Template.BasePath "/configmap-extra.yaml") $ | sha256sum }} + checksum/config-users-extra: {{ include (print $.Template.BasePath "/configmap-users-extra.yaml") $ | sha256sum }} + {{- if $.Values.podAnnotations }} + {{- include "common.tplvalues.render" (dict "value" $.Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + {{- if and $.Values.metrics.enabled $.Values.metrics.podAnnotations }} + {{- include "common.tplvalues.render" (dict "value" $.Values.metrics.podAnnotations "context" $) | nindent 8 }} + {{- end }} + labels: {{- include "common.labels.standard" ( dict "customLabels" $podLabels "context" $ ) | nindent 8 }} + app.kubernetes.io/component: clickhouse + shard: {{ $i | quote }} + spec: + serviceAccountName: {{ template "clickhouse.serviceAccountName" $ }} + {{- include "clickhouse.imagePullSecrets" $ | nindent 6 }} + automountServiceAccountToken: {{ $.Values.automountServiceAccountToken }} + {{- if $.Values.hostAliases }} + hostAliases: {{- include "common.tplvalues.render" (dict "value" $.Values.hostAliases "context" $) | nindent 8 }} + {{- end }} + {{- if $.Values.affinity }} + affinity: {{- include "common.tplvalues.render" ( dict "value" $.Values.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" $.Values.podAffinityPreset "component" "clickhouse" "customLabels" $podLabels "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" $.Values.podAntiAffinityPreset "component" "clickhouse" "customLabels" $podLabels "extraPodAffinityTerms" (ternary (list (dict "extraMatchLabels" (dict "shard" $i) "topologyKey" "topology.kubernetes.io/zone")) (list) $.Values.distributeReplicasByZone) "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" $.Values.nodeAffinityPreset.type "key" $.Values.nodeAffinityPreset.key "values" $.Values.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} + {{- if $.Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" ( dict "value" $.Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if $.Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" $.Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if $.Values.priorityClassName }} + priorityClassName: {{ $.Values.priorityClassName | quote }} + {{- end }} + {{- if $.Values.schedulerName }} + schedulerName: {{ $.Values.schedulerName | quote }} + {{- end }} + {{- if $.Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" $.Values.topologySpreadConstraints "context" $) | nindent 8 }} + {{- end }} + {{- if $.Values.podSecurityContext.enabled }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" $.Values.podSecurityContext "context" $) | nindent 8 }} + {{- end }} + {{- if $.Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ $.Values.terminationGracePeriodSeconds }} + {{- end }} + initContainers: + {{- if and $.Values.tls.enabled (not $.Values.volumePermissions.enabled) }} + - name: copy-certs + image: {{ include "clickhouse.volumePermissions.image" $ }} + imagePullPolicy: {{ $.Values.volumePermissions.image.pullPolicy | quote }} + {{- if $.Values.resources }} + resources: {{- toYaml $.Values.resources | nindent 12 }} + {{- else if ne $.Values.resourcesPreset "none" }} + resources: {{- include "common.resources.preset" (dict "type" $.Values.resourcesPreset) | nindent 12 }} + {{- end }} + {{- if $.Values.containerSecurityContext.enabled }} + # We don't require a privileged container in this case + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" $.Values.containerSecurityContext "context" $) | nindent 12 }} + {{- end }} + command: + - /bin/sh + - -ec + - | + cp -L /tmp/certs/* /opt/bitnami/clickhouse/certs/ + chmod 600 {{ include "clickhouse.tlsCertKey" $ }} + volumeMounts: + - name: raw-certificates + mountPath: /tmp/certs + - name: clickhouse-certificates + mountPath: /opt/bitnami/clickhouse/certs + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + {{- else if and $.Values.volumePermissions.enabled $.Values.persistence.enabled }} + - name: volume-permissions + image: {{ include "clickhouse.volumePermissions.image" $ }} + imagePullPolicy: {{ $.Values.volumePermissions.image.pullPolicy | quote }} + command: + - /bin/sh + - -ec + - | + mkdir -p /bitnami/clickhouse/data + chmod 700 /bitnami/clickhouse/data + {{- if $.Values.keeper.enabled }} + mkdir -p /bitnami/clickhouse/keeper + chmod 700 /bitnami/clickhouse/keeper + {{- end }} + chown {{ $.Values.containerSecurityContext.runAsUser }}:{{ $.Values.podSecurityContext.fsGroup }} /bitnami/clickhouse + find /bitnami/clickhouse -mindepth 1 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" | \ + xargs -r chown -R {{ $.Values.containerSecurityContext.runAsUser }}:{{ $.Values.podSecurityContext.fsGroup }} + {{- if $.Values.tls.enabled }} + cp /tmp/certs/* /opt/bitnami/clickhouse/certs/ + {{- if eq ( toString ( $.Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} + chown -R `id -u`:`id -G | cut -d " " -f2` /opt/bitnami/clickhouse/certs/ + {{- else }} + chown -R {{ $.Values.containerSecurityContext.runAsUser }}:{{ $.Values.podSecurityContext.fsGroup }} /opt/bitnami/clickhouse/certs/ + {{- end }} + chmod 600 {{ include "clickhouse.tlsCertKey" $ }} + {{- end }} + securityContext: {{- include "common.tplvalues.render" (dict "value" $.Values.volumePermissions.containerSecurityContext "context" $) | nindent 12 }} + {{- if $.Values.volumePermissions.resources }} + resources: {{- toYaml $.Values.volumePermissions.resources | nindent 12 }} + {{- else if ne $.Values.volumePermissions.resourcesPreset "none" }} + resources: {{- include "common.resources.preset" (dict "type" $.Values.volumePermissions.resourcesPreset) | nindent 12 }} + {{- end }} + volumeMounts: + - name: data + mountPath: /bitnami/clickhouse + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + {{- if $.Values.tls.enabled }} + - name: raw-certificates + mountPath: /tmp/certs + - name: clickhouse-certificates + mountPath: /opt/bitnami/clickhouse/certs + {{- end }} + {{- end }} + {{- if $.Values.initContainers }} + {{- include "common.tplvalues.render" (dict "value" $.Values.initContainers "context" $) | nindent 8 }} + {{- end }} + containers: + - name: clickhouse + image: {{ template "clickhouse.image" $ }} + imagePullPolicy: {{ $.Values.image.pullPolicy }} + {{- if $.Values.containerSecurityContext.enabled }} + securityContext: {{- include "common.compatibility.renderSecurityContext" (dict "secContext" $.Values.containerSecurityContext "context" $) | nindent 12 }} + {{- end }} + {{- if $.Values.diagnosticMode.enabled }} + command: {{- include "common.tplvalues.render" (dict "value" $.Values.diagnosticMode.command "context" $) | nindent 12 }} + {{- else if $.Values.command }} + command: {{- include "common.tplvalues.render" (dict "value" $.Values.command "context" $) | nindent 12 }} + {{- end }} + {{- if $.Values.diagnosticMode.enabled }} + args: {{- include "common.tplvalues.render" (dict "value" $.Values.diagnosticMode.args "context" $) | nindent 12 }} + {{- else if $.Values.args }} + args: {{- include "common.tplvalues.render" (dict "value" $.Values.args "context" $) | nindent 12 }} + {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" (or $.Values.image.debug $.Values.diagnosticMode.enabled) | quote }} + - name: CLICKHOUSE_HTTP_PORT + value: {{ $.Values.containerPorts.http | quote }} + - name: CLICKHOUSE_TCP_PORT + value: {{ $.Values.containerPorts.tcp | quote }} + - name: CLICKHOUSE_MYSQL_PORT + value: {{ $.Values.containerPorts.mysql | quote }} + - name: CLICKHOUSE_POSTGRESQL_PORT + value: {{ $.Values.containerPorts.postgresql | quote }} + - name: CLICKHOUSE_INTERSERVER_HTTP_PORT + value: {{ $.Values.containerPorts.interserver | quote }} + {{- if $.Values.tls.enabled }} + - name: CLICKHOUSE_TCP_SECURE_PORT + value: {{ $.Values.containerPorts.tcpSecure | quote }} + - name: CLICKHOUSE_HTTPS_PORT + value: {{ $.Values.containerPorts.https | quote }} + {{- end }} + {{- if $.Values.keeper.enabled }} + - name: CLICKHOUSE_KEEPER_PORT + value: {{ $.Values.containerPorts.keeper | quote }} + - name: CLICKHOUSE_KEEPER_INTER_PORT + value: {{ $.Values.containerPorts.keeperInter | quote }} + {{- if $.Values.tls.enabled }} + - name: CLICKHOUSE_KEEPER_SECURE_PORT + value: {{ $.Values.containerPorts.keeperSecure | quote }} + {{- end }} + {{- end }} + {{- if $.Values.metrics.enabled }} + - name: CLICKHOUSE_METRICS_PORT + value: {{ $.Values.containerPorts.metrics | quote }} + {{- end }} + - name: CLICKHOUSE_ADMIN_USER + value: {{ $.Values.auth.username | quote }} + - name: CLICKHOUSE_SHARD_ID + value: {{ printf "shard%d" $i | quote }} + - name: CLICKHOUSE_REPLICA_ID + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CLICKHOUSE_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "clickhouse.secretName" $ }} + key: {{ include "clickhouse.secretKey" $ }} + {{- if $.Values.tls.enabled }} + - name: CLICKHOUSE_TLS_CERT_FILE + value: {{ include "clickhouse.tlsCert" $ | quote}} + - name: CLICKHOUSE_TLS_KEY_FILE + value: {{ include "clickhouse.tlsCertKey" $ | quote }} + - name: CLICKHOUSE_TLS_CA_FILE + value: {{ include "clickhouse.tlsCACert" $ | quote }} + {{- end }} + {{- if $.Values.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" $.Values.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + {{- if $.Values.keeper.enabled }} + {{- $replicas := $.Values.replicaCount | int }} + {{- range $j, $r := until $replicas }} + - name: {{ printf "KEEPER_NODE_%d" $j }} + value: {{ printf "%s-shard%d-%d.%s.%s.svc.%s" (include "common.names.fullname" $ ) $i $j (include "clickhouse.headlessServiceName" $) (include "common.names.namespace" $) $.Values.clusterDomain }} + {{- end }} + {{- else if $.Values.zookeeper.enabled }} + {{- $replicas := $.Values.zookeeper.replicaCount | int }} + {{- range $j, $r := until $replicas }} + - name: {{ printf "KEEPER_NODE_%d" $j }} + value: {{ printf "%s-%d.%s.%s.svc.%s" (include "clickhouse.zookeeper.fullname" $ ) $j (include "clickhouse.zookeeper.headlessServiceName" $) (include "common.names.namespace" $) $.Values.clusterDomain }} + {{- end }} + {{- end }} + envFrom: + {{- if $.Values.extraEnvVarsCM }} + - configMapRef: + name: {{ include "common.tplvalues.render" (dict "value" $.Values.extraEnvVarsCM "context" $) }} + {{- end }} + {{- if $.Values.extraEnvVarsSecret }} + - secretRef: + name: {{ include "common.tplvalues.render" (dict "value" $.Values.extraEnvVarsSecret "context" $) }} + {{- end }} + {{- if $.Values.resources }} + resources: {{- toYaml $.Values.resources | nindent 12 }} + {{- else if ne $.Values.resourcesPreset "none" }} + resources: {{- include "common.resources.preset" (dict "type" $.Values.resourcesPreset) | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ $.Values.containerPorts.http }} + - name: tcp + containerPort: {{ $.Values.containerPorts.tcp }} + {{- if $.Values.tls.enabled }} + - name: https + containerPort: {{ $.Values.containerPorts.https }} + - name: tcp-secure + containerPort: {{ $.Values.containerPorts.tcpSecure }} + {{- end }} + {{- if $.Values.keeper.enabled }} + - name: tcp-keeper + containerPort: {{ $.Values.containerPorts.keeper }} + - name: tcp-keeperinter + containerPort: {{ $.Values.containerPorts.keeperInter }} + {{- if $.Values.tls.enabled }} + - name: tcp-keepertls + containerPort: {{ $.Values.containerPorts.keeperSecure }} + {{- end }} + {{- end }} + - name: tcp-postgresql + containerPort: {{ $.Values.containerPorts.postgresql }} + - name: tcp-mysql + containerPort: {{ $.Values.containerPorts.mysql }} + - name: http-intersrv + containerPort: {{ $.Values.containerPorts.interserver }} + {{- if $.Values.metrics.enabled }} + - name: http-metrics + containerPort: {{ $.Values.containerPorts.metrics }} + {{- end }} + {{- if not $.Values.diagnosticMode.enabled }} + {{- if $.Values.customLivenessProbe }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" $.Values.customLivenessProbe "context" $) | nindent 12 }} + {{- else if $.Values.livenessProbe.enabled }} + livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit $.Values.livenessProbe "enabled") "context" $) | nindent 12 }} + httpGet: + path: /ping + port: http + {{- end }} + {{- if $.Values.customReadinessProbe }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" $.Values.customReadinessProbe "context" $) | nindent 12 }} + {{- else if $.Values.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit $.Values.readinessProbe "enabled") "context" $) | nindent 12 }} + httpGet: + path: /ping + port: http + {{- end }} + {{- if $.Values.customStartupProbe }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" $.Values.customStartupProbe "context" $) | nindent 12 }} + {{- else if $.Values.startupProbe.enabled }} + startupProbe: {{- include "common.tplvalues.render" (dict "value" (omit $.Values.startupProbe "enabled") "context" $) | nindent 12 }} + httpGet: + path: /ping + port: http + {{- end }} + {{- end }} + {{- if $.Values.lifecycleHooks }} + lifecycle: {{- include "common.tplvalues.render" (dict "value" $.Values.lifecycleHooks "context" $) | nindent 12 }} + {{- end }} + volumeMounts: + - name: empty-dir + mountPath: /opt/bitnami/clickhouse/etc + subPath: app-conf-dir + - name: empty-dir + mountPath: /opt/bitnami/clickhouse/logs + subPath: app-logs-dir + - name: empty-dir + mountPath: /opt/bitnami/clickhouse/tmp + subPath: app-tmp-dir + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + - name: scripts + mountPath: /scripts/setup.sh + subPath: setup.sh + - name: data + mountPath: /bitnami/clickhouse + - name: config + mountPath: /bitnami/clickhouse/etc/conf.d/default + {{- if or $.Values.extraOverridesConfigmap $.Values.extraOverrides }} + - name: extra-config + mountPath: /bitnami/clickhouse/etc/conf.d/extra-configmap + {{- end }} + {{- if or $.Values.usersExtraOverridesConfigmap $.Values.usersExtraOverrides }} + - name: users-extra-config + mountPath: /bitnami/clickhouse/etc/users.d/users-extra-configmap + {{- end }} + {{- if $.Values.extraOverridesSecret }} + - name: extra-secret + mountPath: /bitnami/clickhouse/etc/conf.d/extra-secret + {{- end }} + {{- if $.Values.usersExtraOverridesSecret }} + - name: users-extra-secret + mountPath: /bitnami/clickhouse/etc/users.d/users-extra-secret + {{- end }} + {{- if $.Values.tls.enabled }} + - name: clickhouse-certificates + mountPath: /bitnami/clickhouse/certs + {{- end }} + {{- if or $.Values.initdbScriptsSecret $.Values.initdbScripts }} + - name: custom-init-scripts + mountPath: /docker-entrypoint-initdb.d + {{- end }} + {{- if or $.Values.startdbScriptsSecret $.Values.startdbScripts }} + - name: custom-start-scripts + mountPath: /docker-entrypoint-startdb.d + {{- end }} + {{- if $.Values.extraVolumeMounts }} + {{- include "common.tplvalues.render" (dict "value" $.Values.extraVolumeMounts "context" $) | nindent 12 }} + {{- end }} + {{- if $.Values.sidecars }} + {{- include "common.tplvalues.render" ( dict "value" $.Values.sidecars "context" $) | nindent 8 }} + {{- end }} + volumes: + - name: scripts + configMap: + name: {{ printf "%s-scripts" (include "common.names.fullname" $) }} + defaultMode: 0755 + - name: empty-dir + emptyDir: {} + - name: config + configMap: + name: {{ template "clickhouse.configmapName" $ }} + {{- if or $.Values.initdbScriptsSecret $.Values.initdbScripts }} + - name: custom-init-scripts + secret: + secretName: {{ include "clickhouse.initdbScriptsSecret" $ }} + {{- end }} + {{- if or $.Values.startdbScriptsSecret $.Values.startdbScripts }} + - name: custom-start-scripts + secret: + secretName: {{ include "clickhouse.startdbScriptsSecret" $ }} + {{- end }} + {{- if or $.Values.extraOverridesConfigmap $.Values.extraOverrides }} + - name: extra-config + configMap: + name: {{ template "clickhouse.extraConfigmapName" $ }} + {{- end }} + {{- if or $.Values.usersExtraOverridesConfigmap $.Values.usersExtraOverrides }} + - name: users-extra-config + configMap: + name: {{ template "clickhouse.usersExtraConfigmapName" $ }} + {{- end }} + {{- if $.Values.extraOverridesSecret }} + - name: extra-secret + secret: + secretName: {{ $.Values.extraOverridesSecret }} + {{- end }} + {{- if $.Values.usersExtraOverridesSecret }} + - name: users-extra-secret + secret: + secretName: {{ $.Values.usersExtraOverridesSecret }} + {{- end }} + {{- if not $.Values.persistence.enabled }} + - name: data + emptyDir: {} + {{- else if $.Values.persistence.existingClaim }} + - name: data + persistentVolumeClaim: + claimName: {{ tpl $.Values.persistence.existingClaim $ }} + {{- end }} + {{- if $.Values.tls.enabled }} + - name: raw-certificates + secret: + secretName: {{ include "clickhouse.tlsSecretName" $ }} + - name: clickhouse-certificates + emptyDir: {} + {{- end }} + {{- if $.Values.extraVolumes }} + {{- include "common.tplvalues.render" (dict "value" $.Values.extraVolumes "context" $) | nindent 8 }} + {{- end }} + {{- if or $.Values.extraVolumeClaimTemplates (and $.Values.persistence.enabled (not $.Values.persistence.existingClaim)) }} + volumeClaimTemplates: + {{- if and $.Values.persistence.enabled (not $.Values.persistence.existingClaim) }} + - metadata: + name: data + {{- if or $.Values.persistence.annotations $.Values.commonAnnotations }} + {{- $claimAnnotations := include "common.tplvalues.merge" ( dict "values" ( list $.Values.persistence.annotations $.Values.commonLabels ) "context" $ ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $claimAnnotations "context" $ ) | nindent 10 }} + {{- end }} + {{- $claimLabels := include "common.tplvalues.merge" ( dict "values" ( list $.Values.persistence.labels $.Values.commonLabels ) "context" $ ) }} + labels: {{- include "common.labels.matchLabels" ( dict "customLabels" $claimLabels "context" $ ) | nindent 10 }} + app.kubernetes.io/component: clickhouse + spec: + accessModes: + {{- range $.Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ $.Values.persistence.size | quote }} + {{- if $.Values.persistence.selector }} + selector: {{- include "common.tplvalues.render" (dict "value" $.Values.persistence.selector "context" $) | nindent 10 }} + {{- end }} + {{- if $.Values.persistence.dataSource }} + dataSource: {{- include "common.tplvalues.render" (dict "value" $.Values.persistence.dataSource "context" $) | nindent 10 }} + {{- end }} + {{- include "common.storage.class" (dict "persistence" $.Values.persistence "global" $.Values.global) | nindent 8 }} + {{- end }} + {{- if $.Values.extraVolumeClaimTemplates }} + {{- include "common.tplvalues.render" ( dict "value" $.Values.extraVolumeClaimTemplates "context" $) | nindent 4 }} + {{- end }} + {{- end }} +--- +{{- end }} diff --git a/charts/clickhouse/templates/tls-secret.yaml b/charts/clickhouse/templates/tls-secret.yaml new file mode 100644 index 000000000..04b188e14 --- /dev/null +++ b/charts/clickhouse/templates/tls-secret.yaml @@ -0,0 +1,29 @@ +{{- /* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{- if (include "clickhouse.createTlsSecret" . ) }} +{{- $secretName := printf "%s-crt" (include "common.names.fullname" .) }} +{{- $ca := genCA "clickhouse-ca" 365 }} +{{- $fullname := include "common.names.fullname" . }} +{{- $releaseNamespace := .Release.Namespace }} +{{- $clusterDomain := .Values.clusterDomain }} +{{- $primaryHeadlessServiceName := printf "%s-headless" (include "common.names.fullname" .)}} +{{- $altNames := list (printf "*.%s.%s.svc.%s" $fullname $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $fullname $releaseNamespace $clusterDomain) (printf "*.%s.%s.svc.%s" $primaryHeadlessServiceName $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $primaryHeadlessServiceName $releaseNamespace $clusterDomain) $fullname }} +{{- $cert := genSignedCert $fullname nil $altNames 365 $ca }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: kubernetes.io/tls +data: + tls.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.crt" "defaultValue" $cert.Cert "context" $) }} + tls.key: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.key" "defaultValue" $cert.Key "context" $) }} + ca.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "ca.crt" "defaultValue" $ca.Cert "context" $) }} +{{- end }} diff --git a/charts/clickhouse/values.yaml b/charts/clickhouse/values.yaml new file mode 100644 index 000000000..8d7de64fe --- /dev/null +++ b/charts/clickhouse/values.yaml @@ -0,0 +1,1200 @@ +# Copyright VMware, Inc. +# SPDX-License-Identifier: APACHE-2.0 + +## @section Global parameters +## Global Docker image parameters +## Please, note that this will override the image parameters, including dependencies, configured to use the global value +## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass +## + +## @param global.imageRegistry Global Docker image registry +## @param global.imagePullSecrets Global Docker registry secret names as an array +## @param global.storageClass Global StorageClass for Persistent Volume(s) +## +global: + imageRegistry: "" + ## E.g. + ## imagePullSecrets: + ## - myRegistryKeySecretName + ## + imagePullSecrets: [] + storageClass: "" + ## Compatibility adaptations for Kubernetes platforms + ## + compatibility: + ## Compatibility adaptations for Openshift + ## + openshift: + ## @param global.compatibility.openshift.adaptSecurityContext Adapt the securityContext sections of the deployment to make them compatible with Openshift restricted-v2 SCC: remove runAsUser, runAsGroup and fsGroup and let the platform use their allowed default IDs. Possible values: auto (apply if the detected running cluster is Openshift), force (perform the adaptation always), disabled (do not perform adaptation) + ## + adaptSecurityContext: disabled +## @section Common parameters +## + +## @param kubeVersion Override Kubernetes version +## +kubeVersion: "" +## @param nameOverride String to partially override common.names.name +## +nameOverride: "" +## @param fullnameOverride String to fully override common.names.fullname +## +fullnameOverride: "" +## @param namespaceOverride String to fully override common.names.namespace +## +namespaceOverride: "" +## @param commonLabels Labels to add to all deployed objects +## +commonLabels: {} +## @param commonAnnotations Annotations to add to all deployed objects +## +commonAnnotations: {} +## @param clusterDomain Kubernetes cluster domain name +## +clusterDomain: cluster.local +## @param extraDeploy Array of extra objects to deploy with the release +## +extraDeploy: [] +## Enable diagnostic mode in the deployment +## +diagnosticMode: + ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden) + ## + enabled: false + ## @param diagnosticMode.command Command to override all containers in the deployment + ## + command: + - sleep + ## @param diagnosticMode.args Args to override all containers in the deployment + ## + args: + - infinity +## @section ClickHouse Parameters +## + +## Bitnami ClickHouse image +## ref: https://hub.docker.com/r/bitnami/clickhouse/tags/ +## @param image.registry [default: REGISTRY_NAME] ClickHouse image registry +## @param image.repository [default: REPOSITORY_NAME/clickhouse] ClickHouse image repository +## @skip image.tag ClickHouse image tag (immutable tags are recommended) +## @param image.digest ClickHouse image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag +## @param image.pullPolicy ClickHouse image pull policy +## @param image.pullSecrets ClickHouse image pull secrets +## @param image.debug Enable ClickHouse image debug mode +## +image: + registry: docker.io + repository: bitnami/clickhouse + tag: 24.2.2-debian-12-r0 + digest: "" + ## Specify a imagePullPolicy + ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + ## ref: http://kubernetes.io/docs/concepts/containers/images/#pre-pulled-images + ## + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## e.g: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Enable debug mode + ## + debug: false +## @param shards Number of ClickHouse shards to deploy +## +shards: 2 +## @param replicaCount Number of ClickHouse replicas per shard to deploy +## if keeper enable, same as keeper count, keeper cluster by shards. +## +replicaCount: 3 +## @param distributeReplicasByZone Schedules replicas of the same shard to different availability zones +## +distributeReplicasByZone: false +## @param containerPorts.http ClickHouse HTTP container port +## @param containerPorts.https ClickHouse HTTPS container port +## @param containerPorts.tcp ClickHouse TCP container port +## @param containerPorts.tcpSecure ClickHouse TCP (secure) container port +## @param containerPorts.keeper ClickHouse keeper TCP container port +## @param containerPorts.keeperSecure ClickHouse keeper TCP (secure) container port +## @param containerPorts.keeperInter ClickHouse keeper interserver TCP container port +## @param containerPorts.mysql ClickHouse MySQL container port +## @param containerPorts.postgresql ClickHouse PostgreSQL container port +## @param containerPorts.interserver ClickHouse Interserver container port +## @param containerPorts.metrics ClickHouse metrics container port +## +containerPorts: + http: 8123 + https: 8443 + tcp: 9000 + tcpSecure: 9440 + keeper: 2181 + keeperSecure: 3181 + keeperInter: 9444 + mysql: 9004 + postgresql: 9005 + interserver: 9009 + metrics: 8001 +## Configure extra options for ClickHouse containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## @param livenessProbe.enabled Enable livenessProbe on ClickHouse containers +## @param livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## @param livenessProbe.periodSeconds Period seconds for livenessProbe +## @param livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## @param livenessProbe.failureThreshold Failure threshold for livenessProbe +## @param livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 +## @param readinessProbe.enabled Enable readinessProbe on ClickHouse containers +## @param readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## @param readinessProbe.periodSeconds Period seconds for readinessProbe +## @param readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## @param readinessProbe.failureThreshold Failure threshold for readinessProbe +## @param readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 +## @param startupProbe.enabled Enable startupProbe on ClickHouse containers +## @param startupProbe.initialDelaySeconds Initial delay seconds for startupProbe +## @param startupProbe.periodSeconds Period seconds for startupProbe +## @param startupProbe.timeoutSeconds Timeout seconds for startupProbe +## @param startupProbe.failureThreshold Failure threshold for startupProbe +## @param startupProbe.successThreshold Success threshold for startupProbe +## +startupProbe: + enabled: false + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 +## @param customLivenessProbe Custom livenessProbe that overrides the default one +## +customLivenessProbe: {} +## @param customReadinessProbe Custom readinessProbe that overrides the default one +## +customReadinessProbe: {} +## @param customStartupProbe Custom startupProbe that overrides the default one +## +customStartupProbe: {} +## ClickHouse resource requests and limits +## ref: http://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ +## @param resourcesPreset Set container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if resources is set (resources is recommended for production). +## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 +## +resourcesPreset: "none" +## @param resources Set container requests and limits for different resources like CPU or memory (essential for production workloads) +## Example: +## resources: +## requests: +## cpu: 2 +## memory: 512Mi +## limits: +## cpu: 3 +## memory: 1024Mi +## +resources: {} +## Configure Pods Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +## @param podSecurityContext.enabled Enabled ClickHouse pods' Security Context +## @param podSecurityContext.fsGroupChangePolicy Set filesystem group change policy +## @param podSecurityContext.sysctls Set kernel settings using the sysctl interface +## @param podSecurityContext.supplementalGroups Set filesystem extra groups +## @param podSecurityContext.fsGroup Set ClickHouse pod's Security Context fsGroup +## If you are using Kubernetes 1.18, the following code needs to be commented out. +## +podSecurityContext: + enabled: true + fsGroupChangePolicy: Always + sysctls: [] + supplementalGroups: [] + fsGroup: 1001 +## Configure Container Security Context +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +## @param containerSecurityContext.enabled Enable containers' Security Context +## @param containerSecurityContext.seLinuxOptions [object,nullable] Set SELinux options in container +## @param containerSecurityContext.runAsUser Set containers' Security Context runAsUser +## @param containerSecurityContext.runAsGroup Set containers' Security Context runAsGroup +## @param containerSecurityContext.runAsNonRoot Set containers' Security Context runAsNonRoot +## @param containerSecurityContext.readOnlyRootFilesystem Set read only root file system pod's +## @param containerSecurityContext.privileged Set contraller container's Security Context privileged +## @param containerSecurityContext.allowPrivilegeEscalation Set contraller container's Security Context allowPrivilegeEscalation +## @param containerSecurityContext.capabilities.drop List of capabilities to be droppedn +## @param containerSecurityContext.seccompProfile.type Set container's Security Context seccomp profile +## +containerSecurityContext: + enabled: true + seLinuxOptions: null + runAsUser: 1001 + runAsGroup: 0 + runAsNonRoot: true + privileged: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: "RuntimeDefault" +## Authentication +## @param auth.username ClickHouse Admin username +## @param auth.password ClickHouse Admin password +## @param auth.existingSecret Name of a secret containing the Admin password +## @param auth.existingSecretKey Name of the key inside the existing secret +## +auth: + username: default + password: "" + existingSecret: "" + existingSecretKey: "" +## @param logLevel Logging level +## +logLevel: information +## @section ClickHouse keeper configuration parameters +## @param keeper.enabled Deploy ClickHouse keeper. Support is experimental. +## +keeper: + enabled: false +## @param defaultConfigurationOverrides [string] Default configuration overrides (evaluated as a template) +## +defaultConfigurationOverrides: | + + + + + + {{ include "common.names.fullname" . }} + + + + {{ .Values.logLevel }} + + {{- if or (ne (int .Values.shards) 1) (ne (int .Values.replicaCount) 1)}} + + + + {{- $shards := $.Values.shards | int }} + {{- range $shard, $e := until $shards }} + + {{- $replicas := $.Values.replicaCount | int }} + {{- range $i, $_e := until $replicas }} + + {{ printf "%s-shard%d-%d.%s.%s.svc.%s" (include "common.names.fullname" $ ) $shard $i (include "clickhouse.headlessServiceName" $) (include "common.names.namespace" $) $.Values.clusterDomain }} + {{ $.Values.service.ports.tcp }} + + + + {{- end }} + + {{- end }} + + + {{- end }} + {{- if .Values.keeper.enabled }} + + + {{/*ClickHouse keeper configuration using the helm chart */}} + {{ $.Values.containerPorts.keeper }} + {{- if .Values.tls.enabled }} + {{ $.Values.containerPorts.keeperSecure }} + {{- end }} + + /bitnami/clickhouse/keeper/coordination/log + /bitnami/clickhouse/keeper/coordination/snapshots + + + 10000 + 30000 + trace + + + + {{- $nodes := .Values.replicaCount | int }} + {{- range $node, $e := until $nodes }} + + {{ $node | int }} + + {{ $.Values.service.ports.keeperInter }} + + {{- end }} + + + {{- end }} + {{- if or .Values.keeper.enabled .Values.zookeeper.enabled .Values.externalZookeeper.servers }} + + + {{- if or .Values.keeper.enabled }} + {{- $nodes := .Values.replicaCount | int }} + {{- range $node, $e := until $nodes }} + + + {{ $.Values.service.ports.keeper }} + + {{- end }} + {{- else if .Values.zookeeper.enabled }} + {{/* Zookeeper configuration using the helm chart */}} + {{- $nodes := .Values.zookeeper.replicaCount | int }} + {{- range $node, $e := until $nodes }} + + + {{ $.Values.zookeeper.service.ports.client }} + + {{- end }} + {{- else if .Values.externalZookeeper.servers }} + {{/* Zookeeper configuration using an external instance */}} + {{- range $node :=.Values.externalZookeeper.servers }} + + {{ $node }} + {{ $.Values.externalZookeeper.port }} + + {{- end }} + {{- end }} + + {{- end }} + {{- if .Values.tls.enabled }} + + + + + + {{- $certFileName := default "tls.crt" .Values.tls.certFilename }} + {{- $keyFileName := default "tls.key" .Values.tls.certKeyFilename }} + /bitnami/clickhouse/certs/{{$certFileName}} + /bitnami/clickhouse/certs/{{$keyFileName}} + none + true + sslv2,sslv3 + true + {{- if or .Values.tls.autoGenerated .Values.tls.certCAFilename }} + {{- $caFileName := default "ca.crt" .Values.tls.certCAFilename }} + /bitnami/clickhouse/certs/{{$caFileName}} + {{- else }} + true + {{- end }} + + + true + true + sslv2,sslv3 + true + none + + AcceptCertificateHandler + + + + {{- end }} + {{- if .Values.metrics.enabled }} + + + /metrics + + true + true + true + + {{- end }} + +## @param existingOverridesConfigmap The name of an existing ConfigMap with your custom configuration for ClickHouse +## +existingOverridesConfigmap: "" +## @param extraOverrides Extra configuration overrides (evaluated as a template) apart from the default +## +extraOverrides: "" +## @param extraOverridesConfigmap The name of an existing ConfigMap with extra configuration for ClickHouse +## +extraOverridesConfigmap: "" +## @param extraOverridesSecret The name of an existing ConfigMap with your custom configuration for ClickHouse +## +extraOverridesSecret: "" +## @param usersExtraOverrides Users extra configuration overrides (evaluated as a template) apart from the default +## +usersExtraOverrides: "" +## @param usersExtraOverridesConfigmap The name of an existing ConfigMap with users extra configuration for ClickHouse +## +usersExtraOverridesConfigmap: "" +## @param usersExtraOverridesSecret The name of an existing ConfigMap with your custom users configuration for ClickHouse +## +usersExtraOverridesSecret: "" +## @param initdbScripts Dictionary of initdb scripts +## Specify dictionary of scripts to be run at first boot +## Example: +## initdbScripts: +## my_init_script.sh: | +## #!/bin/bash +## echo "Do something." +## +initdbScripts: {} +## @param initdbScriptsSecret ConfigMap with the initdb scripts (Note: Overrides `initdbScripts`) +## +initdbScriptsSecret: "" +## @param startdbScripts Dictionary of startdb scripts +## Specify dictionary of scripts to be run on every start +## Example: +## startdbScripts: +## my_start_script.sh: | +## #!/bin/bash +## echo "Do something." +## +startdbScripts: {} +## @param startdbScriptsSecret ConfigMap with the startdb scripts (Note: Overrides `startdbScripts`) +## +startdbScriptsSecret: "" +## @param command Override default container command (useful when using custom images) +## +command: + - /scripts/setup.sh +## @param args Override default container args (useful when using custom images) +## +args: [] +## @param automountServiceAccountToken Mount Service Account token in pod +## +automountServiceAccountToken: false +## @param hostAliases ClickHouse pods host aliases +## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ +## +hostAliases: [] +## @param podLabels Extra labels for ClickHouse pods +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +## +podLabels: {} +## @param podAnnotations Annotations for ClickHouse pods +## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +## +podAnnotations: {} +## @param podAffinityPreset Pod affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard` +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity +## +podAffinityPreset: "" +## @param podAntiAffinityPreset Pod anti-affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard` +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity +## +podAntiAffinityPreset: soft +## Node affinity preset +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity +## +nodeAffinityPreset: + ## @param nodeAffinityPreset.type Node affinity preset type. Ignored if `affinity` is set. Allowed values: `soft` or `hard` + ## + type: "" + ## @param nodeAffinityPreset.key Node label key to match. Ignored if `affinity` is set + ## + key: "" + ## @param nodeAffinityPreset.values Node label values to match. Ignored if `affinity` is set + ## E.g. + ## values: + ## - e2e-az1 + ## - e2e-az2 + ## + values: [] +## @param affinity Affinity for ClickHouse pods assignment +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## NOTE: `podAffinityPreset`, `podAntiAffinityPreset`, and `nodeAffinityPreset` will be ignored when it's set +## +affinity: {} +## @param nodeSelector Node labels for ClickHouse pods assignment +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ +## +nodeSelector: {} +## @param tolerations Tolerations for ClickHouse pods assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] +## @param updateStrategy.type ClickHouse statefulset strategy type +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies +## +updateStrategy: + ## StrategyType + ## Can be set to RollingUpdate or OnDelete + ## + type: RollingUpdate +## @param podManagementPolicy Statefulset Pod management policy, it needs to be Parallel to be able to complete the cluster join +## Ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies +## +podManagementPolicy: Parallel +## @param priorityClassName ClickHouse pods' priorityClassName +## +priorityClassName: "" +## @param topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template +## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods +## +topologySpreadConstraints: [] +## @param schedulerName Name of the k8s scheduler (other than default) for ClickHouse pods +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +schedulerName: "" +## @param terminationGracePeriodSeconds Seconds Redmine pod needs to terminate gracefully +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods +## +terminationGracePeriodSeconds: "" +## @param lifecycleHooks for the ClickHouse container(s) to automate configuration before or after startup +## +lifecycleHooks: {} +## @param extraEnvVars Array with extra environment variables to add to ClickHouse nodes +## e.g: +## extraEnvVars: +## - name: FOO +## value: "bar" +## +extraEnvVars: [] +## @param extraEnvVarsCM Name of existing ConfigMap containing extra env vars for ClickHouse nodes +## +extraEnvVarsCM: "" +## @param extraEnvVarsSecret Name of existing Secret containing extra env vars for ClickHouse nodes +## +extraEnvVarsSecret: "" +## @param extraVolumes Optionally specify extra list of additional volumes for the ClickHouse pod(s) +## +extraVolumes: [] +## @param extraVolumeMounts Optionally specify extra list of additional volumeMounts for the ClickHouse container(s) +## +extraVolumeMounts: [] +## @param extraVolumeClaimTemplates Optionally specify extra list of additional volumeClaimTemplates for the ClickHouse container(s) +## +extraVolumeClaimTemplates: [] +## @param sidecars Add additional sidecar containers to the ClickHouse pod(s) +## e.g: +## sidecars: +## - name: your-image-name +## image: your-image +## imagePullPolicy: Always +## ports: +## - name: portname +## containerPort: 1234 +## +sidecars: [] +## @param initContainers Add additional init containers to the ClickHouse pod(s) +## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +## e.g: +## initContainers: +## - name: your-image-name +## image: your-image +## imagePullPolicy: Always +## command: ['sh', '-c', 'echo "hello world"'] +## +initContainers: [] +## TLS configuration +## +tls: + ## @param tls.enabled Enable TLS traffic support + ## + enabled: false + ## @param tls.autoGenerated Generate automatically self-signed TLS certificates + ## + autoGenerated: false + ## @param tls.certificatesSecret Name of an existing secret that contains the certificates + ## + certificatesSecret: "" + ## @param tls.certFilename Certificate filename + ## + certFilename: "" + ## @param tls.certKeyFilename Certificate key filename + ## + certKeyFilename: "" + ## @param tls.certCAFilename CA Certificate filename + ## If provided, PostgreSQL will authenticate TLS/SSL clients by requesting them a certificate + ## ref: https://www.postgresql.org/docs/9.6/auth-methods.html + ## + certCAFilename: "" +## @section Traffic Exposure Parameters +## + +## ClickHouse service parameters +## +service: + ## @param service.type ClickHouse service type + ## + type: ClusterIP + ## @param service.ports.http ClickHouse service HTTP port + ## @param service.ports.https ClickHouse service HTTPS port + ## @param service.ports.tcp ClickHouse service TCP port + ## @param service.ports.tcpSecure ClickHouse service TCP (secure) port + ## @param service.ports.keeper ClickHouse keeper TCP container port + ## @param service.ports.keeperSecure ClickHouse keeper TCP (secure) container port + ## @param service.ports.keeperInter ClickHouse keeper interserver TCP container port + ## @param service.ports.mysql ClickHouse service MySQL port + ## @param service.ports.postgresql ClickHouse service PostgreSQL port + ## @param service.ports.interserver ClickHouse service Interserver port + ## @param service.ports.metrics ClickHouse service metrics port + ## + ports: + http: 8123 + https: 443 + tcp: 9000 + tcpSecure: 9440 + keeper: 2181 + keeperSecure: 3181 + keeperInter: 9444 + mysql: 9004 + postgresql: 9005 + interserver: 9009 + metrics: 8001 + ## Node ports to expose + ## @param service.nodePorts.http Node port for HTTP + ## @param service.nodePorts.https Node port for HTTPS + ## @param service.nodePorts.tcp Node port for TCP + ## @param service.nodePorts.tcpSecure Node port for TCP (with TLS) + ## @param service.nodePorts.keeper ClickHouse keeper TCP container port + ## @param service.nodePorts.keeperSecure ClickHouse keeper TCP (secure) container port + ## @param service.nodePorts.keeperInter ClickHouse keeper interserver TCP container port + ## @param service.nodePorts.mysql Node port for MySQL + ## @param service.nodePorts.postgresql Node port for PostgreSQL + ## @param service.nodePorts.interserver Node port for Interserver + ## @param service.nodePorts.metrics Node port for metrics + ## NOTE: choose port between <30000-32767> + ## + nodePorts: + http: "" + https: "" + tcp: "" + tcpSecure: "" + keeper: "" + keeperSecure: "" + keeperInter: "" + mysql: "" + postgresql: "" + interserver: "" + metrics: "" + ## @param service.clusterIP ClickHouse service Cluster IP + ## e.g.: + ## clusterIP: None + ## + clusterIP: "" + ## @param service.loadBalancerIP ClickHouse service Load Balancer IP + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer + ## + loadBalancerIP: "" + ## @param service.loadBalancerSourceRanges ClickHouse service Load Balancer sources + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## e.g: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## @param service.externalTrafficPolicy ClickHouse service external traffic policy + ## ref http://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + ## @param service.annotations Additional custom annotations for ClickHouse service + ## + annotations: {} + ## @param service.extraPorts Extra ports to expose in ClickHouse service (normally used with the `sidecars` value) + ## + extraPorts: [] + ## @param service.sessionAffinity Control where client requests go, to the same pod or round-robin + ## Values: ClientIP or None + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/ + ## + sessionAffinity: None + ## @param service.sessionAffinityConfig Additional settings for the sessionAffinity + ## sessionAffinityConfig: + ## clientIP: + ## timeoutSeconds: 300 + ## + sessionAffinityConfig: {} + ## Headless service properties + ## + headless: + ## @param service.headless.annotations Annotations for the headless service. + ## + annotations: {} +## External Access to ClickHouse configuration +## +externalAccess: + ## @param externalAccess.enabled Enable Kubernetes external cluster access to ClickHouse + ## + enabled: false + ## Parameters to configure K8s service(s) used to externally access ClickHouse + ## Note: A new service per will be created + ## + service: + ## @param externalAccess.service.type Kubernetes Service type for external access. It can be NodePort, LoadBalancer or ClusterIP + ## + type: LoadBalancer + ## @param externalAccess.service.ports.http ClickHouse service HTTP port + ## @param externalAccess.service.ports.https ClickHouse service HTTPS port + ## @param externalAccess.service.ports.tcp ClickHouse service TCP port + ## @param externalAccess.service.ports.tcpSecure ClickHouse service TCP (secure) port + ## @param externalAccess.service.ports.keeper ClickHouse keeper TCP container port + ## @param externalAccess.service.ports.keeperSecure ClickHouse keeper TCP (secure) container port + ## @param externalAccess.service.ports.keeperInter ClickHouse keeper interserver TCP container port + ## @param externalAccess.service.ports.mysql ClickHouse service MySQL port + ## @param externalAccess.service.ports.postgresql ClickHouse service PostgreSQL port + ## @param externalAccess.service.ports.interserver ClickHouse service Interserver port + ## @param externalAccess.service.ports.metrics ClickHouse service metrics port + ## + ports: + http: 80 + https: 443 + tcp: 9000 + tcpSecure: 9440 + keeper: 2181 + keeperSecure: 3181 + keeperInter: 9444 + mysql: 9004 + postgresql: 9005 + interserver: 9009 + metrics: 8001 + ## @param externalAccess.service.loadBalancerIPs Array of load balancer IPs for each ClickHouse . Length must be the same as replicaCount + ## e.g: + ## loadBalancerIPs: + ## - X.X.X.X + ## - Y.Y.Y.Y + ## + loadBalancerIPs: [] + ## @param externalAccess.service.loadBalancerAnnotations Array of load balancer annotations for each ClickHouse . Length must be the same as shards multiplied by replicaCount + ## e.g: + ## loadBalancerAnnotations: + ## - external-dns.alpha.kubernetes.io/hostname: 1.external.example.com. + ## - external-dns.alpha.kubernetes.io/hostname: 2.external.example.com. + ## + loadBalancerAnnotations: [] + ## @param externalAccess.service.loadBalancerSourceRanges Address(es) that are allowed when service is LoadBalancer + ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## e.g: + ## loadBalancerSourceRanges: + ## - 10.10.10.0/24 + ## + loadBalancerSourceRanges: [] + ## @param externalAccess.service.nodePorts.http Node port for HTTP + ## @param externalAccess.service.nodePorts.https Node port for HTTPS + ## @param externalAccess.service.nodePorts.tcp Node port for TCP + ## @param externalAccess.service.nodePorts.tcpSecure Node port for TCP (with TLS) + ## @param externalAccess.service.nodePorts.keeper ClickHouse keeper TCP container port + ## @param externalAccess.service.nodePorts.keeperSecure ClickHouse keeper TCP container port (with TLS) + ## @param externalAccess.service.nodePorts.keeperInter ClickHouse keeper interserver TCP container port + ## @param externalAccess.service.nodePorts.mysql Node port for MySQL + ## @param externalAccess.service.nodePorts.postgresql Node port for PostgreSQL + ## @param externalAccess.service.nodePorts.interserver Node port for Interserver + ## @param externalAccess.service.nodePorts.metrics Node port for metrics + ## NOTE: choose port between <30000-32767> + ## e.g: + ## nodePorts: + ## tls: + ## - 30001 + ## - 30002 + ## + nodePorts: + http: [] + https: [] + tcp: [] + tcpSecure: [] + keeper: [] + keeperSecure: [] + keeperInter: [] + mysql: [] + postgresql: [] + interserver: [] + metrics: [] + ## @param externalAccess.service.labels Service labels for external access + ## + labels: {} + ## @param externalAccess.service.annotations Service annotations for external access + ## + annotations: {} + ## @param externalAccess.service.extraPorts Extra ports to expose in the ClickHouse external service + ## + extraPorts: [] +## ClickHouse ingress parameters +## ref: http://kubernetes.io/docs/concepts/services-networking/ingress/ +## +ingress: + ## @param ingress.enabled Enable ingress record generation for ClickHouse + ## + enabled: false + ## @param ingress.pathType Ingress path type + ## + pathType: ImplementationSpecific + ## @param ingress.apiVersion Force Ingress API version (automatically detected if not set) + ## + apiVersion: "" + ## @param ingress.hostname Default host for the ingress record + ## + hostname: clickhouse.local + ## @param ingress.ingressClassName IngressClass that will be be used to implement the Ingress (Kubernetes 1.18+) + ## This is supported in Kubernetes 1.18+ and required if you have more than one IngressClass marked as the default for your cluster . + ## ref: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/ + ## + ingressClassName: "" + ## @param ingress.path Default path for the ingress record + ## NOTE: You may need to set this to '/*' in order to use this with ALB ingress controllers + ## + path: / + ## @param ingress.annotations Additional annotations for the Ingress resource. To enable certificate autogeneration, place here your cert-manager annotations. + ## Use this parameter to set the required annotations for cert-manager, see + ## ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations + ## e.g: + ## annotations: + ## kubernetes.io/ingress.class: nginx + ## cert-manager.io/cluster-issuer: cluster-issuer-name + ## + annotations: {} + ## @param ingress.tls Enable TLS configuration for the host defined at `ingress.hostname` parameter + ## TLS certificates will be retrieved from a TLS secret with name: `{{- printf "%s-tls" .Values.ingress.hostname }}` + ## You can: + ## - Use the `ingress.secrets` parameter to create this TLS secret + ## - Rely on cert-manager to create it by setting the corresponding annotations + ## - Rely on Helm to create self-signed certificates by setting `ingress.selfSigned=true` + ## + tls: false + ## @param ingress.selfSigned Create a TLS secret for this ingress record using self-signed certificates generated by Helm + ## + selfSigned: false + ## @param ingress.extraHosts An array with additional hostname(s) to be covered with the ingress record + ## e.g: + ## extraHosts: + ## - name: clickhouse.local + ## path: / + ## + extraHosts: [] + ## @param ingress.extraPaths An array with additional arbitrary paths that may need to be added to the ingress under the main host + ## e.g: + ## extraPaths: + ## - path: /* + ## backend: + ## serviceName: ssl-redirect + ## servicePort: use-annotation + ## + extraPaths: [] + ## @param ingress.extraTls TLS configuration for additional hostname(s) to be covered with this ingress record + ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls + ## e.g: + ## extraTls: + ## - hosts: + ## - clickhouse.local + ## secretName: clickhouse.local-tls + ## + extraTls: [] + ## @param ingress.secrets Custom TLS certificates as secrets + ## NOTE: 'key' and 'certificate' are expected in PEM format + ## NOTE: 'name' should line up with a 'secretName' set further up + ## If it is not set and you're using cert-manager, this is unneeded, as it will create a secret for you with valid certificates + ## If it is not set and you're NOT using cert-manager either, self-signed certificates will be created valid for 365 days + ## It is also possible to create and manage the certificates outside of this helm chart + ## Please see README.md for more information + ## e.g: + ## secrets: + ## - name: clickhouse.local-tls + ## key: |- + ## -----BEGIN RSA PRIVATE KEY----- + ## ... + ## -----END RSA PRIVATE KEY----- + ## certificate: |- + ## -----BEGIN CERTIFICATE----- + ## ... + ## -----END CERTIFICATE----- + ## + secrets: [] + ## @param ingress.extraRules Additional rules to be covered with this ingress record + ## ref: https://kubernetes.io/docs/concepts/services-networking/ingress/#ingress-rules + ## e.g: + ## extraRules: + ## - host: example.local + ## http: + ## path: / + ## backend: + ## service: + ## name: example-svc + ## port: + ## name: http + ## + extraRules: [] +## @section Persistence Parameters +## + +## Enable persistence using Persistent Volume Claims +## ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/ +## +persistence: + ## @param persistence.enabled Enable persistence using Persistent Volume Claims + ## + enabled: true + ## @param persistence.existingClaim Name of an existing PVC to use + ## + existingClaim: "" + ## @param persistence.storageClass Storage class of backing PVC + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + storageClass: "" + ## @param persistence.labels Persistent Volume Claim labels + ## + labels: {} + ## @param persistence.annotations Persistent Volume Claim annotations + ## + annotations: {} + ## @param persistence.accessModes Persistent Volume Access Modes + ## + accessModes: + - ReadWriteOnce + ## @param persistence.size Size of data volume + ## + size: 8Gi + ## @param persistence.selector Selector to match an existing Persistent Volume for ClickHouse data PVC + ## If set, the PVC can't have a PV dynamically provisioned for it + ## E.g. + ## selector: + ## matchLabels: + ## app: my-app + ## + selector: {} + ## @param persistence.dataSource Custom PVC data source + ## + dataSource: {} +## @section Init Container Parameters +## + +## 'volumePermissions' init container parameters +## Changes the owner and group of the persistent volume mount point to runAsUser:fsGroup values +## based on the *podSecurityContext/*containerSecurityContext parameters +## +volumePermissions: + ## @param volumePermissions.enabled Enable init container that changes the owner/group of the PV mount point to `runAsUser:fsGroup` + ## + enabled: false + ## OS Shell + Utility image + ## ref: https://hub.docker.com/r/bitnami/os-shell/tags/ + ## @param volumePermissions.image.registry [default: REGISTRY_NAME] OS Shell + Utility image registry + ## @param volumePermissions.image.repository [default: REPOSITORY_NAME/os-shell] OS Shell + Utility image repository + ## @skip volumePermissions.image.tag OS Shell + Utility image tag (immutable tags are recommended) + ## @param volumePermissions.image.pullPolicy OS Shell + Utility image pull policy + ## @param volumePermissions.image.pullSecrets OS Shell + Utility image pull secrets + ## + image: + registry: docker.io + repository: bitnami/os-shell + tag: 12-debian-12-r16 + pullPolicy: IfNotPresent + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## e.g: + ## pullSecrets: + ## - myRegistryKeySecretName + ## + pullSecrets: [] + ## Init container's resource requests and limits + ## ref: http://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + ## @param volumePermissions.resourcesPreset Set container resources according to one common preset (allowed values: none, nano, small, medium, large, xlarge, 2xlarge). This is ignored if volumePermissions.resources is set (volumePermissions.resources is recommended for production). + ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 + ## + resourcesPreset: "none" + ## @param volumePermissions.resources Set container requests and limits for different resources like CPU or memory (essential for production workloads) + ## Example: + ## resources: + ## requests: + ## cpu: 2 + ## memory: 512Mi + ## limits: + ## cpu: 3 + ## memory: 1024Mi + ## + resources: {} + ## Init container Container Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + ## @param volumePermissions.containerSecurityContext.seLinuxOptions [object,nullable] Set SELinux options in container + ## @param volumePermissions.containerSecurityContext.runAsUser Set init container's Security Context runAsUser + ## NOTE: when runAsUser is set to special value "auto", init container will try to chown the + ## data folder to auto-determined user&group, using commands: `id -u`:`id -G | cut -d" " -f2` + ## "auto" is especially useful for OpenShift which has scc with dynamic user ids (and 0 is not allowed) + ## + containerSecurityContext: + seLinuxOptions: null + runAsUser: 0 +## @section Other Parameters +## + +## ServiceAccount configuration +## +serviceAccount: + ## @param serviceAccount.create Specifies whether a ServiceAccount should be created + ## + create: true + ## @param serviceAccount.name The name of the ServiceAccount to use. + ## If not set and create is true, a name is generated using the common.names.fullname template + ## + name: "" + ## @param serviceAccount.annotations Additional Service Account annotations (evaluated as a template) + ## + annotations: {} + ## @param serviceAccount.automountServiceAccountToken Automount service account token for the server service account + ## + automountServiceAccountToken: false +## Prometheus metrics +## +metrics: + ## @param metrics.enabled Enable the export of Prometheus metrics + ## + enabled: false + ## @param metrics.podAnnotations [object] Annotations for metrics scraping + ## + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "{{ .Values.containerPorts.metrics }}" + ## Prometheus Operator ServiceMonitor configuration + ## + serviceMonitor: + ## @param metrics.serviceMonitor.enabled if `true`, creates a Prometheus Operator ServiceMonitor (also requires `metrics.enabled` to be `true`) + ## + enabled: false + ## @param metrics.serviceMonitor.namespace Namespace in which Prometheus is running + ## + namespace: "" + ## @param metrics.serviceMonitor.annotations Additional custom annotations for the ServiceMonitor + ## + annotations: {} + ## @param metrics.serviceMonitor.labels Extra labels for the ServiceMonitor + ## + labels: {} + ## @param metrics.serviceMonitor.jobLabel The name of the label on the target service to use as the job name in Prometheus + ## + jobLabel: "" + ## @param metrics.serviceMonitor.honorLabels honorLabels chooses the metric's labels on collisions with target labels + ## + honorLabels: false + ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped. + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## e.g: + ## interval: 10s + ## + interval: "" + ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended + ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint + ## e.g: + ## scrapeTimeout: 10s + ## + scrapeTimeout: "" + ## @param metrics.serviceMonitor.metricRelabelings Specify additional relabeling of metrics + ## + metricRelabelings: [] + ## @param metrics.serviceMonitor.relabelings Specify general relabeling + ## + relabelings: [] + ## @param metrics.serviceMonitor.selector Prometheus instance selector labels + ## ref: https://github.com/bitnami/charts/tree/main/bitnami/prometheus-operator#prometheus-configuration + ## selector: + ## prometheus: my-prometheus + ## + selector: {} + ## Prometheus Operator PrometheusRule configuration + ## + prometheusRule: + ## @param metrics.prometheusRule.enabled Create a PrometheusRule for Prometheus Operator + ## + enabled: false + ## @param metrics.prometheusRule.namespace Namespace for the PrometheusRule Resource (defaults to the Release Namespace) + ## + namespace: "" + ## @param metrics.prometheusRule.additionalLabels Additional labels that can be used so PrometheusRule will be discovered by Prometheus + ## + additionalLabels: {} + ## @param metrics.prometheusRule.rules PrometheusRule definitions + ## - alert: ClickhouseServerRestart + ## annotations: + ## message: Clickhouse-server started recently + ## expr: ClickHouseAsyncMetrics_Uptime > 1 < 180 + ## for: 5m + ## labels: + ## severity: warning + rules: [] +## @section External Zookeeper paramaters +## +externalZookeeper: + ## @param externalZookeeper.servers List of external zookeeper servers to use + ## @param externalZookeeper.port Port of the Zookeeper servers + ## + servers: [] + port: 2888 +## @section Zookeeper subchart parameters +## +## @param zookeeper.enabled Deploy Zookeeper subchart +## @param zookeeper.replicaCount Number of Zookeeper instances +## @param zookeeper.service.ports.client Zookeeper client port +## +zookeeper: + enabled: true + ## Override zookeeper default image as 3.9 is not supported https://github.com/ClickHouse/ClickHouse/issues/53749 + ## ref: https://github.com/bitnami/containers/tree/main/bitnami/zookeeper + ## @param zookeeper.image.registry [default: REGISTRY_NAME] Zookeeper image registry + ## @param zookeeper.image.repository [default: REPOSITORY_NAME/zookeeper] Zookeeper image repository + ## @skip zookeeper.image.tag Zookeeper image tag (immutable tags are recommended) + ## @param zookeeper.image.pullPolicy Zookeeper image pull policy + image: + registry: docker.io + repository: bitnami/zookeeper + tag: 3.8.4-debian-12-r0 + pullPolicy: IfNotPresent + replicaCount: 3 + service: + ports: + client: 2181 +## @section Network Policies +## Ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/ +## +networkPolicy: + ## @param networkPolicy.enabled Specifies whether a NetworkPolicy should be created + ## + enabled: true + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to the ports Clickhouse is listening + ## on. When true, Clickhouse will accept connections from any source + ## (with the correct destination port). + ## + allowExternal: true + ## @param networkPolicy.allowExternalEgress Allow the pod to access any range of port and all destinations. + ## + allowExternalEgress: true + ## @param networkPolicy.extraIngress [array] Add extra ingress rules to the NetworkPolice + ## e.g: + ## extraIngress: + ## - ports: + ## - port: 1234 + ## from: + ## - podSelector: + ## - matchLabels: + ## - role: frontend + ## - podSelector: + ## - matchExpressions: + ## - key: role + ## operator: In + ## values: + ## - frontend + extraIngress: [] + ## @param networkPolicy.extraEgress [array] Add extra ingress rules to the NetworkPolicy + ## e.g: + ## extraEgress: + ## - ports: + ## - port: 1234 + ## to: + ## - podSelector: + ## - matchLabels: + ## - role: frontend + ## - podSelector: + ## - matchExpressions: + ## - key: role + ## operator: In + ## values: + ## - frontend + ## + extraEgress: [] + ## @param networkPolicy.ingressNSMatchLabels [object] Labels to match to allow traffic from other namespaces + ## @param networkPolicy.ingressNSPodMatchLabels [object] Pod labels to match to allow traffic from other namespaces + ## + ingressNSMatchLabels: {} + ingressNSPodMatchLabels: {} From c844b7142e325bd7abc6927098c576fb87132dc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Wed, 20 Mar 2024 18:12:53 +0000 Subject: [PATCH 33/34] add reloader (#109) --- charts/vouch/templates/statefulset.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/charts/vouch/templates/statefulset.yaml b/charts/vouch/templates/statefulset.yaml index 1bcc4f518..8f16ea5fe 100644 --- a/charts/vouch/templates/statefulset.yaml +++ b/charts/vouch/templates/statefulset.yaml @@ -137,6 +137,31 @@ spec: port: {{ .Values.readinessProbe.httpGet.port }} scheme: {{ .Values.readinessProbe.httpGet.scheme }} {{- end }} + - name: fetch-keys-syncer + image: "{{ .Values.cliImage.repository }}:{{ .Values.cliImage.tag }}" + imagePullPolicy: {{ .Values.cliImage.pullPolicy }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 12 }} + envFrom: + - secretRef: + name: eso-{{ include "common.names.fullname" . }} + args: + - "sync-vouch-config" + - "--db-url-env" + - "ESO_DB_KEYSTORE_URL" + - "--vouch-dir" + - "{{ .Values.vouchDataDir }}" + - "--relays" + - '{{ join "," .Values.relays }}' + - "--refresh-frequency" + - "1800" + volumeMounts: + - name: data + mountPath: /data + - name: config + mountPath: /config + - name: scripts + mountPath: /scripts volumes: - name: config configMap: From c99db0cd86c1a791e1669aba2b92f6e98a99f1c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20J=C4=99drzejewski?= <33068017+matilote@users.noreply.github.com> Date: Tue, 6 Aug 2024 21:42:54 +0200 Subject: [PATCH 34/34] Feat/balval (#107) * add balval helm chart * remove redundant * fix version * add hostname to ch * fix hostname * fix network * configmap and volumes * add Chart.lock * add clickhouse from bitnami * remove clickhouse and use official * update lock file * add init image values * hardcode service type * not using eso env for init * use busybox for init * fix stx * read custom yaml * try all perms * revert * values fix * add yarn cache volume * remove alertlabels * fix path service monitor * add prom rules * fix template --- charts/balval/.helmignore | 23 ++ charts/balval/Chart.lock | 6 + charts/balval/Chart.yaml | 20 ++ charts/balval/templates/_helpers.tpl | 62 ++++++ charts/balval/templates/configmap.yaml | 13 ++ charts/balval/templates/external-secret.yaml | 18 ++ charts/balval/templates/prometheusrules.yaml | 124 +++++++++++ charts/balval/templates/service.yaml | 15 ++ charts/balval/templates/serviceaccount.yaml | 12 ++ charts/balval/templates/servicemonitor.yaml | 41 ++++ charts/balval/templates/statefulset.yaml | 158 ++++++++++++++ charts/balval/values.yaml | 214 +++++++++++++++++++ 12 files changed, 706 insertions(+) create mode 100644 charts/balval/.helmignore create mode 100644 charts/balval/Chart.lock create mode 100644 charts/balval/Chart.yaml create mode 100644 charts/balval/templates/_helpers.tpl create mode 100644 charts/balval/templates/configmap.yaml create mode 100644 charts/balval/templates/external-secret.yaml create mode 100644 charts/balval/templates/prometheusrules.yaml create mode 100644 charts/balval/templates/service.yaml create mode 100644 charts/balval/templates/serviceaccount.yaml create mode 100644 charts/balval/templates/servicemonitor.yaml create mode 100644 charts/balval/templates/statefulset.yaml create mode 100644 charts/balval/values.yaml diff --git a/charts/balval/.helmignore b/charts/balval/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/balval/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/balval/Chart.lock b/charts/balval/Chart.lock new file mode 100644 index 000000000..365356b87 --- /dev/null +++ b/charts/balval/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: file://../common + version: 1.0.0 +digest: sha256:07cebde439abe4ba19bb28e844b7419dab83c7f613886416aaf3ec08e8059144 +generated: "2024-03-17T01:44:51.25393669+01:00" diff --git a/charts/balval/Chart.yaml b/charts/balval/Chart.yaml new file mode 100644 index 000000000..fa38452e0 --- /dev/null +++ b/charts/balval/Chart.yaml @@ -0,0 +1,20 @@ +apiVersion: v2 +name: validator-balval +description: A Helm chart for installing and configuring Lido's validator-balval +type: application +version: 1.0.0 +appVersion: "v4.6.0" + +keywords: + - ethereum + - lido + - validators + - monitoring + - balval + +home: https://nethermind.io/ + +dependencies: +- name: common + repository: file://../common + version: 1.0.0 \ No newline at end of file diff --git a/charts/balval/templates/_helpers.tpl b/charts/balval/templates/_helpers.tpl new file mode 100644 index 000000000..318275f4e --- /dev/null +++ b/charts/balval/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "balval.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "balval.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "balval.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "balval.labels" -}} +helm.sh/chart: {{ include "balval.chart" . }} +{{ include "balval.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "balval.selectorLabels" -}} +app.kubernetes.io/name: {{ include "balval.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "balval.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "balval.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/balval/templates/configmap.yaml b/charts/balval/templates/configmap.yaml new file mode 100644 index 000000000..3a4c758f0 --- /dev/null +++ b/charts/balval/templates/configmap.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} +data: + custom_mainnet.yaml: | + operators: + - name: Test + keys: + - "0xb1294eeeab227054786e3da41566f0472c581e4dc74834fca7287817a64edb4a4a4e912c7ae8ae703d417fe43a221cdb" \ No newline at end of file diff --git a/charts/balval/templates/external-secret.yaml b/charts/balval/templates/external-secret.yaml new file mode 100644 index 000000000..8adf62f98 --- /dev/null +++ b/charts/balval/templates/external-secret.yaml @@ -0,0 +1,18 @@ +{{- if .Values.global.externalSecrets.enabled }} +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: eso-{{ include "common.names.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} +spec: + refreshInterval: 10m + secretStoreRef: + name: {{ .Values.global.externalSecrets.secretStoreRef.name }} + kind: {{ .Values.global.externalSecrets.secretStoreRef.kind }} + target: + name: eso-{{ include "common.names.fullname" . }} + creationPolicy: Owner + data: + {{- .Values.global.externalSecrets.data | toYaml | trim | nindent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/balval/templates/prometheusrules.yaml b/charts/balval/templates/prometheusrules.yaml new file mode 100644 index 000000000..a7b6e3b00 --- /dev/null +++ b/charts/balval/templates/prometheusrules.yaml @@ -0,0 +1,124 @@ +{{- if .Values.prometheusRule.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "common.names.fullname" . }} + {{- if .Values.prometheusRule.namespace }} + namespace: {{ .Values.prometheusRule.namespace }} + {{- else }} + namespace: {{ .Release.Namespace | quote }} + {{- end }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.prometheusRule.additionalLabels }} + {{- toYaml .Values.prometheusRule.additionalLabels | nindent 4 }} + {{- end }} +spec: + groups: + {{- with .Values.prometheusRule.rules }} + - name: {{ include "common.names.fullname" $ }} + rules: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- if .Values.prometheusRule.default }} + - name: {{ include "common.names.fullname" $ }}-default + rules: + - alert: UserSlashedValidators + expr: (ethereum_validators_monitoring_user_validators{status="slashed"} - ethereum_validators_monitoring_user_validators{status="slashed"} offset 1m) > 0 + labels: + severity: critical + annotations: + emoji: 🔪 + summary: "Operators have slashed validators" + description: 'Number of slashed validators per operator' + - alert: DataActuality + expr: absent(ethereum_validators_monitoring_data_actuality) OR (ethereum_validators_monitoring_data_actuality / 1000 > 3600) + for: 1m + labels: + severity: critical + annotations: + emoji: ⏳ + summary: "Data actuality greater then 1 hour" + resolved_summary: "Data actuality is back to normal and now less then 1 hour" + description: "It's not OK. Please, check app health" + - alert: NumValidatorsWithNegativeDelta + expr: ethereum_validators_monitoring_validator_count_with_negative_balances_delta > 0 AND ON() changes(ethereum_validators_monitoring_epoch_number[1m]) > 0 + labels: + severity: critical + annotations: + emoji: 💸 + summary: 'Operators have a negative balance delta' + resolved_summary: 'Operators have a positive balance delta' + description: 'Number of validators per operator who have a negative balance delta.' + - alert: NumValidatorsWithMissAttestationLastNEpoch + expr: ethereum_validators_monitoring_validator_count_miss_attestation_last_n_epoch > 0 AND ON() changes(ethereum_validators_monitoring_epoch_number[1m]) > 0 + labels: + severity: critical + annotations: + emoji: 📝❌ + summary: 'Operators have missed attestation in last {{ printf "{{ $labels.epoch_interval }}" }} finalized epochs' + resolved_summary: 'Operators not have missed attestation in last {{ printf "{{ $labels.epoch_interval }}" }} finalized epochs' + description: 'Number of validators per operator who have missed attestations.' + - alert: NumValidatorsWithHighIncDelayAttestationLastNEpoch + expr: ethereum_validators_monitoring_validator_count_high_inc_delay_last_n_epoch > 0 AND ON() changes(ethereum_validators_monitoring_epoch_number[1m]) > 0 + labels: + severity: critical + annotations: + emoji: 📝🐢 + summary: 'Operators have attestation inc. delay greater than 2 in last {{ printf "{{ $labels.epoch_interval }}" }} finalized epochs' + description: 'Number of validators per operator who have attestation with high inc. delay.' + - alert: NumValidatorsWithInvalidPropertyAttestationLastNEpoch + expr: ethereum_validators_monitoring_validator_count_invalid_attestation_property_last_n_epoch > 0 AND ON() changes(ethereum_validators_monitoring_epoch_number[1m]) > 0 + labels: + severity: critical + annotations: + emoji: '📝🏷️' + summary: 'Operators have invalid attestation property in last {{ printf "{{ $labels.epoch_interval }}" }} finalized epochs' + description: 'Number of validators per operator who have invalid attestation property.' + - alert: NumValidatorsWithMissPropose + expr: ethereum_validators_monitoring_validator_count_miss_propose > 0 AND ON() changes(ethereum_validators_monitoring_epoch_number[1m]) > 0 + labels: + severity: critical + annotations: + emoji: 📥 + summary: 'Operators missed block propose in the last finalized epoch' + resolved_summary: 'Operators not missed block propose in the last finalized epoch' + description: 'Number of validators per operator who missed block propose.' + - alert: NumValidatorsWithSyncParticipationLessAvgLastNEpoch + expr: ethereum_validators_monitoring_validator_count_with_sync_participation_less_avg_last_n_epoch > 0 AND ON() changes(ethereum_validators_monitoring_epoch_number[1m]) > 0 + labels: + severity: critical + annotations: + emoji: 🔄 + summary: 'Operators sync participation less than average in last {{ printf "{{ $labels.epoch_interval }}" }} finalized epochs' + resolved_summary: 'Operators sync participation higher or equal than average in last {{ printf "{{ $labels.epoch_interval }}" }} finalized epochs' + description: 'Number of validators per operator whose sync participation less than average.' + - alert: HighRewardNumValidatorsWithSyncParticipationLessAvgLastNEpoch + expr: ethereum_validators_monitoring_high_reward_validator_count_with_sync_participation_less_avg_last_n_epoch > 0 AND ON() changes(ethereum_validators_monitoring_epoch_number[1m]) > 0 + labels: + severity: critical + annotations: + emoji: '📈🔄' + summary: 'Operators may get high rewards in the future, but sync participation less than average in last {{ printf "{{ $labels.epoch_interval }}" }} finalized epochs!' + resolved_summary: 'Operators sync participation higher or equal than average in last {{ printf "{{ $labels.epoch_interval }}" }} finalized epoch. Now may get high rewards in the future!' + description: 'Number of validators per operator whose sync participation less than average.' + - alert: HighRewardNumValidatorsWithMissAttestationLastNEpoch + expr: ethereum_validators_monitoring_high_reward_validator_count_miss_attestation_last_n_epoch > 0 AND ON() changes(ethereum_validators_monitoring_epoch_number[1m]) > 0 + labels: + severity: critical + annotations: + emoji: '📈📝❌' + summary: 'Operators may get high rewards in the future, but missed attestation in last {{ printf "{{ $labels.epoch_interval }}" }} finalized epochs!' + resolved_summary: 'Operators not have missed attestation in last {{ printf "{{ $labels.epoch_interval }}" }} finalized epochs. Now may get high rewards in the future!' + description: 'Number of validators per operator who have missed attestations.' + - alert: HighRewardNumValidatorsWithMissPropose + expr: ethereum_validators_monitoring_high_reward_validator_count_miss_propose > 0 AND ON() changes(ethereum_validators_monitoring_epoch_number[1m]) > 0 + labels: + severity: critical + annotations: + emoji: '📈📥' + summary: 'Operators may get high rewards in the future, but missed block propose in the last finalized epoch!' + resolved_summary: 'Operators not missed block propose in the last finalized epoch. Now may get high rewards in the future!' + description: 'Number of validators per operator who missed block propose.' + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/balval/templates/service.yaml b/charts/balval/templates/service.yaml new file mode 100644 index 000000000..3b4ea88f7 --- /dev/null +++ b/charts/balval/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "balval.fullname" . }} + labels: + {{- include "balval.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.balval.httpPort }} + targetPort: metrics + protocol: TCP + name: metrics + selector: + {{- include "balval.selectorLabels" . | nindent 4 }} diff --git a/charts/balval/templates/serviceaccount.yaml b/charts/balval/templates/serviceaccount.yaml new file mode 100644 index 000000000..cb2edd11f --- /dev/null +++ b/charts/balval/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if or .Values.global.serviceAccount.create .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "common.names.serviceAccountName" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/balval/templates/servicemonitor.yaml b/charts/balval/templates/servicemonitor.yaml new file mode 100644 index 000000000..bd99c3387 --- /dev/null +++ b/charts/balval/templates/servicemonitor.yaml @@ -0,0 +1,41 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "common.names.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ .Values.serviceMonitor.namespace }} + {{- else }} + namespace: {{ .Release.Namespace | quote }} + {{- end }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- if .Values.serviceMonitor.additionalLabels }} + {{- toYaml .Values.serviceMonitor.additionalLabels | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: metrics + path: /metrics + {{- if .Values.serviceMonitor.interval }} + interval: {{ .Values.serviceMonitor.interval }} + {{- end }} + {{- if .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} + {{- end }} + {{- if .Values.serviceMonitor.honorLabels }} + honorLabels: {{ .Values.serviceMonitor.honorLabels }} + {{- end }} + {{- if .Values.serviceMonitor.relabelings }} + relabelings: {{- toYaml .Values.serviceMonitor.relabelings | nindent 6 }} + {{- end }} + {{- if .Values.serviceMonitor.metricRelabelings }} + metricRelabelings: {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "common.labels.matchLabels" . | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/charts/balval/templates/statefulset.yaml b/charts/balval/templates/statefulset.yaml new file mode 100644 index 000000000..923b5f9b9 --- /dev/null +++ b/charts/balval/templates/statefulset.yaml @@ -0,0 +1,158 @@ +apiVersion: {{ include "common.capabilities.statefulset.apiVersion" . }} +kind: StatefulSet +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "common.labels.standard" . | nindent 4 }} +spec: + replicas: {{ .Values.global.replicaCount }} + selector: + matchLabels: + {{- include "common.labels.matchLabels" . | nindent 6 }} + serviceName: {{ include "common.names.fullname" . }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "common.labels.matchLabels" . | nindent 8 }} + spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "balval.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.global.podSecurityContext | nindent 8 }} + initContainers: + - name: init + image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}" + imagePullPolicy: {{ .Values.initImage.pullPolicy }} + securityContext: + {{- toYaml .Values.global.securityContext | nindent 12 }} + command: + - sh + - -ac + - > + mkdir -p {{ .Values.balval.dataDir }}; + cp /config/custom_mainnet.yaml {{ .Values.balval.dataDir }}/; + chown -R {{ .Values.global.podSecurityContext.runAsUser }}:{{ .Values.global.podSecurityContext.fsGroup }} {{ .Values.balval.dataDir }}; + cat {{ .Values.balval.dataDir }}/custom_mainnet.yaml + volumeMounts: + - name: data + mountPath: /data + - name: config + mountPath: /config + containers: + - name: {{ .Chart.Name }} + securityContext: + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 10000 + image: "{{ .Values.global.image.repository }}:{{ .Values.global.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.global.image.pullPolicy }} + env: + - name: WORKING_MODE + value: "{{ .Values.balval.workingMode }}" + - name: NODE_ENV + value: "{{ .Values.balval.env }}" + - name: DRY_RUN + value: "{{ .Values.balval.dryRun }}" + - name: LOG_LEVEL + value: "{{ .Values.balval.logLevel }}" + - name: LOG_FORMAT + value: "{{ .Values.balval.logFormat }}" + - name: HTTP_PORT + value: "{{ .Values.balval.httpPort }}" + {{- with .Values.clickhouse }} + - name: DB_NAME + value: "{{ .auth.database }}" + - name: DB_HOST + value: "{{ .hostname }}" + - name: DB_USER + value: "{{ .auth.username }}" + - name: DB_PASSWORD + value: "{{ .auth.password }}" + - name: DB_PORT + value: "{{ .service.ports.http }}" + {{- end }} + - name: DB_MAX_RETRIES + value: "{{ .Values.balval.dbMaxRetries }}" + - name: DB_MIN_BACKOFF_SEC + value: "{{ .Values.balval.dbMinBackoffSec }}" + - name: DB_MAX_BACKOFF_SEC + value: "{{ .Values.balval.dbMaxBackoffSec }}" + - name: START_EPOCH + value: "{{ .Values.balval.startEpoch }}" + - name: ETH_NETWORK + value: "{{ .Values.balval.network }}" + - name: VALIDATOR_REGISTRY_SOURCE + value: "{{ .Values.balval.validatorRegistrySource }}" + - name: VALIDATOR_REGISTRY_FILE_SOURCE_PATH + value: "{{ .Values.balval.validatorRegistryFileSourcePath }}" + - name: VALIDATOR_REGISTRY_LIDO_SOURCE_SQLITE_CACHE_PATH + value: "{{ .Values.balval.validatorRegistryLidoSourceSqliteCachePath }}" + - name: EL_RPC_URLS + value: "{{ .Values.balval.elRpcUrls }}" + - name: CL_API_URLS + value: "{{ .Values.balval.clRpcUrls }}" + - name: CL_API_RETRY_DELAY_MS + value: "{{ .Values.balval.clApiRetryDelayMs }}" + - name: CL_API_GET_RESPONSE_TIMEOUT + value: "{{ .Values.balval.clApiGetResponseTimeout }}" + - name: CL_API_MAX_RETRIES + value: "{{ .Values.balval.clApiMaxRetries }}" + - name: CL_API_GET_BLOCK_INFO_MAX_RETRIES + value: "{{ .Values.balval.clApiGetBlockInfoMaxRetries }}" + - name: FETCH_INTERVAL_SLOTS + value: "{{ .Values.balval.fetchIntervalSlots }}" + - name: CHAIN_SLOT_TIME_SECONDS + value: "{{ .Values.balval.chainSlotTimeSeconds }}" + - name: SYNC_PARTICIPATION_DISTANCE_DOWN_FROM_CHAIN_AVG + value: "{{ .Values.balval.syncParticipationDistanceDownFromChainAvg }}" + - name: SYNC_PARTICIPATION_EPOCHS_LESS_THAN_CHAIN_AVG + value: "{{ .Values.balval.syncParticipationEpochsLessThanChainAvg }}" + - name: BAD_ATTESTATION_EPOCHS + value: "{{ .Values.balval.badAttestationEpochs }}" + - name: CRITICAL_ALERTS_ALERTMANAGER_URL + value: "{{ .Values.balval.alertmanagerUrl }}" + - name: CRITICAL_ALERTS_MIN_VAL_COUNT + value: "{{ .Values.balval.minValCount }}" + {{- if .Values.global.externalSecrets.enabled }} + envFrom: + - secretRef: + name: eso-{{ include "common.names.fullname" . }} + {{- end }} + ports: + - name: metrics + containerPort: {{ .Values.balval.httpPort }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /data + - name: config + mountPath: /config + volumes: + - name: config + configMap: + name: {{ include "common.names.fullname" . }} + - name: data + emptyDir: + medium: Memory + sizeLimit: 128Mi \ No newline at end of file diff --git a/charts/balval/values.yaml b/charts/balval/values.yaml new file mode 100644 index 000000000..d508627a3 --- /dev/null +++ b/charts/balval/values.yaml @@ -0,0 +1,214 @@ +# Default values for web3signer. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + serviceAccount: + create: true + imagePullSecrets: [] + ## Pod Security Context + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + podSecurityContext: + runAsNonRoot: true + runAsUser: 10000 + fsGroup: 10000 + + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10000 + capabilities: + drop: + - ALL + + replicaCount: 1 + + externalSecrets: + enabled: false + secretStoreRef: secretStoreRef + data: + - secretKey: DB_NAME + remoteRef: + key: validatorBalvalSecrets + property: dbName + - secretKey: DB_PORT + remoteRef: + key: validatorBalvalSecrets + property: dbPort + - secretKey: DB_HOST + remoteRef: + key: validatorBalvalSecrets + property: dbHost + - secretKey: DB_USER + remoteRef: + key: validatorBalvalSecrets + property: dbUser + - secretKey: DB_PASSWORD + remoteRef: + key: validatorBalvalSecrets + property: dbPassword + + image: + repository: lidofinance/ethereum-validators-monitoring + pullPolicy: IfNotPresent + tag: "4.6.0" + +nameOverride: "" +fullnameOverride: "" + +## Init image is used to chown data volume, etc. +## +initImage: + repository: busybox + tag: "1.36" + pullPolicy: IfNotPresent + +balval: + env: "production" + workingMode: "head" + dryRun: false + logLevel: "debug" + logFormat: "json" + httpPort: 8080 + dbMaxRetries: 10 + dbMinBackoffSec: 1 + dbMaxBackoffSec: 120 + startEpoch: 270292 + network: 1 + validatorRegistrySource: "file" + dataDir: "/data/balval" + validatorRegistryFileSourcePath: "/data/balval/custom_mainnet.yaml" + validatorRegistryLidoSourceSqliteCachePath: "/data/balval/lido_mainnet.db" + elRpcUrls: "http://localhost:8545" + clRpcUrls: "http://localhost:5052" + clApiRetryDelayMs: 500 + clApiGetResponseTimeout: 30000 + clApiMaxRetries: 2 + clApiGetBlockInfoMaxRetries: 2 + fetchIntervalSlots: 32 + chainSlotTimeSeconds: 12 + syncParticipationDistanceDownFromChainAvg: 0 + syncParticipationEpochsLessThanChainAvg: 3 + badAttestationEpochs: 3 + alertmanagerUrl: "http://localhost:9093" + minValCount: 100 + +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +## +serviceAccount: + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +clickhouse: + enabled: true + hostname: clickhouse + auth: + username: default + password: "" + database: default + existingSecret: "" + existingSecretKey: "" + service: + ports: + http: 8123 + +## Configure resource requests and limits. +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: {} + +## Affinity for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Prometheus Service Monitor +## ref: https://github.com/coreos/prometheus-operator +## https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint +## +serviceMonitor: + ## @param metrics.serviceMonitor.enabled Create ServiceMonitor resource(s) for scraping metrics using PrometheusOperator + ## + enabled: false + ## @param metrics.serviceMonitor.namespace The namespace in which the ServiceMonitor will be created + ## + namespace: "" + ## @param metrics.serviceMonitor.interval The interval at which metrics should be scraped + ## + interval: 30s + ## @param metrics.serviceMonitor.scrapeTimeout The timeout after which the scrape is ended + ## + scrapeTimeout: "" + ## @param metrics.serviceMonitor.relabellings Metrics RelabelConfigs to apply to samples before scraping. + ## + relabelings: [] + ## @param metrics.serviceMonitor.metricRelabelings Metrics RelabelConfigs to apply to samples before ingestion. + ## + metricRelabelings: [] + ## @param metrics.serviceMonitor.honorLabels Specify honorLabels parameter to add the scrape endpoint + ## + honorLabels: false + ## @param metrics.serviceMonitor.additionalLabels Additional labels that can be used so ServiceMonitor resource(s) can be discovered by Prometheus + ## + additionalLabels: {} + +prometheusRule: + ## @param metrics.prometheusRule.enabled Create a custom prometheusRule Resource for scraping metrics using PrometheusOperator + ## + enabled: false + ## @param metrics.prometheusRule.default Create a default set of Alerts + ## + default: true + ## @param metrics.prometheusRule.namespace The namespace in which the prometheusRule will be created + ## + namespace: "" + ## @param metrics.prometheusRule.additionalLabels Additional labels for the prometheusRule + ## + additionalLabels: {} + ## @param metrics.prometheusRule.rules Custom Prometheus rules + ## e.g: + ## rules: + ## - alert: PrysmValidatorHourlyEarningLessOrEqual0 + ## expr: sum(validator_balance) - sum(validator_balance offset 1h) - count(validator_balance > 16)*32 + count(validator_balance offset 1h > 0)*32 + ## for: 5m + ## labels: + ## severity: critical + ## annotations: + ## summary: Prysm validator hourly earning <= 0 + ## description: Check validators immediately. Pod - {{ printf "{{ $labels.pod }}" }}. Namespace - {{ printf "{{ $labels.namespace }}" }} + ## - alert: PrysmValidatorAlotOfErrorsLastHour + ## expr: sum(delta(log_entries_total{job='{{ include "operator.fullname" . }}-validator', level="error"}[1h]) > 0) + ## for: 5m + ## labels: + ## severity: warning + ## annotations: + ## summary: Many validator errors or warnings last hour + ## description: Check validator {{ printf "{{ $labels.pod }}" }}. Namespace - {{ printf "{{ $labels.namespace }}" }} + ## + rules: {}