-
Notifications
You must be signed in to change notification settings - Fork 37
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[refinery] Add option to run refinery as a statefulset
Using a statefulset allows giving the pods a stable network identity. Setting `setHostnameAsFQDN` means that this stable network identity is what `os.Hostname` reports. Together, they allow using the hostname in the peer list, so that the peer list remains stable even as pods are rescheduled. This improves trace routing stability during refinery upgrades and Kubernetes cluster operations (upgrades / scale-downs), and even makes it possible to run refinery with an affinity preference for spot instances.
- Loading branch information
1 parent
b7a3ac6
commit 41e6685
Showing
6 changed files
with
204 additions
and
116 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| {{- define "refinery.pod" -}} | ||
| metadata: | ||
| annotations: | ||
| checksum/config: {{ include (print $.Template.BasePath "/configmap-config.yaml") . | sha256sum }} | ||
| {{- with .Values.podAnnotations }} | ||
| {{- toYaml . | nindent 4 }} | ||
| {{- end }} | ||
| {{- if eq .Values.config.Metrics "prometheus" }} | ||
| prometheus.io/port: "9090" | ||
| prometheus.io/scrape: "true" | ||
| {{- end }} | ||
| {{- if hasKey .Values.rules "LiveReload" | ternary (not .Values.rules.LiveReload) false }} | ||
| checksum/rules: {{ include (print $.Template.BasePath "/configmap-rules.yaml") . | sha256sum }} | ||
| {{- end }} | ||
| labels: | ||
| {{- include "refinery.selectorLabels" . | nindent 4 }} | ||
| {{- with .Values.podLabels }} | ||
| {{- toYaml . | nindent 4 }} | ||
| {{- end }} | ||
| spec: | ||
| {{- with .Values.imagePullSecrets }} | ||
| imagePullSecrets: | ||
| {{- toYaml . | nindent 4 }} | ||
| {{- end }} | ||
| serviceAccountName: {{ include "refinery.serviceAccountName" . }} | ||
| {{- if eq .Values.mode "statefulset" }} | ||
| # The makes the pod hostnames be resolvable. | ||
| setHostnameAsFQDN: true | ||
| {{- end }} | ||
| securityContext: | ||
| {{- toYaml .Values.podSecurityContext | nindent 4 }} | ||
| containers: | ||
| - name: {{ .Chart.Name }} | ||
| securityContext: | ||
| {{- toYaml .Values.securityContext | nindent 8 }} | ||
| image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" | ||
| imagePullPolicy: {{ .Values.image.pullPolicy }} | ||
| command: | ||
| - "refinery" | ||
| - "-c" | ||
| - "/etc/refinery/config.yaml" | ||
| - "-r" | ||
| - "/etc/refinery/rules.yaml" | ||
| {{- with .Values.environment }} | ||
| env: | ||
| {{- toYaml . | nindent 8 }} | ||
| {{- end }} | ||
| ports: | ||
| - name: data | ||
| containerPort: 8080 | ||
| protocol: TCP | ||
| - name: grpc | ||
| containerPort: 4317 | ||
| protocol: TCP | ||
| - name: peer | ||
| containerPort: 8081 | ||
| protocol: TCP | ||
| {{- if eq .Values.config.Metrics "prometheus" }} | ||
| - name: metrics | ||
| containerPort: 9090 | ||
| protocol: TCP | ||
| {{- end }} | ||
| volumeMounts: | ||
| - name: refinery-config | ||
| mountPath: /etc/refinery/ | ||
| {{- with .Values.extraVolumeMounts }} | ||
| {{- toYaml . | nindent 8 }} | ||
| {{- end }} | ||
| livenessProbe: | ||
| httpGet: | ||
| path: /alive | ||
| port: data | ||
| initialDelaySeconds: 10 | ||
| periodSeconds: 10 | ||
| failureThreshold: 3 | ||
| readinessProbe: | ||
| httpGet: | ||
| path: /alive | ||
| port: data | ||
| initialDelaySeconds: 0 | ||
| periodSeconds: 3 | ||
| failureThreshold: 5 | ||
| resources: | ||
| {{- toYaml .Values.resources | nindent 8 }} | ||
| volumes: | ||
| - name: refinery-config | ||
| projected: | ||
| sources: | ||
| - configMap: | ||
| name: {{ include "refinery.fullname" . }}-config | ||
| items: | ||
| - key: config.yaml | ||
| path: config.yaml | ||
| - configMap: | ||
| {{- if .Values.config.RulesConfigMapName }} | ||
| name: {{ .Values.config.RulesConfigMapName }} | ||
| {{- else }} | ||
| name: {{ include "refinery.fullname" . }}-rules | ||
| {{- end }} | ||
| items: | ||
| - key: rules.yaml | ||
| path: rules.yaml | ||
| {{- with .Values.extraVolumes }} | ||
| {{- toYaml . | nindent 4 }} | ||
| {{- end }} | ||
| {{- with .Values.nodeSelector }} | ||
| nodeSelector: | ||
| {{- toYaml . | nindent 4 }} | ||
| {{- end }} | ||
| {{- with .Values.affinity }} | ||
| affinity: | ||
| {{- toYaml . | nindent 4 }} | ||
| {{- end }} | ||
| {{- with .Values.tolerations }} | ||
| tolerations: | ||
| {{- toYaml . | nindent 4 }} | ||
| {{- end }} | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| {{- if eq .Values.mode "statefulset" -}} | ||
| # Governing service to provide stable network ID for StatefulSet pods: | ||
| # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id | ||
| apiVersion: v1 | ||
| kind: Service | ||
| metadata: | ||
| name: {{ include "refinery.fullname" . }}-cluster | ||
| namespace: {{ .Release.Namespace }} | ||
| labels: | ||
| {{- include "refinery.labels" . | nindent 4 }} | ||
| {{- with .Values.service.labels }} | ||
| {{- toYaml . | nindent 4 }} | ||
| {{- end }} | ||
| spec: | ||
| type: ClusterIP | ||
| clusterIP: None | ||
| selector: | ||
| {{- include "refinery.selectorLabels" . | nindent 4 }} | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| {{- if eq .Values.mode "statefulset" -}} | ||
| apiVersion: apps/v1 | ||
| kind: StatefulSet | ||
| metadata: | ||
| name: {{ include "refinery.fullname" . }} | ||
| namespace: {{ .Release.Namespace }} | ||
| labels: | ||
| {{- include "refinery.labels" . | nindent 4 }} | ||
| {{- with .Values.deploymentAnnotations }} | ||
| annotations: {{ toYaml . | nindent 4 }} | ||
| {{- end }} | ||
| spec: | ||
| # Governing service to provide stable network ID for StatefulSet pods: | ||
| # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-network-id | ||
| serviceName: {{ include "refinery.fullname" . }}-cluster | ||
| # Refinery doesn't really have any state, so in theory there's no | ||
| # need for the controlled scale-up / scale-down of the default | ||
| # OrderedReady policy. However if all the pods come up at once while | ||
| # the redis peer list exists, most pods will crash loop because | ||
| # they're unable to reach some of the peers. The OrderedReady delay | ||
| # gives time for membership to expire, and makes it quicker overall | ||
| # unless you also take care to blow away the redis peer list when | ||
| # scaling. | ||
| podManagementPolicy: OrderedReady | ||
| {{- if not .Values.autoscaling.enabled }} | ||
| replicas: {{ .Values.replicaCount }} | ||
| {{- end }} | ||
| selector: | ||
| matchLabels: | ||
| {{- include "refinery.selectorLabels" . | nindent 6 }} | ||
| template: | ||
| {{- include "refinery.pod" . | nindent 4 }} | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters