Merge pull request #234 from oldgiova/MC-6856-converge-helm-kustomize…

…-pt2

Switch to storage gateway from minio and CustomEnvs

README.md

@@ -301,8 +301,11 @@ The following table lists the parameters for the `api-gateway` component and the
 | `api_gateway.service.httpNodePort` | Node port for the HTTP service | `nil` |
 | `api_gateway.service.httpsNodePort` | Node port for the HTTPS service | `nil` |
 | `api_gateway.env.SSL` | SSL termination flag | `true` |
-| `api_gateway.minio.enabled` | Enable routing of S3 requests to the minio service | `true` |
-| `api_gateway.minio.url` | URL of the minio service | `http://minio:9000` |
+| `api_gateway.minio.enabled` | Enable routing of S3 requests to the minio service. **[Deprecated from 5.5.0, use `api_gateway.storage_proxy`` instead]** | `true` |
+| `api_gateway.minio.url` | URL of the minio service. **[Deprecated from 5.5.0, use `api_gateway.storage_proxy`` instead]** | `http://minio:9000` |
+| `api_gateway.storage_proxy.enabled` | Enable storage proxy to the S3/minio service | `false` |
+| `api_gateway.storage_proxy.url` | URL of the storage proxy. Should point to the AWS Bucket/Minio URL | `nil` |
+| `api_gateway.storage_proxy.customRule` | Custom rules for the storage proxy. | ``HostRegexp(`{domain:^artifacts.*$}`)`` |
 | `api_gateway.rateLimit.average` | See the [Traefik rate limit configuration options](https://doc.traefik.io/traefik/v2.6/middlewares/http/ratelimit/#configuration-options) | `100` |
 | `api_gateway.rateLimit.burst` | See the [Traefik rate limit configuration options](https://doc.traefik.io/traefik/v2.6/middlewares/http/ratelimit/#configuration-options) | `100` |
 | `api_gateway.rateLimit.period` | See the [Traefik rate limit configuration options](https://doc.traefik.io/traefik/v2.6/middlewares/http/ratelimit/#configuration-options) | `1s` |
@@ -317,7 +320,7 @@ The following table lists the parameters for the `api-gateway` component and the
 | `api_gateway.containerSecurityContext.runAsUser` | User ID for the container | `65534` |
 | `api_gateway.compression` | Enable Traefik Compression | `true` |
 | `api_gateway.security_redirect` | Custom redirect to a company security page | `null` |
-| `api_gateway.minio.customRule` | Custom redirect for MinIO. Uses the default one if not specified | `null` |
+| `api_gateway.minio.customRule` | Custom redirect for MinIO. Uses the default one if not specified. **[Deprecated from 5.5.0, use `api_gateway.storage_proxy`` instead]** | `null` |
 | `api_gateway.hpa` | HorizontalPodAutoscaler support | `nil` |
 | `api_gateway.hpa.enabled` | HorizontalPodAutoscaler enabled | `nil` |
 | `api_gateway.hpa.minReplicas` | HorizontalPodAutoscaler minReplicas | `nil` |

mender/CHANGELOG.md

@@ -1,7 +1,20 @@
 # Mender Helm chart
 
-## master
+## 5.5.0
 * Fix mongodb uri creation when using the mongodb subchart and replicast architecture
+* Added customEnv option to set default or per-service custom env variables
+* Added generic `storage_proxy` service, that could
+  work for both minio and Amazon S3, and it's going to replace the `api_gateway.minio` configuration.
+* **Deprecations**:
+  * `api_gateway.minio` is deprecated in favor of `api_gateway.storage_proxy`.
+    This entry could be used, but it is no longer maintained, and could be removed
+    in future releases.  
+    **How to upgrade**: 
+    * set `api_gateway.minio.enabled=false`
+    * set `api_gateway.storage_proxy.enabled=true`
+    * set `api_gateway.storage_proxy.url` to the external storage url that you want to map externally. For example `https://fleetstorage.example.com`.
+      If you leave it empty, it uses the Amazon S3 external URL.
+
 
 ## Version 5.4.1
 * Upgrade to Mender version `3.7.1`.

mender/templates/_helpers.tpl

@@ -279,3 +279,40 @@ Create the name of the service account
 {{- default "default" $dot.Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+Synopsis:
+{{- include "mender.customEnvs" (merge (deepCopy .dot.Values.<service>) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
+*/}}
+{{- define "mender.customEnvs" -}}
+{{- with .customEnvs }}
+{{- toYaml . }}
+{{- println "" }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Define mender.storageProxyUrl
+*/}}
+{{- define "mender.storageProxyUrl" -}}
+{{- $dot := (ternary . .dot (empty .dot)) -}}
+{{- with $dot.Values.api_gateway.storage_proxy }}
+  {{- if .url }}
+    {{- printf "%s" .url }}
+  {{- else if eq $dot.Values.global.storage "aws" }}
+    {{- printf "https://%s.s3.%s.amazonaws.com" $dot.Values.global.s3.AWS_BUCKET $dot.Values.global.s3.AWS_REGION}}
+  {{- else }}
+    {{- required "A valid storage proxy URL is required" $dot.Values.api_gateway.storage_proxy.url }}
+  {{- end }}
+{{- else }}
+{{- printf "" }}
+{{- end }}
+{{- end }}
+
+{{/*
+Storage Proxy Rule
+*/}}
+{{- define "mender.storageProxyRule" -}}
+  {{- default "HostRegexp(`{domain:^artifacts.*$}`)" .Values.api_gateway.storage_proxy.customRule | quote }}
+{{- end -}}
+

mender/templates/api-gateway/configmap.yaml

@@ -417,7 +417,9 @@ data:
           rule: "!Path(`/api/management/{ver:v[0-9]+}/useradm/auth/logout`)&&PathPrefix(`/api/management/{ver:v[0-9]+}/useradm/{ep:(auth|oauth2|oidc)}`)"
           service: useradm
           tls: {{ $isTls }}
-{{- if .Values.api_gateway.minio.enabled }}
+{{- if and .Values.api_gateway.storage_proxy.enabled .Values.api_gateway.minio.enabled }}
+    {{- fail "You cannot set both api_gateway.storage_proxy and api_gateway.minio. Choose one" }}
+{{- else if and .Values.api_gateway.minio.enabled (not .Values.api_gateway.storage_proxy.enabled) }}
 
         #
         # minio
@@ -430,6 +432,19 @@ data:
           priority: 900
           service: minio
           tls: {{ $isTls }}
+{{- else if and .Values.api_gateway.storage_proxy.enabled (not .Values.api_gateway.minio.enabled) }}
+
+        #
+        # storage_proxy
+        #
+        storage_proxy:
+          entrypoints: {{ $scheme }}
+          middlewares:
+          - ratelimit
+          rule: {{ include "mender.storageProxyRule" . }}
+          priority: 65535
+          service: storage_proxy
+          tls: {{ $isTls }}
 {{- end }}
 
       #
@@ -496,12 +511,20 @@ data:
           loadBalancer:
             servers:
             - url: "http://{{ .Values.useradm.service.name }}:{{ .Values.useradm.service.port }}"
-{{- if .Values.api_gateway.minio.enabled }}
+{{- if and .Values.api_gateway.storage_proxy.enabled .Values.api_gateway.minio.enabled }}
+    {{- fail "You cannot set both api_gateway.storage_proxy and api_gateway.minio. Choose one" }}
+{{- else if and .Values.api_gateway.minio.enabled (not .Values.api_gateway.storage_proxy.enabled) }}
 
         minio:
           loadBalancer:
             servers:
             - url: "{{ .Values.api_gateway.minio.url }}"
+{{- else if and .Values.api_gateway.storage_proxy.enabled (not .Values.api_gateway.minio.enabled) }}
+
+        storage_proxy:
+          loadBalancer:
+            servers:
+            - url: {{ include "mender.storageProxyUrl" . }}
 {{- end }}
 
       #

mender/templates/auditlogs/_podtemplate.yaml

@@ -67,6 +67,7 @@ spec:
     env:
     - name: AUDITLOGS_AUDITLOG_EXPIRE_SECONDS
       value: {{ .dot.Values.auditlogs.logRetentionSeconds | int | toString | quote }}
+    {{- include "mender.customEnvs" (merge (deepCopy .dot.Values.auditlogs) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
     # Supported configuration settings: https://github.com/mendersoftware/auditlogs/blob/master/config.yaml
     # Set in order, last value for the key will be used in case duplications.
     envFrom:

mender/templates/create-artifact-worker/deployment.yaml

@@ -78,6 +78,7 @@ spec:
           value: "1"
         - name: CREATE_ARTIFACT_DEPLOYMENTS_URL
           value: http://{{ .Values.deployments.service.name }}:{{ .Values.deployments.service.port }}
+        {{- include "mender.customEnvs" (merge (deepCopy .Values.create_artifact_worker) (deepCopy (default (dict) .Values.default))) | nindent 8 }}
         envFrom:
         - prefix: WORKFLOWS_
           secretRef:

mender/templates/deployments/_podtemplate.yaml

@@ -93,6 +93,7 @@ spec:
       value: "true"
     {{- end }}
     {{- end }}
+    {{- include "mender.customEnvs" (merge (deepCopy .dot.Values.deployments) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
 
     # Supported configuration settings: https://github.com/mendersoftware/deployments/blob/master/config.yaml
     # Set in order, last value for the key will be used in case duplications.

mender/templates/device-auth/_podtemplate.yaml

@@ -120,6 +120,7 @@ spec:
     - name: DEVICEAUTH_REDIS_LIMITS_EXPIRE_SEC
       value: {{ .dot.Values.useradm.env.DEVICEAUTH_REDIS_LIMITS_EXPIRE_SEC | default "3600" | quote }}
     {{- end }}
+    {{- include "mender.customEnvs" (merge (deepCopy .dot.Values.device_auth) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
 
     # Supported configuration settings: https://github.com/mendersoftware/deviceauth/blob/master/config.yaml
     # Set in order, last value for the key will be used in case duplications.

mender/templates/deviceconfig/_podtemplate.yaml

@@ -74,6 +74,7 @@ spec:
     # Workflows orchestrator address
     - name: DEVICECONFIG_WORKFLOWS_URL
       value: http://mender-workflows-server:8080/
+    {{- include "mender.customEnvs" (merge (deepCopy .dot.Values.deviceconfig) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
 
     # Supported configuration settings: https://github.com/mendersoftware/deviceconfig/blob/master/config.yaml
     # Set in order, last value for the key will be used in case duplications.

mender/templates/deviceconnect/_podtemplate.yaml

@@ -90,6 +90,8 @@ spec:
         {{.hostname}} https://{{.hostname}} wss://{{.hostname}}
         {{- end }}
 
+    {{- include "mender.customEnvs" (merge (deepCopy .dot.Values.deviceconnect) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
+
     # Supported configuration settings: https://github.com/mendersoftware/deviceconnect/blob/master/config.yaml
     # Set in order, last value for the key will be used in case duplications.
     envFrom:

mender/templates/devicemonitor/_podtemplate.yaml

@@ -73,6 +73,8 @@ spec:
     - name: DEVICEMONITOR_WORKFLOWS_URL
       value: {{ .dot.Values.device_auth.env.DEVICEMONITOR_WORKFLOWS_URL | quote }}
 
+    {{- include "mender.customEnvs" (merge (deepCopy .dot.Values.devicemonitor) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
+
     # Supported configuration settings: https://github.com/mendersoftware/devicemonitor/blob/master/config.yaml
     # Set in order, last value for the key will be used in case duplications.
     envFrom:

mender/templates/generate-delta-worker/deployment.yaml

@@ -73,6 +73,7 @@ spec:
           value: "1"
         - name: GENERATE_DELTA_DEPLOYMENTS_URL
           value: http://{{ .Values.deployments.service.name }}:{{ .Values.deployments.service.port }}
+        {{- include "mender.customEnvs" (merge (deepCopy .Values.generate_delta_worker) (deepCopy (default (dict) .Values.default))) | nindent 8 }}
         envFrom:
         - prefix: WORKFLOWS_
           secretRef:

mender/templates/gui/deployment.yaml

@@ -97,6 +97,7 @@ spec:
         - name: HAVE_MONITOR
           value: "true"
         {{- end }}
+        {{- include "mender.customEnvs" (merge (deepCopy .Values.gui) (deepCopy (default (dict) .Values.default))) | nindent 8 }}
 
 {{- if .Values.global.image.username }}
       imagePullSecrets:

mender/templates/inventory/_podtemplate.yaml

@@ -83,6 +83,7 @@ spec:
       value: {{ .dot.Values.useradm.env.INVENTORY_REDIS_CACHE_EXPIRE_SEC | default "1800" | quote }}
     {{- end }}
     {{- end }}
+    {{- include "mender.customEnvs" (merge (deepCopy .dot.Values.inventory) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
 
     # Supported configuration settings: https://github.com/mendersoftware/inventory-enterprise/blob/master/config.yaml
     # Set in order, last value for the key will be used in case duplications.

mender/templates/iot-manager/_podtemplate.yaml

@@ -66,6 +66,8 @@ spec:
 
     env:
     # Supported configuration settings: https://github.com/mendersoftware/iot-manager/blob/master/config.yaml
+    {{- include "mender.customEnvs" (merge (deepCopy .dot.Values.iot_manager) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
+
     # Set in order, last value for the key will be used in case duplications.
     envFrom:
     - prefix: IOT_MANAGER_

mender/templates/tenantadm/_podtemplate.yaml

@@ -82,6 +82,7 @@ spec:
     - name: TENANTADM_ENABLE_SELF_SERVICE_SIGN_UP
       value: {{ .dot.Values.global.hosted | quote }}
     {{- end }}
+    {{- include "mender.customEnvs" (merge (deepCopy .dot.Values.tenantadm) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
 
     # Supported configuration settings: https://github.com/mendersoftware/tenantadm/blob/master/config.yaml
     # Set in order, last value for the key will be used in case duplications.

mender/templates/useradm/_podtemplate.yaml

@@ -124,6 +124,7 @@ spec:
     - name: USERADM_REDIS_LIMITS_EXPIRE_SEC
       value: {{ .dot.Values.useradm.env.USERADM_REDIS_LIMITS_EXPIRE_SEC | default "3600" | quote }}
     {{- end }}
+    {{- include "mender.customEnvs" (merge (deepCopy .dot.Values.useradm) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
 
     # Supported configuration settings: https://github.com/mendersoftware/useradm/blob/master/config.yaml
     # Set in order, last value for the key will be used in case duplications.

mender/templates/workflows/_podtemplate.yaml

@@ -94,6 +94,7 @@ spec:
       value: "true"
     {{- end }}
     {{- end }}
+    {{- include "mender.customEnvs" (merge (deepCopy .dot.Values.workflows) (deepCopy (default (dict) .dot.Values.default))) | nindent 4 }}
 
     # Supported configuration settings: https://github.com/mendersoftware/workflows/blob/master/config.yaml
     # Set in order, last value for the key will be used in case duplications.