Support local registry caches (#222)

Co-authored-by: Erkan Erol <[email protected]>
giantswarm · Jun 25, 2024 · 3f40036 · 3f40036
1 parent 3bca124
commit 3f40036
Show file tree

Hide file tree

Showing 6 changed files with 129 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,11 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add `.global.connectivity.localRegistryCache` Helm values and support for in-cluster, local registry cache mirrors in containerd configuration.
+  In such cases, the registry should be exposed via node ports and containerd connects via that port at 127.0.0.1 via HTTP (only allowed for this single use case).
+
+### Fixed
+
+- Fixed `containerd` config file generation when multiple registries are set with authentication
+
 ### Removed
 
 - Stop setting `defaultPolicies.enabled=true` in `cilium-app` when `internal.ciliumNetworkPolicy.enabled=true` after all clusters are migrated.
 - Stop setting `extraPolicies.remove=true` in `cilium-app` after all clusters are migrated.
 
+
 ## [0.53.1] - 2024-06-09
 
 ### Fixed

diff --git a/helm/cluster-vsphere/README.md b/helm/cluster-vsphere/README.md
@@ -56,6 +56,11 @@ Configurations related to cluster connectivity such as container registries.
 | `global.connectivity.containerRegistries.*[*].credentials.password` | **Password** - Used to authenticate for the registry with username/password.|**Type:** `string`<br/>|
 | `global.connectivity.containerRegistries.*[*].credentials.username` | **Username** - Used to authenticate for the registry with username/password.|**Type:** `string`<br/>|
 | `global.connectivity.containerRegistries.*[*].endpoint` | **Endpoint** - Endpoint for the container registry.|**Type:** `string`<br/>|
+| `global.connectivity.localRegistryCache` | **Local registry cache** - Caching container registry within the cluster.|**Type:** `object`<br/>|
+| `global.connectivity.localRegistryCache.enabled` | **Enable** - Enabling this will deploy the Zot registry service in the cluster. To make use of it as a pull-through cache, you also have to specify registries to cache images for.|**Type:** `boolean`<br/>**Default:** `false`|
+| `global.connectivity.localRegistryCache.mirroredRegistries` | **Registries to cache** - Here you must specify each registry to cache container images for. Please also make sure to have an entry for each registry in Global > Components > Containerd > Container registries.|**Type:** `array`<br/>**Default:** `[]`|
+| `global.connectivity.localRegistryCache.mirroredRegistries[*]` |**None**|**Type:** `string`<br/>|
+| `global.connectivity.localRegistryCache.port` | **Service port** - NodePort used by the local registry service.|**Type:** `integer`<br/>**Default:** `32767`|
 | `global.connectivity.network` | **Network**|**Type:** `object`<br/>|
 | `global.connectivity.network.controlPlaneEndpoint` | **Endpoint** - Kubernetes API configuration.|**Type:** `object`<br/>|
 | `global.connectivity.network.controlPlaneEndpoint.host` | **Host** - IP for access to the Kubernetes API. Manually select an IP for kube API. Empty string for auto selection from the ipPoolName pool.|**Type:** `string`<br/>|

diff --git a/helm/cluster-vsphere/ci/test-multiple-mirrors-with-auth-and-local-caching.yaml b/helm/cluster-vsphere/ci/test-multiple-mirrors-with-auth-and-local-caching.yaml
@@ -0,0 +1,69 @@
+global:
+  metadata:
+    description: "test cluster"
+    organization: "giantswarm"
+  connectivity:
+    containerRegistries:
+      gsoci.azurecr.io:
+        - endpoint: zot-test-1.golem.gaws.gigantic.io
+          credentials:
+            username: example
+            password: xxxxx
+      gsociprivate.azurecr.io:
+        - endpoint: zot-test-2.golem.gaws.gigantic.io
+          credentials:
+            username: example
+            password: yyyyy
+    localRegistryCache:
+      enabled: true
+      mirroredRegistries:
+        - gsoci.azurecr.io
+        - gsociprivate.azurecr.io
+    network:
+      controlPlaneEndpoint:
+        host: "10.10.222.241"
+        port: 6443
+      loadBalancers:
+        cidrBlocks:
+          - "10.10.222.224/30"
+    baseDomain: k8s.test
+  controlPlane:
+    replicas: 1
+    machineTemplate:
+      cloneMode: "linkedClone"
+      diskGiB: 25
+      numCPUs: 2
+      memoryMiB: 8192
+      resourcePool: "grasshopper"
+      network:
+        devices:
+          - networkName: 'grasshopper-capv'
+            dhcp4: true
+  nodePools:
+    worker:
+      class: "default"
+      replicas: 2
+  providerSpecific:
+    vcenter:
+      server: "https://foo.example.com"
+      username: "vcenter-admin"
+      password: "vcenter-admin-password"
+      datacenter: "Datacenter"
+      datastore: "vsanDatastore"
+      # openssl s_client -connect https://foo.example.com < /dev/null 2>/dev/null | openssl x509 -fingerprint -noout -in /dev/stdin
+      thumbprint: "F7:CF:F9:E5:99:39:FF:C1:D7:14:F1:3F:8A:42:21:95:3B:A1:6E:16"
+      region: "k8s-region"
+      zone: "k8s-zone"
+  nodeClasses:
+    default:
+      cloneMode: "linkedClone"
+      diskGiB: 25
+      numCPUs: 2
+      memoryMiB: 8192
+      resourcePool: "grasshopper"
+      network:
+        devices:
+          - networkName: 'grasshopper-capv'
+            dhcp4: true
+internal:
+  enableEncryptionProvider: false
diff --git a/helm/cluster-vsphere/files/etc/containerd/config.toml b/helm/cluster-vsphere/files/etc/containerd/config.toml
@@ -28,21 +28,24 @@ sandbox_image = "{{ .Values.internal.sandboxContainerImage.registry }}/{{ .Value
   {{- range $host, $config := .Values.global.connectivity.containerRegistries }}
     [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$host}}"]
       endpoint = [
+        {{- if and $.Values.global.connectivity.localRegistryCache.enabled (has $host $.Values.global.connectivity.localRegistryCache.mirroredRegistries) -}}
+        "http://127.0.0.1:{{ $.Values.global.connectivity.localRegistryCache.port }}",
+        {{- end -}}
         {{- range $value := $config -}}
         "https://{{$value.endpoint}}",
         {{- end -}}
-    ]
+      ]
   {{- end }}
 [plugins."io.containerd.grpc.v1.cri".registry.configs]
-  {{ range $host, $config := .Values.global.connectivity.containerRegistries -}}
+  {{- range $host, $config := .Values.global.connectivity.containerRegistries -}}
     {{ range $value := $config -}}
-      {{ with $value.credentials -}}
-      [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$value.endpoint}}".auth]
+      {{- with $value.credentials }}
+    [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$value.endpoint}}".auth]
       {{ if and .username .password -}}
       auth = {{ printf "%s:%s" .username .password | b64enc | quote }}
       {{- else if .auth -}}
       auth = {{ .auth | quote }}
-      {{ else if .identitytoken -}}
+      {{- else if .identitytoken -}}
       identitytoken = {{ .identitytoken  | quote }}
       {{- end }}
     {{- end }}

diff --git a/helm/cluster-vsphere/values.schema.json b/helm/cluster-vsphere/values.schema.json
@@ -185,6 +185,39 @@
                             },
                             "default": {}
                         },
+                        "localRegistryCache": {
+                            "type": "object",
+                            "title": "Local registry cache",
+                            "description": "Caching container registry within the cluster.",
+                            "required": [
+                                "enabled",
+                                "port"
+                            ],
+                            "additionalProperties": false,
+                            "properties": {
+                                "enabled": {
+                                    "type": "boolean",
+                                    "title": "Enable",
+                                    "description": "Enabling this will deploy the Zot registry service in the cluster. To make use of it as a pull-through cache, you also have to specify registries to cache images for.",
+                                    "default": false
+                                },
+                                "mirroredRegistries": {
+                                    "type": "array",
+                                    "title": "Registries to cache",
+                                    "description": "Here you must specify each registry to cache container images for. Please also make sure to have an entry for each registry in Global > Components > Containerd > Container registries.",
+                                    "items": {
+                                        "type": "string"
+                                    },
+                                    "default": []
+                                },
+                                "port": {
+                                    "type": "integer",
+                                    "title": "Service port",
+                                    "description": "NodePort used by the local registry service.",
+                                    "default": 32767
+                                }
+                            }
+                        },
                         "network": {
                             "type": "object",
                             "title": "Network",

diff --git a/helm/cluster-vsphere/values.yaml b/helm/cluster-vsphere/values.yaml
@@ -3,6 +3,10 @@
 global:
   connectivity:
     containerRegistries: {}
+    localRegistryCache:
+      enabled: false
+      mirroredRegistries: []
+      port: 32767
     network:
       controlPlaneEndpoint:
         ipPoolName: wc-cp-ips