From 7c7c48cdc386091225faac26c0589c59ab144a31 Mon Sep 17 00:00:00 2001 From: kayrus Date: Wed, 18 Dec 2024 11:13:08 +0100 Subject: [PATCH 1/2] [ceph] prepare upgrade to rook v1.16 --- system/cc-ceph/Chart.lock | 8 ++++---- system/cc-ceph/Chart.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/system/cc-ceph/Chart.lock b/system/cc-ceph/Chart.lock index 0731771e991..b3d632ad9b9 100644 --- a/system/cc-ceph/Chart.lock +++ b/system/cc-ceph/Chart.lock @@ -4,9 +4,9 @@ dependencies: version: 1.0.0 - name: rook-ceph repository: https://charts.rook.io/release - version: v1.15.0 + version: v1.16.0 - name: rook-crds repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm - version: 0.0.2-rook.1.15.0 -digest: sha256:d8d894ba706ae4a8216ecf28bb9561fafc3b88908f183ba0b6d4b77f296e92f8 -generated: "2024-09-12T10:38:14.973622-04:00" + version: 0.0.2-rook.1.16.0 +digest: sha256:d9a8ec1509dcec3a634aada46cbf9434897bfa7bf85bad9adb8c6af157aa08d4 +generated: "2024-12-18T11:34:58.26072907+01:00" diff --git a/system/cc-ceph/Chart.yaml b/system/cc-ceph/Chart.yaml index 79568fb7d38..8cf0b10c477 100644 --- a/system/cc-ceph/Chart.yaml +++ b/system/cc-ceph/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: cc-ceph description: A Helm chart for the Rook / Ceph Objects inside the Storage Clusters type: application -version: 1.1.1 -appVersion: "1.15.0" +version: 1.1.2 +appVersion: "1.16.0" dependencies: - name: owner-info repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm @@ -11,7 +11,7 @@ dependencies: - name: rook-ceph # version update should be done in the rook-crds chart as well repository: https://charts.rook.io/release - version: 1.15.0 + version: 1.16.0 - name: rook-crds repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm - version: '0.0.2-rook.1.15.0' + version: '0.0.2-rook.1.16.0' From df0fee26f92eedba3593142a27b70af7a75d123c Mon Sep 17 00:00:00 2001 From: Artem Date: Thu, 19 Dec 2024 15:54:24 +0100 Subject: [PATCH 2/2] [ceph] support multi-instance rgw (#7578) * support multi-instance rgw --- .../templates/cephobjectstore-extra.yaml | 110 ++++++++++++++++++ .../cephobjectstore-placement-pools.yaml | 2 - system/cc-ceph/templates/cephobjectstore.yaml | 18 +++ .../cc-ceph/templates/certificate-extra.yaml | 25 ++++ system/cc-ceph/templates/record-extra.yaml | 26 +++++ system/cc-ceph/templates/service-extra.yaml | 25 ++++ system/cc-ceph/values.yaml | 28 ++++- 7 files changed, 231 insertions(+), 3 deletions(-) create mode 100644 system/cc-ceph/templates/cephobjectstore-extra.yaml create mode 100644 system/cc-ceph/templates/certificate-extra.yaml create mode 100644 system/cc-ceph/templates/record-extra.yaml create mode 100644 system/cc-ceph/templates/service-extra.yaml diff --git a/system/cc-ceph/templates/cephobjectstore-extra.yaml b/system/cc-ceph/templates/cephobjectstore-extra.yaml new file mode 100644 index 00000000000..3772a5ad81a --- /dev/null +++ b/system/cc-ceph/templates/cephobjectstore-extra.yaml @@ -0,0 +1,110 @@ +{{- if .Values.objectstore.multiInstance.enabled }} +apiVersion: ceph.rook.io/v1 +kind: CephObjectRealm +metadata: + name: {{ .Values.objectstore.name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectZoneGroup +metadata: + name: {{ .Values.objectstore.name }} + namespace: {{ .Release.Namespace }} +spec: + realm: {{ .Values.objectstore.name }} +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectZone +metadata: + name: {{ .Values.objectstore.name }} + namespace: {{ .Release.Namespace }} +spec: + zoneGroup: {{ .Values.objectstore.name }} +{{- if and .Values.rgwTargetPlacements.useRookCRD .Values.rgwTargetPlacements.placements }} + sharedPools: + poolPlacements: +{{- range $target := .Values.rgwTargetPlacements.placements }} + - name: {{ $target.name }} + metadataPoolName: {{ $target.name }}.rgw.buckets.index + dataPoolName: {{ $target.name }}.rgw.buckets.data + dataNonECPoolName: {{ $target.name }}.rgw.buckets.non-ec + default: {{ $target.default | default false }} +{{- end }} +{{- else }} + metadataPool: {{ toYaml .Values.objectstore.metadataPool | nindent 4 }} + dataPool: {{ toYaml .Values.objectstore.dataPool | nindent 4 }} +{{- end }} +{{- range $instance := .Values.objectstore.multiInstance.extraInstances }} +--- +apiVersion: ceph.rook.io/v1 +kind: CephObjectStore +metadata: + name: {{ $instance.name }} + namespace: {{ $.Release.Namespace }} +spec: + zone: + name: {{ $.Values.objectstore.name }} + hosting: +{{- if gt (len $instance.gateway.dnsNames) 0 }} + advertiseEndpoint: + dnsName: {{ $instance.gateway.dnsNames | first }} + port: 443 + useTls: true + dnsNames: {{ toYaml $instance.gateway.dnsNames | nindent 8 }} +{{- end }} + gateway: + instances: {{ $instance.gateway.instances | default $.Values.objectstore.gateway.instances }} + {{- if or $instance.gateway.port $.Values.objectstore.gateway.port }} + port: {{ $instance.gateway.port | default $.Values.objectstore.gateway.port }} + {{- end }} + {{- if or $instance.gateway.securePort $.Values.objectstore.gateway.securePort }} + securePort: {{ $instance.gateway.securePort | default $.Values.objectstore.gateway.securePort }} + {{- end }} + placement: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.metal.cloud.sap/role + operator: In + values: + - {{ $.Values.osd.nodeRole }} + # since the CephCluster's network provider is "host", we need to isolate 80/443 port listeners from each other + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - rook-ceph-rgw + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + sslCertificateRef: {{ $instance.gateway.sslCertificateRef | default $.Values.objectstore.gateway.sslCertificateRef }} + resources: {{ toYaml ( $instance.gateway.resources | default $.Values.objectstore.gateway.resources) | nindent 6 }} + preservePoolsOnDelete: true +{{- if and $.Values.objectstore.keystone.enabled }} +{{- with $.Values.objectstore.keystone }} + auth: + keystone: + acceptedRoles: +{{- range $_, $role := .accepted_roles }} + - {{ $role }} +{{- end }} + implicitTenants: {{ .implicit_tenants | quote }} + serviceUserSecretName: ceph-keystone-secret + tokenCacheSize: {{ .token_cache_size }} + url: {{ .url }} + protocols: +{{- if $instance.enabledAPIs }} + enableAPIs: {{ toYaml $instance.enabledAPIs | nindent 6 }} +{{- end }} + s3: + authUseKeystone: true + swift: + accountInUrl: {{ .swift_account_in_url }} + versioningEnabled: {{ .swift_versioning_enabled }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/system/cc-ceph/templates/cephobjectstore-placement-pools.yaml b/system/cc-ceph/templates/cephobjectstore-placement-pools.yaml index 44f18cf52c5..5eba4595b61 100644 --- a/system/cc-ceph/templates/cephobjectstore-placement-pools.yaml +++ b/system/cc-ceph/templates/cephobjectstore-placement-pools.yaml @@ -1,4 +1,3 @@ -{{- if .Values.rgwTargetPlacements.enabled }} {{- range $target := .Values.rgwTargetPlacements.placements }} --- apiVersion: ceph.rook.io/v1 @@ -80,7 +79,6 @@ spec: nodelete: {{ $.Values.pool.nodelete | quote }} nosizechange: {{ $.Values.pool.nosizechange | quote }} {{- end }} -{{- end }} {{- if .Values.rgwTargetPlacements.premiumPlacements }} {{- range $target := .Values.rgwTargetPlacements.premiumPlacements }} --- diff --git a/system/cc-ceph/templates/cephobjectstore.yaml b/system/cc-ceph/templates/cephobjectstore.yaml index a17a95da7c9..8f8b729dfa4 100644 --- a/system/cc-ceph/templates/cephobjectstore.yaml +++ b/system/cc-ceph/templates/cephobjectstore.yaml @@ -4,8 +4,23 @@ metadata: name: {{ .Values.objectstore.name }} namespace: {{ .Release.Namespace }} spec: +{{- if .Values.objectstore.multiInstance.enabled }} + zone: + name: {{ .Values.objectstore.name }} +{{- else if and .Values.rgwTargetPlacements.useRookCRD .Values.rgwTargetPlacements.placements }} + sharedPools: + poolPlacements: +{{- range $target := .Values.rgwTargetPlacements.placements }} + - name: {{ $target.name }} + metadataPoolName: {{ $target.name }}.rgw.buckets.index + dataPoolName: {{ $target.name }}.rgw.buckets.data + dataNonECPoolName: {{ $target.name }}.rgw.buckets.non-ec + default: {{ $target.default | default false }} +{{- end }} +{{- else }} metadataPool: {{ toYaml .Values.objectstore.metadataPool | nindent 4 }} dataPool: {{ toYaml .Values.objectstore.dataPool | nindent 4 }} +{{- end }} hosting: {{- if gt (len .Values.objectstore.gateway.dnsNames) 0 }} advertiseEndpoint: @@ -60,6 +75,9 @@ spec: tokenCacheSize: {{ .token_cache_size }} url: {{ .url }} protocols: +{{- if $.Values.objectstore.enabledAPIs }} + enableAPIs: {{ toYaml $.Values.objectstore.enabledAPIs | nindent 6 }} +{{- end }} s3: authUseKeystone: true swift: diff --git a/system/cc-ceph/templates/certificate-extra.yaml b/system/cc-ceph/templates/certificate-extra.yaml new file mode 100644 index 00000000000..852c5168623 --- /dev/null +++ b/system/cc-ceph/templates/certificate-extra.yaml @@ -0,0 +1,25 @@ +{{- if .Values.objectstore.multiInstance.enabled }} +{{- range $instance := .Values.objectstore.multiInstance.extraInstances }} +{{- range $key, $record := $instance.gateway.dnsNames }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ $record }} +spec: + dnsNames: + - "*.{{ $record }}" + - "{{ $record }}" + uris: + - rook-ceph-rgw-{{ $instance.name }}.rook-ceph.svc + issuerRef: + group: certmanager.cloud.sap + kind: ClusterIssuer + name: digicert-issuer + secretName: {{ $instance.gateway.sslCertificateRef }} + usages: + - digital signature + - key encipherment +{{- end }} +{{- end }} +{{- end }} diff --git a/system/cc-ceph/templates/record-extra.yaml b/system/cc-ceph/templates/record-extra.yaml new file mode 100644 index 00000000000..ad2c07d5c81 --- /dev/null +++ b/system/cc-ceph/templates/record-extra.yaml @@ -0,0 +1,26 @@ +{{- if .Values.objectstore.multiInstance.enabled }} +{{- range $instance := .Values.objectstore.multiInstance.extraInstances }} +{{- range $key, $record := $instance.gateway.dnsNames }} +--- +apiVersion: disco.stable.sap.cc/v1 +kind: Record +metadata: + name: "{{ $record }}" +spec: + type: A + record: {{ $instance.service.externalIP }} + hosts: + - "{{ $record }}." +--- +apiVersion: disco.stable.sap.cc/v1 +kind: Record +metadata: + name: "{{ $record }}-wildcard" +spec: + type: CNAME + record: "{{ $record }}." + hosts: + - "*.{{ $record }}." +{{- end }} +{{- end }} +{{- end }} diff --git a/system/cc-ceph/templates/service-extra.yaml b/system/cc-ceph/templates/service-extra.yaml new file mode 100644 index 00000000000..3259f479bfc --- /dev/null +++ b/system/cc-ceph/templates/service-extra.yaml @@ -0,0 +1,25 @@ +{{- if .Values.objectstore.multiInstance.enabled }} +{{- range $instance := .Values.objectstore.multiInstance.extraInstances }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $instance.service.name }} + namespace: {{ $.Release.Namespace }} +spec: + externalIPs: + - {{ $instance.service.externalIP }} + type: NodePort + sessionAffinity: None + externalTrafficPolicy: Local + ports: + - port: {{ $instance.service.port }} + targetPort: {{ $instance.service.port }} + protocol: TCP + name: rgw-ssl + selector: + app: {{ $instance.service.selector.app }} + rook_cluster: {{ $instance.service.selector.rook_cluster }} + rook_object_store: {{ $instance.name }} +{{- end }} +{{- end }} diff --git a/system/cc-ceph/values.yaml b/system/cc-ceph/values.yaml index 4fb53bd0a44..c03a65f36bf 100644 --- a/system/cc-ceph/values.yaml +++ b/system/cc-ceph/values.yaml @@ -90,6 +90,7 @@ dashboard: objectstore: enabled: true name: objectstore + enabledAPIs: [] # empty - all enabled. See: https://docs.ceph.com/en/reef/radosgw/config-ref/#confval-rgw_enable_apis gateway: instances: 6 port: 80 @@ -149,6 +150,30 @@ objectstore: password: XXX domain: XXX project: XXX + multiInstance: + enabled: false + extraInstances: + # can inherit/override all config options from objectstore: + # - name: objectstore-admin + # gateway: + # instances: 2 + # sslCertificateRef: "" + # dnsNames: + # - dns1-adm + # - dns2-adm + # resources: + # requests: + # cpu: 1 + # memory: 2Gi + # service: + # name: ceph-objectstore-admin-external + # port: 443 + # externalIP: "10.0.0.1" + # selector: + # app: rgw + # define other RGW instances here: + # - name: other-instance-name + prysm: enabled: true repository: @@ -158,7 +183,8 @@ objectstore: interval: "10" rgwTargetPlacements: - enabled: false + # enabled: false deprecate rgwTargetPlacements.enabled because it is true on all envs + useRookCRD: false # !!!WARNING set 'true' only for new clusters. Upgrade will not work now. defaultRgwPools: enabled: false # create default rgw pools, see: https://github.com/sapcc/helm-charts/issues/6670