From 8e06206f98605079491c05f099244f2c4c93c715 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Tue, 17 Sep 2024 17:47:09 +0300 Subject: [PATCH] Support ReadOnlyRootFilesystem with DSE 6.8 / DSE 6.9 (#702) * If the ReadOnlyRootFilesystem is selected, always force the usage of k8ssandra-client for config building * Add DSE specific cassandra base config path for readOnly * Add different mounts for HCD & DSE * Add buildx options to docker-build * Add annotation to control if k8ssandra-client is used for config building regardless if supported or not and add a /opt/dse/resources/dse/conf for dse.yaml output * Add collectd directory for readOnly * Add unit tests and change the CHANGELOG * Add smoke_test_read_only_fs for DSE 6.8.50 and 6.9.2 also * Fix smoke_test_read_only_fs --- .github/workflows/kindIntegTest.yml | 23 +- CHANGELOG.md | 4 + Makefile | 4 +- .../v1beta1/cassandradatacenter_types.go | 14 + .../v1beta1/cassandradatacenter_types_test.go | 65 +++++ .../construct_podtemplatespec.go | 62 ++++- .../construct_podtemplatespec_test.go | 245 +++++++++++++++++- ..._fs.go => smoke_test_read_only_fs_test.go} | 0 8 files changed, 395 insertions(+), 22 deletions(-) rename tests/smoke_test_read_only_fs/{smoke_test_read_only_fs.go => smoke_test_read_only_fs_test.go} (100%) diff --git a/.github/workflows/kindIntegTest.yml b/.github/workflows/kindIntegTest.yml index 6bf76f49..8595edaa 100644 --- a/.github/workflows/kindIntegTest.yml +++ b/.github/workflows/kindIntegTest.yml @@ -155,7 +155,7 @@ jobs: strategy: matrix: version: - - "4.1.5" + - "4.1.6" integration_test: # Single worker tests: - additional_serviceoptions @@ -198,7 +198,7 @@ jobs: - scale_up - scale_up_stop_resume - seed_selection - - smoke_test_read_only_fs + # - smoke_test_read_only_fs #- config_fql # OSS only - decommission_dc # - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA @@ -233,25 +233,28 @@ jobs: version: - "3.11.17" - "4.0.13" - - "4.1.5" + - "4.1.6" - "6.8.50" - - "6.9.0" + - "6.9.2" - "1.0.0" integration_test: - test_all_the_things + - smoke_test_read_only_fs include: - version: 6.8.50 serverImage: datastax/dse-mgmtapi-6_8:6.8.50-ubi8 # DSE 6.8.50 serverType: dse - integration_test: "test_all_the_things" - - version: 6.9.0 - serverImage: datastax/dse-mgmtapi-6_8:6.9.0-ubi8 # DSE 6.9.0 + - version: 6.9.2 + serverImage: datastax/dse-mgmtapi-6_8:6.9.2-ubi # DSE 6.9.2 serverType: dse - integration_test: "test_all_the_things" - version: 1.0.0 serverImage: datastax/hcd:1.0.0-ubi # HCD 1.0.0 serverType: hcd - integration_test: "test_all_the_things" + exclude: + - version: 3.11.17 + integration_test: "smoke_test_read_only_fs" + - version: 4.0.13 + integration_test: "smoke_test_read_only_fs" fail-fast: true runs-on: ubuntu-latest env: @@ -282,7 +285,7 @@ jobs: strategy: matrix: version: - - "4.1.5" + - "4.1.6" integration_test: - pvc_expansion fail-fast: true diff --git a/CHANGELOG.md b/CHANGELOG.md index e7ea7f48..5b7215f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti * [BUGFIX] ``` +## unreleased + +* [FEATURE] [#701](https://github.com/k8ssandra/cass-operator/issues/701) Allow ReadOnlyRootFilesystem for DSE also with extra mounts to provide support for cass-config-builder setups + ## v1.22.2 * [BUGFIX] [#703](https://github.com/k8ssandra/cass-operator/issues/703) Fix HCD config path from /etc/cassandra to /opt/hcd/resources/cassandra/conf diff --git a/Makefile b/Makefile index e603f4a0..c67e0d45 100644 --- a/Makefile +++ b/Makefile @@ -86,6 +86,8 @@ endif SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec +BUILDX_OPTIONS ?= + .PHONY: all all: build @@ -160,7 +162,7 @@ run: manifests generate fmt vet ## Run a controller from your host. .PHONY: docker-build docker-build: ## Build docker image with the manager. - docker buildx build --build-arg VERSION=${VERSION} -t ${IMG} . --load + docker buildx build --build-arg VERSION=${VERSION} -t ${IMG} ${BUILDX_OPTIONS} . --load .PHONY: docker-kind docker-kind: docker-build ## Build docker image and load to kind cluster diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types.go b/apis/cassandra/v1beta1/cassandradatacenter_types.go index ec9aa20e..351430b0 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types.go @@ -74,6 +74,9 @@ const ( // AllowStorageChangesAnnotation indicates the CassandraDatacenter StorageConfig can be modified for existing datacenters AllowStorageChangesAnnotation = "cassandra.datastax.com/allow-storage-changes" + // UseClientBuilderAnnotation enforces the usage of new config builder from k8ssandra-client for versions that would otherwise use the cass-config-builder + UseClientBuilderAnnotation = "cassandra.datastax.com/use-new-config-builder" + AllowUpdateAlways AllowUpdateType = "always" AllowUpdateOnce AllowUpdateType = "once" @@ -985,6 +988,10 @@ func (dc *CassandraDatacenter) DatacenterName() string { } func (dc *CassandraDatacenter) UseClientImage() bool { + if metav1.HasAnnotation(dc.ObjectMeta, UseClientBuilderAnnotation) && dc.Annotations[UseClientBuilderAnnotation] == "true" { + return true + } + if dc.Spec.ServerType == "hcd" { return true } @@ -998,3 +1005,10 @@ func (dc *CassandraDatacenter) UseClientImage() bool { func (dc *CassandraDatacenter) GenerationChanged() bool { return dc.Status.ObservedGeneration < dc.Generation } + +func (dc *CassandraDatacenter) ReadOnlyFs() bool { + if dc.Spec.ReadOnlyRootFilesystem != nil { + return *dc.Spec.ReadOnlyRootFilesystem + } + return false +} diff --git a/apis/cassandra/v1beta1/cassandradatacenter_types_test.go b/apis/cassandra/v1beta1/cassandradatacenter_types_test.go index 266897c2..ed339317 100644 --- a/apis/cassandra/v1beta1/cassandradatacenter_types_test.go +++ b/apis/cassandra/v1beta1/cassandradatacenter_types_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" ) var internodeEnabledAll = ` @@ -125,3 +127,66 @@ func TestUseClientImage(t *testing.T) { } } } + +func TestUseClientImageEnforce(t *testing.T) { + assert := assert.New(t) + + tests := []struct { + serverType string + version string + }{ + { + serverType: "cassandra", + version: "4.1.0", + }, + { + serverType: "cassandra", + version: "4.1.2", + }, + { + serverType: "cassandra", + version: "5.0.0", + }, + { + serverType: "cassandra", + version: "3.11.17", + }, + { + serverType: "cassandra", + version: "4.0.8", + }, + { + serverType: "dse", + version: "6.8.39", + }, + { + serverType: "dse", + version: "6.9.0", + }, + { + serverType: "hcd", + version: "1.0.0", + }, + { + serverType: "dse", + version: "4.1.2", + }, + } + + for _, tt := range tests { + dc := CassandraDatacenter{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + UseClientBuilderAnnotation: "true", + }, + }, + Spec: CassandraDatacenterSpec{ + ServerVersion: tt.version, + ServerType: tt.serverType, + ReadOnlyRootFilesystem: ptr.To[bool](true), + }, + } + + assert.True(dc.UseClientImage()) + } +} diff --git a/pkg/reconciliation/construct_podtemplatespec.go b/pkg/reconciliation/construct_podtemplatespec.go index cbc4426c..a581d5db 100644 --- a/pkg/reconciliation/construct_podtemplatespec.go +++ b/pkg/reconciliation/construct_podtemplatespec.go @@ -307,7 +307,7 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe volumeDefaults := []corev1.Volume{vServerConfig, vServerLogs} - if readOnlyFs(dc) { + if dc.ReadOnlyFs() { tmp := corev1.Volume{ Name: "tmp", VolumeSource: corev1.VolumeSource{ @@ -323,6 +323,34 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe } volumeDefaults = append(volumeDefaults, tmp, etcCass) + + if dc.Spec.ServerType == "dse" { + dseConf := corev1.Volume{ + Name: "dse-conf", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + volumeDefaults = append(volumeDefaults, dseConf) + + if !dc.UseClientImage() { + sparkConf := corev1.Volume{ + Name: "spark-conf", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + + collectDConf := corev1.Volume{ + Name: "collectd-conf", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + + volumeDefaults = append(volumeDefaults, sparkConf, collectDConf) + } + } } if dc.UseClientImage() { @@ -476,6 +504,8 @@ func buildInitContainers(dc *api.CassandraDatacenter, rackName string, baseTempl configContainer.Command = []string{"/bin/sh"} if dc.Spec.ServerType == "cassandra" { configContainer.Args = []string{"-c", "cp -rf /etc/cassandra/* /cassandra-base-config/"} + } else if dc.Spec.ServerType == "dse" { + configContainer.Args = []string{"-c", "cp -rf /opt/dse/resources/cassandra/conf/* /cassandra-base-config/"} } else if dc.Spec.ServerType == "hcd" { configContainer.Args = []string{"-c", "cp -rf /opt/hcd/resources/cassandra/conf/* /cassandra-base-config/"} } @@ -649,7 +679,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } } - if readOnlyFs(dc) { + if dc.ReadOnlyFs() { cassContainer.SecurityContext = &corev1.SecurityContext{ ReadOnlyRootFilesystem: ptr.To[bool](true), } @@ -680,7 +710,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla envDefaults = append(envDefaults, corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"}) } - if readOnlyFs(dc) { + if dc.ReadOnlyFs() { envDefaults = append(envDefaults, corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"}) } @@ -737,7 +767,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla } } - if readOnlyFs(dc) { + if dc.ReadOnlyFs() { cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ Name: "tmp", MountPath: "/tmp", @@ -748,6 +778,26 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla Name: "etc-cassandra", MountPath: "/opt/hcd/resources/cassandra/conf", }) + } else if dc.Spec.ServerType == "dse" { + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "etc-cassandra", + MountPath: "/opt/dse/resources/cassandra/conf", + }) + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "dse-conf", + MountPath: "/opt/dse/resources/dse/conf", + }) + + if !dc.UseClientImage() { + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "spark-conf", + MountPath: "/opt/dse/resources/spark/conf", + }) + cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ + Name: "collectd-conf", + MountPath: "/opt/dse/resources/dse/collectd/etc/collectd", + }) + } } else { cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{ Name: "etc-cassandra", @@ -813,10 +863,6 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla return nil } -func readOnlyFs(dc *api.CassandraDatacenter) bool { - return dc.Spec.ReadOnlyRootFilesystem != nil && *dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage() -} - func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) { baseTemplate := dc.Spec.PodTemplateSpec.DeepCopy() diff --git a/pkg/reconciliation/construct_podtemplatespec_test.go b/pkg/reconciliation/construct_podtemplatespec_test.go index 98db5f8a..7335d675 100644 --- a/pkg/reconciliation/construct_podtemplatespec_test.go +++ b/pkg/reconciliation/construct_podtemplatespec_test.go @@ -1970,8 +1970,38 @@ func TestReadOnlyRootFilesystemVolumeChanges(t *testing.T) { podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) assert.NoError(err, "failed to build PodTemplateSpec") + initContainers := podTemplateSpec.Spec.InitContainers + assert.NotNil(initContainers, "No init containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(initContainers, 2, "Unexpected number of init containers returned") + assert.Equal(ServerBaseConfigContainerName, initContainers[0].Name) + assert.Equal(ServerConfigContainerName, initContainers[1].Name) + + assert.True(reflect.DeepEqual(initContainers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + + assert.Equal(initContainers[0].Args[1], "cp -rf /etc/cassandra/* /cassandra-base-config/") + + assert.True(reflect.DeepEqual(initContainers[1].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config", + MountPath: "/config", + }, + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + containers := podTemplateSpec.Spec.Containers - assert.NotNil(containers, "Unexpected containers containers received") + assert.NotNil(containers, "No containers were found") assert.NoError(err, "Unexpected error encountered") assert.Len(containers, 2, "Unexpected number of containers containers returned") @@ -2026,8 +2056,38 @@ func TestReadOnlyRootFilesystemVolumeChangesHCD(t *testing.T) { podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) assert.NoError(err, "failed to build PodTemplateSpec") + initContainers := podTemplateSpec.Spec.InitContainers + assert.NotNil(initContainers, "No init containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(initContainers, 2, "Unexpected number of init containers returned") + assert.Equal(ServerBaseConfigContainerName, initContainers[0].Name) + assert.Equal(ServerConfigContainerName, initContainers[1].Name) + + assert.True(reflect.DeepEqual(initContainers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + + assert.Equal(initContainers[0].Args[1], "cp -rf /opt/hcd/resources/cassandra/conf/* /cassandra-base-config/") + + assert.True(reflect.DeepEqual(initContainers[1].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config", + MountPath: "/config", + }, + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + containers := podTemplateSpec.Spec.Containers - assert.NotNil(containers, "Unexpected containers containers received") + assert.NotNil(containers, "No containers were found") assert.NoError(err, "Unexpected error encountered") assert.Len(containers, 2, "Unexpected number of containers containers returned") @@ -2058,7 +2118,186 @@ func TestReadOnlyRootFilesystemVolumeChangesHCD(t *testing.T) { }, }), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts)) - // TODO Verify MCAC is disabled since it will fail with ReadOnlyRootFilesystem + mcacDisabled := corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"} + assert.True(envVarsContains(containers[0].Env, mcacDisabled)) +} + +func TestReadOnlyRootFilesystemVolumeChangesDSE(t *testing.T) { + assert := assert.New(t) + dc := &api.CassandraDatacenter{ + Spec: api.CassandraDatacenterSpec{ + ClusterName: "bob", + ServerType: "dse", + ServerVersion: "6.9.2", + ReadOnlyRootFilesystem: ptr.To[bool](true), + Racks: []api.Rack{ + { + Name: "r1", + }, + }, + }, + } + + podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) + assert.NoError(err, "failed to build PodTemplateSpec") + + initContainers := podTemplateSpec.Spec.InitContainers + assert.NotNil(initContainers, "No init containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(initContainers, 1, "Unexpected number of init containers returned") + assert.Equal(ServerConfigContainerName, initContainers[0].Name) + + assert.True(reflect.DeepEqual(initContainers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config", + MountPath: "/config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + + containers := podTemplateSpec.Spec.Containers + assert.NotNil(containers, "No containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(containers, 2, "Unexpected number of containers containers returned") + assert.Equal("cassandra", containers[0].Name) + assert.Equal(ptr.To[bool](true), containers[0].SecurityContext.ReadOnlyRootFilesystem) + + assert.True(reflect.DeepEqual(containers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "tmp", + MountPath: "/tmp", + }, + { + Name: "etc-cassandra", + MountPath: "/opt/dse/resources/cassandra/conf", + }, + { + Name: "dse-conf", + MountPath: "/opt/dse/resources/dse/conf", + }, + { + Name: "spark-conf", + MountPath: "/opt/dse/resources/spark/conf", + }, + { + Name: "collectd-conf", + MountPath: "/opt/dse/resources/dse/collectd/etc/collectd", + }, + { + Name: "server-logs", + MountPath: "/var/log/cassandra", + }, + { + Name: "server-data", + MountPath: "/var/lib/cassandra", + }, + { + Name: "server-config", + MountPath: "/config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts)) + + // Test that cassandra-base-config isn't here + + mcacDisabled := corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"} + assert.True(envVarsContains(containers[0].Env, mcacDisabled)) +} + +func TestReadOnlyRootFilesystemVolumeChangesDSEWithClient(t *testing.T) { + assert := assert.New(t) + dc := &api.CassandraDatacenter{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + api.UseClientBuilderAnnotation: "true", + }, + }, + Spec: api.CassandraDatacenterSpec{ + ClusterName: "bob", + ServerType: "dse", + ServerVersion: "6.9.2", + ReadOnlyRootFilesystem: ptr.To[bool](true), + Racks: []api.Rack{ + { + Name: "r1", + }, + }, + }, + } + + podTemplateSpec, err := buildPodTemplateSpec(dc, dc.Spec.Racks[0], false) + assert.NoError(err, "failed to build PodTemplateSpec") + + initContainers := podTemplateSpec.Spec.InitContainers + assert.NotNil(initContainers, "No init containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(initContainers, 2, "Unexpected number of init containers returned") + assert.Equal(ServerBaseConfigContainerName, initContainers[0].Name) + assert.Equal(ServerConfigContainerName, initContainers[1].Name) + + assert.True(reflect.DeepEqual(initContainers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + + assert.Equal(initContainers[0].Args[1], "cp -rf /opt/dse/resources/cassandra/conf/* /cassandra-base-config/") + + assert.True(reflect.DeepEqual(initContainers[1].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "server-config", + MountPath: "/config", + }, + { + Name: "server-config-base", + MountPath: "/cassandra-base-config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the base config container: %v", initContainers[0].VolumeMounts)) + + containers := podTemplateSpec.Spec.Containers + assert.NotNil(containers, "No containers were found") + assert.NoError(err, "Unexpected error encountered") + + assert.Len(containers, 2, "Unexpected number of containers containers returned") + assert.Equal("cassandra", containers[0].Name) + assert.Equal(ptr.To[bool](true), containers[0].SecurityContext.ReadOnlyRootFilesystem) + + assert.True(reflect.DeepEqual(containers[0].VolumeMounts, + []corev1.VolumeMount{ + { + Name: "tmp", + MountPath: "/tmp", + }, + { + Name: "etc-cassandra", + MountPath: "/opt/dse/resources/cassandra/conf", + }, + { + Name: "dse-conf", + MountPath: "/opt/dse/resources/dse/conf", + }, + { + Name: "server-logs", + MountPath: "/var/log/cassandra", + }, + { + Name: "server-data", + MountPath: "/var/lib/cassandra", + }, + { + Name: "server-config", + MountPath: "/config", + }, + }), fmt.Sprintf("Unexpected volume mounts for the cassandra container: %v", containers[0].VolumeMounts)) + + // Test that cassandra-base-config isn't here + mcacDisabled := corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"} assert.True(envVarsContains(containers[0].Env, mcacDisabled)) } diff --git a/tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go b/tests/smoke_test_read_only_fs/smoke_test_read_only_fs_test.go similarity index 100% rename from tests/smoke_test_read_only_fs/smoke_test_read_only_fs.go rename to tests/smoke_test_read_only_fs/smoke_test_read_only_fs_test.go