Skip to content

Commit

Permalink
Support ReadOnlyRootFilesystem with DSE 6.8 / DSE 6.9 (#702)
Browse files Browse the repository at this point in the history 
* If the ReadOnlyRootFilesystem is selected, always force the usage of k8ssandra-client for config building

* Add DSE specific cassandra base config path for readOnly

* Add different mounts for HCD & DSE

* Add buildx options to docker-build

* Add annotation to control if k8ssandra-client is used for config building regardless if supported or not and add a /opt/dse/resources/dse/conf for dse.yaml output

* Add collectd directory for readOnly

* Add unit tests and change the CHANGELOG

* Add smoke_test_read_only_fs for DSE 6.8.50 and 6.9.2 also

* Fix smoke_test_read_only_fs
  • Loading branch information
@burmanm
burmanm committed Sep 17, 2024
1 parent 87b1923 commit 8e06206
Show file tree
Hide file tree
Showing 8 changed files with 395 additions and 22 deletions.
23 changes: 13 additions & 10 deletions .github/workflows/kindIntegTest.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ jobs:
strategy:
matrix:
version:
- "4.1.5"
- "4.1.6"
integration_test:
# Single worker tests:
- additional_serviceoptions
Expand Down Expand Up @@ -198,7 +198,7 @@ jobs:
- scale_up
- scale_up_stop_resume
- seed_selection
- smoke_test_read_only_fs
# - smoke_test_read_only_fs
#- config_fql # OSS only
- decommission_dc
# - stop_resume_scale_up # Odd insufficient CPU issues in kind+GHA
Expand Down Expand Up @@ -233,25 +233,28 @@ jobs:
version:
- "3.11.17"
- "4.0.13"
- "4.1.5"
- "4.1.6"
- "6.8.50"
- "6.9.0"
- "6.9.2"
- "1.0.0"
integration_test:
- test_all_the_things
- smoke_test_read_only_fs
include:
- version: 6.8.50
serverImage: datastax/dse-mgmtapi-6_8:6.8.50-ubi8 # DSE 6.8.50
serverType: dse
integration_test: "test_all_the_things"
- version: 6.9.0
serverImage: datastax/dse-mgmtapi-6_8:6.9.0-ubi8 # DSE 6.9.0
- version: 6.9.2
serverImage: datastax/dse-mgmtapi-6_8:6.9.2-ubi # DSE 6.9.2
serverType: dse
integration_test: "test_all_the_things"
- version: 1.0.0
serverImage: datastax/hcd:1.0.0-ubi # HCD 1.0.0
serverType: hcd
integration_test: "test_all_the_things"
exclude:
- version: 3.11.17
integration_test: "smoke_test_read_only_fs"
- version: 4.0.13
integration_test: "smoke_test_read_only_fs"
fail-fast: true
runs-on: ubuntu-latest
env:
Expand Down Expand Up @@ -282,7 +285,7 @@ jobs:
strategy:
matrix:
version:
- "4.1.5"
- "4.1.6"
integration_test:
- pvc_expansion
fail-fast: true
Expand Down
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti
* [BUGFIX]
```

## unreleased

* [FEATURE] [#701](https://github.com/k8ssandra/cass-operator/issues/701) Allow ReadOnlyRootFilesystem for DSE also with extra mounts to provide support for cass-config-builder setups

## v1.22.2

* [BUGFIX] [#703](https://github.com/k8ssandra/cass-operator/issues/703) Fix HCD config path from /etc/cassandra to /opt/hcd/resources/cassandra/conf
Expand Down
4 changes: 3 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ endif
SHELL = /usr/bin/env bash -o pipefail
.SHELLFLAGS = -ec

BUILDX_OPTIONS ?=

.PHONY: all
all: build

Expand Down Expand Up @@ -160,7 +162,7 @@ run: manifests generate fmt vet ## Run a controller from your host.

.PHONY: docker-build
docker-build: ## Build docker image with the manager.
docker buildx build --build-arg VERSION=${VERSION} -t ${IMG} . --load
docker buildx build --build-arg VERSION=${VERSION} -t ${IMG} ${BUILDX_OPTIONS} . --load

.PHONY: docker-kind
docker-kind: docker-build ## Build docker image and load to kind cluster
Expand Down
14 changes: 14 additions & 0 deletions apis/cassandra/v1beta1/cassandradatacenter_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ const (
// AllowStorageChangesAnnotation indicates the CassandraDatacenter StorageConfig can be modified for existing datacenters
AllowStorageChangesAnnotation = "cassandra.datastax.com/allow-storage-changes"

// UseClientBuilderAnnotation enforces the usage of new config builder from k8ssandra-client for versions that would otherwise use the cass-config-builder
UseClientBuilderAnnotation = "cassandra.datastax.com/use-new-config-builder"

AllowUpdateAlways AllowUpdateType = "always"
AllowUpdateOnce AllowUpdateType = "once"

Expand Down Expand Up @@ -985,6 +988,10 @@ func (dc *CassandraDatacenter) DatacenterName() string {
}

func (dc *CassandraDatacenter) UseClientImage() bool {
if metav1.HasAnnotation(dc.ObjectMeta, UseClientBuilderAnnotation) && dc.Annotations[UseClientBuilderAnnotation] == "true" {
return true
}

if dc.Spec.ServerType == "hcd" {
return true
}
Expand All @@ -998,3 +1005,10 @@ func (dc *CassandraDatacenter) UseClientImage() bool {
func (dc *CassandraDatacenter) GenerationChanged() bool {
return dc.Status.ObservedGeneration < dc.Generation
}

func (dc *CassandraDatacenter) ReadOnlyFs() bool {
if dc.Spec.ReadOnlyRootFilesystem != nil {
return *dc.Spec.ReadOnlyRootFilesystem
}
return false
}
65 changes: 65 additions & 0 deletions apis/cassandra/v1beta1/cassandradatacenter_types_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"testing"

"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
)

var internodeEnabledAll = `
Expand Down Expand Up @@ -125,3 +127,66 @@ func TestUseClientImage(t *testing.T) {
}
}
}

func TestUseClientImageEnforce(t *testing.T) {
assert := assert.New(t)

tests := []struct {
serverType string
version string
}{
{
serverType: "cassandra",
version: "4.1.0",
},
{
serverType: "cassandra",
version: "4.1.2",
},
{
serverType: "cassandra",
version: "5.0.0",
},
{
serverType: "cassandra",
version: "3.11.17",
},
{
serverType: "cassandra",
version: "4.0.8",
},
{
serverType: "dse",
version: "6.8.39",
},
{
serverType: "dse",
version: "6.9.0",
},
{
serverType: "hcd",
version: "1.0.0",
},
{
serverType: "dse",
version: "4.1.2",
},
}

for _, tt := range tests {
dc := CassandraDatacenter{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
UseClientBuilderAnnotation: "true",
},
},
Spec: CassandraDatacenterSpec{
ServerVersion: tt.version,
ServerType: tt.serverType,
ReadOnlyRootFilesystem: ptr.To[bool](true),
},
}

assert.True(dc.UseClientImage())
}
}
62 changes: 54 additions & 8 deletions pkg/reconciliation/construct_podtemplatespec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe

volumeDefaults := []corev1.Volume{vServerConfig, vServerLogs}

if readOnlyFs(dc) {
if dc.ReadOnlyFs() {
tmp := corev1.Volume{
Name: "tmp",
VolumeSource: corev1.VolumeSource{
Expand All @@ -323,6 +323,34 @@ func addVolumes(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTemplateSpe
}

volumeDefaults = append(volumeDefaults, tmp, etcCass)

if dc.Spec.ServerType == "dse" {
dseConf := corev1.Volume{
Name: "dse-conf",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
}
volumeDefaults = append(volumeDefaults, dseConf)

if !dc.UseClientImage() {
sparkConf := corev1.Volume{
Name: "spark-conf",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
}

collectDConf := corev1.Volume{
Name: "collectd-conf",
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
}

volumeDefaults = append(volumeDefaults, sparkConf, collectDConf)
}
}
}

if dc.UseClientImage() {
Expand Down Expand Up @@ -476,6 +504,8 @@ func buildInitContainers(dc *api.CassandraDatacenter, rackName string, baseTempl
configContainer.Command = []string{"/bin/sh"}
if dc.Spec.ServerType == "cassandra" {
configContainer.Args = []string{"-c", "cp -rf /etc/cassandra/* /cassandra-base-config/"}
} else if dc.Spec.ServerType == "dse" {
configContainer.Args = []string{"-c", "cp -rf /opt/dse/resources/cassandra/conf/* /cassandra-base-config/"}
} else if dc.Spec.ServerType == "hcd" {
configContainer.Args = []string{"-c", "cp -rf /opt/hcd/resources/cassandra/conf/* /cassandra-base-config/"}
}
Expand Down Expand Up @@ -649,7 +679,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
}
}

if readOnlyFs(dc) {
if dc.ReadOnlyFs() {
cassContainer.SecurityContext = &corev1.SecurityContext{
ReadOnlyRootFilesystem: ptr.To[bool](true),
}
Expand Down Expand Up @@ -680,7 +710,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
envDefaults = append(envDefaults, corev1.EnvVar{Name: "HCD_AUTO_CONF_OFF", Value: "all"})
}

if readOnlyFs(dc) {
if dc.ReadOnlyFs() {
envDefaults = append(envDefaults, corev1.EnvVar{Name: "MGMT_API_DISABLE_MCAC", Value: "true"})
}

Expand Down Expand Up @@ -737,7 +767,7 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
}
}

if readOnlyFs(dc) {
if dc.ReadOnlyFs() {
cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
Name: "tmp",
MountPath: "/tmp",
Expand All @@ -748,6 +778,26 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
Name: "etc-cassandra",
MountPath: "/opt/hcd/resources/cassandra/conf",
})
} else if dc.Spec.ServerType == "dse" {
cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
Name: "etc-cassandra",
MountPath: "/opt/dse/resources/cassandra/conf",
})
cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
Name: "dse-conf",
MountPath: "/opt/dse/resources/dse/conf",
})

if !dc.UseClientImage() {
cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
Name: "spark-conf",
MountPath: "/opt/dse/resources/spark/conf",
})
cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
Name: "collectd-conf",
MountPath: "/opt/dse/resources/dse/collectd/etc/collectd",
})
}
} else {
cassContainer.VolumeMounts = append(cassContainer.VolumeMounts, corev1.VolumeMount{
Name: "etc-cassandra",
Expand Down Expand Up @@ -813,10 +863,6 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
return nil
}

func readOnlyFs(dc *api.CassandraDatacenter) bool {
return dc.Spec.ReadOnlyRootFilesystem != nil && *dc.Spec.ReadOnlyRootFilesystem && dc.UseClientImage()
}

func buildPodTemplateSpec(dc *api.CassandraDatacenter, rack api.Rack, addLegacyInternodeMount bool) (*corev1.PodTemplateSpec, error) {

baseTemplate := dc.Spec.PodTemplateSpec.DeepCopy()
Expand Down
Loading

0 comments on commit 8e06206

Please sign in to comment.