MC-1296 Make k8ssandra-operator deploy Reaper capable of interaction with encrypted mgmt-api

Author: rzvoncek

CHANGELOG/CHANGELOG-1.21.md

@@ -15,4 +15,4 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 
 ## unreleased
 
-
+* [FEATURE] [#1508](https://github.com/riptano/mission-control/issues/1508) Make k8ssandra-operator deploy Reaper capable of interaction with encrypted mgmt-api

controllers/k8ssandra/reaper.go

@@ -29,6 +29,7 @@ import (
 	k8ssandralabels "github.com/k8ssandra/k8ssandra-operator/pkg/labels"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/reaper"
 	"github.com/k8ssandra/k8ssandra-operator/pkg/result"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
@@ -100,6 +101,13 @@ func (r *K8ssandraClusterReconciler) reconcileReaper(
 	// we might have nil-ed the template because a DC got stopped, so we need to re-check
 	if reaperTemplate != nil {
 		if reaperTemplate.HasReaperRef() {
+
+			if uses, conf := usesHttpAuth(kc, actualDc); uses {
+				if err := addManagementApiSecretsToReaper(ctx, remoteClient, kc, actualDc, logger, conf); err != nil {
+					return result.Error(err)
+				}
+			}
+
 			logger.Info("ReaperRef present, registering with referenced Reaper instead of creating a new one")
 			return r.addClusterToExternalReaper(ctx, kc, actualDc, logger)
 		}
@@ -267,6 +275,22 @@ func getSingleReaperDcName(kc *api.K8ssandraCluster) string {
 	return ""
 }
 
+func usesHttpAuth(kc *api.K8ssandraCluster, actualDc *cassdcapi.CassandraDatacenter) (bool, *cassdcapi.ManagementApiAuthManualConfig) {
+	// check for the mgmt api auth config in the cass-dc object of the DC were in
+	for _, dc := range kc.Spec.Cassandra.Datacenters {
+		if !dc.Stopped && dc.DatacenterName == actualDc.DatacenterName() {
+			return true, dc.ManagementApiAuth.Manual
+		}
+	}
+	// if that wasn't found, then check for the mgmt api auth config in the cluster object
+	if kc.Spec.Cassandra.ManagementApiAuth != nil {
+		if kc.Spec.Cassandra.ManagementApiAuth.Manual != nil {
+			return true, kc.Spec.Cassandra.ManagementApiAuth.Manual
+		}
+	}
+	return false, nil
+}
+
 func (r *K8ssandraClusterReconciler) addClusterToExternalReaper(
 	ctx context.Context,
 	kc *api.K8ssandraCluster,
@@ -290,3 +314,69 @@ func (r *K8ssandraClusterReconciler) addClusterToExternalReaper(
 	}
 	return result.Continue()
 }
+
+func addManagementApiSecretsToReaper(
+	ctx context.Context,
+	remoteClient client.Client,
+	kc *api.K8ssandraCluster,
+	actualDc *cassdcapi.CassandraDatacenter,
+	logger logr.Logger,
+	authConfig *cassdcapi.ManagementApiAuthManualConfig,
+) error {
+
+	reaperName := kc.Spec.Reaper.ReaperRef.Name
+
+	reaperNamespace := kc.Spec.Reaper.ReaperRef.Namespace
+	if reaperNamespace == "" {
+		reaperNamespace = kc.Namespace
+	}
+
+	tssName := reaper.GetTruststoresSecretName(reaperName)
+	tssKey := client.ObjectKey{Namespace: reaperNamespace, Name: tssName}
+
+	tss := &corev1.Secret{}
+	if err := remoteClient.Get(ctx, tssKey, tss); err != nil {
+		logger.Error(err, "failed to get Reaper's truststore secret")
+		return err
+	}
+
+	cs := &corev1.Secret{}
+	csName := authConfig.ClientSecretName + "-ks"
+	csKey := client.ObjectKey{Namespace: kc.Namespace, Name: csName}
+	if err := remoteClient.Get(ctx, csKey, cs); err != nil {
+		logger.Error(err, "failed to get k8ssandra cluster client secret", "secretName", csName)
+		return err
+	}
+
+	clusterName := cassdcapi.CleanupForKubernetes(actualDc.Spec.ClusterName)
+	clustersTruststore := clusterName + "-truststore.jks"
+	clustersKeystore := clusterName + "-keystore.jks"
+
+	// check if the reaper's big secret already has entry for this cluster
+	if tss.Data == nil {
+		tss.Data = make(map[string][]byte)
+	}
+	_, hasTruststore := tss.Data[clustersTruststore]
+	_, hasKeystore := tss.Data[clustersKeystore]
+
+	if hasTruststore && hasKeystore {
+		logger.Info("Cluster secrets already present in Reapers secret", "reaperName", reaperName, "clusterName", kc.Name)
+		return nil
+	}
+
+	logger.Info("Patching Reaper's truststores with new secrets", "reaperName", reaperName, "clusterName", kc.Name)
+
+	patch := client.MergeFrom(tss.DeepCopy())
+	if !hasTruststore {
+		tss.Data[clustersTruststore] = cs.Data["truststore.jks"]
+	}
+	if !hasKeystore {
+		tss.Data[clustersKeystore] = cs.Data["keystore.jks"]
+	}
+	if err := remoteClient.Patch(ctx, tss, patch); err != nil {
+		logger.Error(err, "failed to patch reaper's config map")
+		return err
+	}
+
+	return nil
+}

controllers/k8ssandra/reaper_test.go

@@ -235,18 +235,28 @@ func createMultiDcClusterWithReaper(t *testing.T, ctx context.Context, f *framew
 
 func createMultiDcClusterWithControlPlaneReaper(t *testing.T, ctx context.Context, f *framework.Framework, namespace string) {
 	require := require.New(t)
+	reaperName := "reaper"
 
 	cpr := &reaperapi.Reaper{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: namespace,
-			Name:      "reaper",
+			Name:      reaperName,
 		},
 		Spec: newControlPlaneReaper(),
 	}
 
 	err := f.Client.Create(ctx, cpr)
 	require.NoError(err, "failed to create control plane reaper")
 
+	rts := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      reaper.GetTruststoresSecretName(reaperName),
+		},
+	}
+	err = f.Client.Create(ctx, rts)
+	require.NoError(err, "failed to create reaper's truststore secret")
+
 	cpReaperKey := framework.ClusterKey{
 		K8sContext: f.ControlPlaneContext,
 		NamespacedName: types.NamespacedName{
@@ -308,9 +318,6 @@ func createMultiDcClusterWithControlPlaneReaper(t *testing.T, ctx context.Contex
 	verifyReaperAbsent(t, f, ctx, kc, f.DataPlaneContexts[0], dc1Key, namespace)
 	verifyReaperAbsent(t, f, ctx, kc, f.DataPlaneContexts[1], dc2Key, namespace)
 
-	// check the kc is added to reaper
-	verifyClusterRegistered(t, f, ctx, kc, namespace)
-
 	err = f.DeleteK8ssandraCluster(ctx, utils.GetKey(kc), timeout, interval)
 	require.NoError(err, "failed to delete K8ssandraCluster")
 }
@@ -401,7 +408,3 @@ func verifyReaperAbsent(t *testing.T, f *framework.Framework, ctx context.Contex
 	err := f.Get(ctx, reaperKey, reaper)
 	require.True(t, err != nil && errors.IsNotFound(err), fmt.Sprintf("reaper %s should not be created in dc %s", reaperKey, dcKey))
 }
-
-func verifyClusterRegistered(t *testing.T, f *framework.Framework, ctx context.Context, kc *api.K8ssandraCluster, namespace string) {
-
-}

controllers/reaper/reaper_controller.go

@@ -30,6 +30,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"
@@ -206,6 +207,13 @@ func (r *ReaperReconciler) reconcileDeployment(
 		}
 	}
 
+	if actualReaper.Spec.HttpManagement.Enabled {
+		err := r.reconcileTrustStoresSecret(ctx, actualReaper, logger)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
 	logger.Info("Reconciling reaper deployment", "actualReaper", actualReaper)
 
 	// work out how to deploy Reaper
@@ -484,3 +492,31 @@ func (r *ReaperReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(&corev1.Service{}).
 		Complete(r)
 }
+
+func (r *ReaperReconciler) reconcileTrustStoresSecret(ctx context.Context, actualReaper *reaperapi.Reaper, logger logr.Logger) error {
+	sName := reaper.GetTruststoresSecretName(actualReaper.Name)
+	sKey := types.NamespacedName{Namespace: actualReaper.Namespace, Name: sName}
+	s := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      sName,
+			Namespace: actualReaper.Namespace,
+		},
+	}
+	if err := r.Client.Get(ctx, sKey, s); err != nil {
+		if errors.IsNotFound(err) {
+			logger.Info("Creating Reaper's truststore ConfigMap", "ConfigMap", sKey)
+			if err = controllerutil.SetControllerReference(actualReaper, s, r.Scheme); err != nil {
+				logger.Error(err, "Failed to set owner on truststore ConfigMap")
+				return err
+			}
+			if err = r.Client.Create(ctx, s); err != nil {
+				logger.Error(err, "Failed to create Reaper's truststore ConfigMap")
+				return err
+			}
+			return nil
+		}
+		logger.Error(err, "Failed to get Reaper's truststores ConfigMap")
+		return err
+	}
+	return nil
+}

controllers/reaper/reaper_controller_test.go

@@ -2,6 +2,7 @@ package reaper
 
 import (
 	"context"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/utils/ptr"
 	"testing"
@@ -64,6 +65,7 @@ func TestReaper(t *testing.T) {
 	t.Run("CreateReaperWithAuthEnabled", reaperControllerTest(ctx, testEnv, testCreateReaperWithAuthEnabled))
 	t.Run("CreateReaperWithAuthEnabledExternalSecret", reaperControllerTest(ctx, testEnv, testCreateReaperWithAuthEnabledExternalSecret))
 	t.Run("CreateReaperWithLocalStorageBackend", reaperControllerTest(ctx, testEnv, testCreateReaperWithLocalStorageType))
+	t.Run("CreateReaperWithHttpAuthEnabled", reaperControllerTest(ctx, testEnv, testCreateReaperWithHttpAuthEnabled))
 }
 
 func newMockManager() reaper.Manager {
@@ -570,6 +572,72 @@ func testCreateReaperWithLocalStorageType(t *testing.T, ctx context.Context, k8s
 	assert.Equal(t, "reaper-data", dataVolumeMount.Name)
 }
 
+func testCreateReaperWithHttpAuthEnabled(t *testing.T, ctx context.Context, k8sClient client.Client, testNamespace string) {
+	t.Log("create the Reaper object")
+	r := newReaper(testNamespace)
+	r.Spec.StorageType = reaperapi.StorageTypeLocal
+	r.Spec.StorageConfig = newStorageConfig()
+	r.Spec.HttpManagement.Enabled = true
+
+	err := k8sClient.Create(ctx, r)
+	require.NoError(t, err)
+
+	t.Log("check that the stateful set is created")
+	stsKey := types.NamespacedName{Namespace: testNamespace, Name: reaperName}
+	sts := &appsv1.StatefulSet{}
+
+	require.Eventually(t, func() bool {
+		return k8sClient.Get(ctx, stsKey, sts) == nil
+	}, timeout, interval, "stateful set creation check failed")
+
+	// if the http auth is enabled, reaper controller should prepare the secret where k8s controller will place per-cluster secrets
+	secretKey := types.NamespacedName{Namespace: testNamespace, Name: reaper.GetTruststoresSecretName(r.Name)}
+	truststoresSecret := &corev1.Secret{}
+	require.Eventually(t, func() bool {
+		return k8sClient.Get(ctx, secretKey, truststoresSecret) == nil
+	}, timeout, interval, "truststore secret creation check failed")
+	assert.True(t, truststoresSecret.Data == nil)
+	assert.Equal(t, 0, len(truststoresSecret.Data))
+
+	// In this configuration, we expect Reaper to also have a mount for the http auth secrets
+	assert.Len(t, sts.Spec.Template.Spec.Containers[0].VolumeMounts, 3)
+	confVolumeMount := sts.Spec.Template.Spec.Containers[0].VolumeMounts[0].DeepCopy()
+	assert.Equal(t, "conf", confVolumeMount.Name)
+	dataVolumeMount := sts.Spec.Template.Spec.Containers[0].VolumeMounts[2].DeepCopy()
+	assert.Equal(t, "reaper-data", dataVolumeMount.Name)
+	truststoresVolumeMount := sts.Spec.Template.Spec.Containers[0].VolumeMounts[1].DeepCopy()
+	assert.Equal(t, "management-api-keystores-per-cluster", truststoresVolumeMount.Name)
+
+	// when we delete reaper, the STS and the secret both go away due to the owner reference
+	// however, that does not happen in env tests
+	err = k8sClient.Delete(ctx, r)
+	require.NoError(t, err)
+
+	reaperKey := types.NamespacedName{Namespace: testNamespace, Name: reaperName}
+	assert.Eventually(t, func() bool {
+		err = k8sClient.Get(ctx, reaperKey, r)
+		return errors.IsNotFound(err)
+	}, timeout, interval, "reaper stateful set deletion check failed")
+
+	assert.Eventually(t, func() bool {
+		err = k8sClient.Get(ctx, stsKey, sts)
+		// we'd expect errors.IsNotFound(err) here, except for some reason this is not happening in env tests
+		return err == nil
+	}, timeout, interval, "reaper stateful set deletion check failed")
+
+	assert.Eventually(t, func() bool {
+		err = k8sClient.Get(ctx, secretKey, truststoresSecret)
+		// again, we'd expect errors.IsNotFound(err) ...
+		return err == nil
+	}, timeout, interval, "reaper truststore secret deletion check failed")
+
+	// so we delete stuff manually
+	err = k8sClient.Delete(ctx, sts)
+	require.NoError(t, err)
+	err = k8sClient.Delete(ctx, truststoresSecret)
+	require.NoError(t, err)
+}
+
 // Check if env var exists
 func envVarExists(envVars []corev1.EnvVar, name string) bool {
 	for _, envVar := range envVars {

pkg/reaper/deployment.go

@@ -155,6 +155,7 @@ func computeEnvVars(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter) []cor
 			Value: "true",
 		})
 
+		// we might have a general-purpose keystore and truststore
 		if reaper.Spec.HttpManagement.Keystores != nil {
 			envVars = append(envVars, corev1.EnvVar{
 				Name:  "REAPER_HTTP_MANAGEMENT_KEYSTORE_PATH",
@@ -165,6 +166,18 @@ func computeEnvVars(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter) []cor
 				Value: "/etc/encryption/mgmt/truststore.jks",
 			})
 		}
+
+		// when we're a control plane, and we use http
+		// we might need to have specific stores per cluster managed by this Reaper instance
+		// we always deploy the secret that holds them, even if it never gets populated
+		if reaper.Spec.StorageType == api.StorageTypeLocal && reaper.Spec.DatacenterRef.Name == "" {
+			if reaper.Spec.HttpManagement.Enabled {
+				envVars = append(envVars, corev1.EnvVar{
+					Name:  "REAPER_HTTP_MANAGEMENT_TRUSTSTORES_DIR",
+					Value: "/etc/encryption/mgmt/perClusterTruststores",
+				})
+			}
+		}
 	}
 
 	return envVars
@@ -203,6 +216,21 @@ func computeVolumes(reaper *api.Reaper) ([]corev1.Volume, []corev1.VolumeMount)
 		})
 	}
 
+	if reaper.Spec.HttpManagement.Enabled {
+		volumes = append(volumes, corev1.Volume{
+			Name: "management-api-keystores-per-cluster",
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: GetTruststoresSecretName(reaper.Name),
+				},
+			},
+		})
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      "management-api-keystores-per-cluster",
+			MountPath: "/etc/encryption/mgmt/perClusterTruststores",
+		})
+	}
+
 	if reaper.Spec.StorageType == api.StorageTypeLocal {
 		volumes = append(volumes, corev1.Volume{
 			Name: "reaper-data",
@@ -286,7 +314,13 @@ func configureClientEncryption(reaper *api.Reaper, envVars []corev1.EnvVar, volu
 	return envVars, volumes, volumeMounts
 }
 
-func computePodSpec(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, initContainerResources *corev1.ResourceRequirements, keystorePassword *string, truststorePassword *string) corev1.PodSpec {
+func computePodSpec(
+	reaper *api.Reaper,
+	dc *cassdcapi.CassandraDatacenter,
+	initContainerResources *corev1.ResourceRequirements,
+	keystorePassword *string,
+	truststorePassword *string,
+) corev1.PodSpec {
 	envVars := computeEnvVars(reaper, dc)
 	volumes, volumeMounts := computeVolumes(reaper)
 	mainImage := reaper.Spec.ContainerImage.ApplyDefaults(defaultImage)
@@ -365,7 +399,7 @@ func NewStatefulSet(reaper *api.Reaper, dc *cassdcapi.CassandraDatacenter, logge
 	}
 
 	if reaper.Spec.ReaperTemplate.StorageConfig == nil {
-		logger.Error(fmt.Errorf("reaper spec needs storage config when using memory sotrage type"), "missing storage config")
+		logger.Error(fmt.Errorf("reaper spec needs storage config when using 'local' sotrage type"), "missing storage config")
 		return nil
 	}