diff --git a/apis/bases/redis.openstack.org_redises.yaml b/apis/bases/redis.openstack.org_redises.yaml index b1357bd3..b466e784 100644 --- a/apis/bases/redis.openstack.org_redises.yaml +++ b/apis/bases/redis.openstack.org_redises.yaml @@ -44,6 +44,17 @@ spec: description: Size of the redis cluster format: int32 type: integer + tls: + description: TLS settings for Redis service and internal Redis replication + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object required: - containerImage type: object @@ -93,6 +104,11 @@ spec: - type type: object type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track input changes + type: object type: object type: object served: true diff --git a/apis/redis/v1beta1/redis_types.go b/apis/redis/v1beta1/redis_types.go index f114db69..a22c7549 100644 --- a/apis/redis/v1beta1/redis_types.go +++ b/apis/redis/v1beta1/redis_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -34,15 +35,20 @@ type RedisSpec struct { // +kubebuilder:validation:Required // Name of the redis container image to run (will be set to environmental default if empty) ContainerImage string `json:"containerImage"` - // +kubebuilder:validation:Optional // +kubebuilder:default=1 // Size of the redis cluster Replicas *int32 `json:"replicas"` + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS settings for Redis service and internal Redis replication + TLS tls.SimpleService `json:"tls,omitempty"` } // RedisStatus defines the observed state of Redis type RedisStatus struct { + // Map of hashes to track input changes + Hash map[string]string `json:"hash,omitempty"` // Conditions Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` } diff --git a/apis/redis/v1beta1/zz_generated.deepcopy.go b/apis/redis/v1beta1/zz_generated.deepcopy.go index 15dfb3a9..1be4a6e4 100644 --- a/apis/redis/v1beta1/zz_generated.deepcopy.go +++ b/apis/redis/v1beta1/zz_generated.deepcopy.go @@ -108,6 +108,7 @@ func (in *RedisSpec) DeepCopyInto(out *RedisSpec) { *out = new(int32) **out = **in } + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedisSpec. @@ -123,6 +124,13 @@ func (in *RedisSpec) DeepCopy() *RedisSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RedisStatus) DeepCopyInto(out *RedisStatus) { *out = *in + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make(condition.Conditions, len(*in)) diff --git a/config/crd/bases/redis.openstack.org_redises.yaml b/config/crd/bases/redis.openstack.org_redises.yaml index b1357bd3..b466e784 100644 --- a/config/crd/bases/redis.openstack.org_redises.yaml +++ b/config/crd/bases/redis.openstack.org_redises.yaml @@ -44,6 +44,17 @@ spec: description: Size of the redis cluster format: int32 type: integer + tls: + description: TLS settings for Redis service and internal Redis replication + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object required: - containerImage type: object @@ -93,6 +104,11 @@ spec: - type type: object type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track input changes + type: object type: object type: object served: true diff --git a/config/samples/redis_v1beta1_redis_tls.yaml b/config/samples/redis_v1beta1_redis_tls.yaml new file mode 100644 index 00000000..569b7440 --- /dev/null +++ b/config/samples/redis_v1beta1_redis_tls.yaml @@ -0,0 +1,9 @@ +apiVersion: redis.openstack.org/v1beta1 +kind: Redis +metadata: + name: redis +spec: + replicas: 3 + tls: + secretName: redis-tls + caBundleSecretName: redis-tls diff --git a/controllers/redis/redis_controller.go b/controllers/redis/redis_controller.go index e969dc37..6ddb50d6 100644 --- a/controllers/redis/redis_controller.go +++ b/controllers/redis/redis_controller.go @@ -21,17 +21,23 @@ import ( "fmt" "time" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/go-logr/logr" - redisv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" + redisv1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" appsv1 "k8s.io/api/apps/v1" @@ -45,7 +51,6 @@ import ( common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" commonservice "github.com/openstack-k8s-operators/lib-common/modules/common/service" commonstatefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) // GetLogger returns a logger object with a prefix of "controller.name" and additional controller context fields @@ -53,10 +58,24 @@ func (r *Reconciler) GetLogger(ctx context.Context) logr.Logger { return log.FromContext(ctx).WithName("Controllers").WithName("Redis") } +// fields to index to reconcile on CR change +const ( + serviceSecretNameField = ".spec.tls.genericService.SecretName" + caSecretNameField = ".spec.tls.ca.caBundleSecretName" +) + +var ( + allWatchFields = []string{ + serviceSecretNameField, + caSecretNameField, + } +) + // Reconciler reconciles a Redis object type Reconciler struct { client.Client Kclient kubernetes.Interface + config *rest.Config Scheme *runtime.Scheme } @@ -85,7 +104,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct Log := r.GetLogger(ctx) // Fetch the Redis instance - instance := &redisv1beta1.Redis{} + instance := &redisv1.Redis{} err := r.Get(ctx, req.NamespacedName, instance) if err != nil { if k8s_errors.IsNotFound(err) { @@ -137,6 +156,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct instance.Status.Conditions = condition.Conditions{} // initialize conditions used later as Status=Unknown cl := condition.CreateList( + // TLS cert secrets + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), // endpoint for adoption redirect condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), // configmap generation @@ -180,9 +201,37 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct return rbacResult, nil } + // Hash of all resources that may cause a service restart + inputHashEnv := make(map[string]env.Setter) + + // Check and hash inputs + var certHash, caHash string + specTLS := &instance.Spec.TLS + if specTLS.Enabled() { + certHash, _, err = specTLS.GenericService.ValidateCertSecret(ctx, helper, instance.Namespace) + inputHashEnv["Cert"] = env.SetValue(certHash) + } + if err == nil && specTLS.Ca.CaBundleSecretName != "" { + caName := types.NamespacedName{ + Name: specTLS.Ca.CaBundleSecretName, + Namespace: instance.Namespace, + } + caHash, _, err = tls.ValidateCACertSecret(ctx, helper.GetClient(), caName) + inputHashEnv["CA"] = env.SetValue(caHash) + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, fmt.Errorf("error calculating input hash: %w", err) + } + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + // Redis config maps - configMapVars := make(map[string]env.Setter) - err = r.generateConfigMaps(ctx, helper, instance, &configMapVars) + err = r.generateConfigMaps(ctx, helper, instance, &inputHashEnv) if err != nil { instance.Status.Conditions.Set(condition.FalseCondition( condition.ServiceConfigReadyCondition, @@ -194,21 +243,49 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct } instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + hashOfHashes, err := util.HashOfInputHashes(inputHashEnv) + if err != nil { + return ctrl.Result{}, err + } + if hashMap, changed := util.SetHash(instance.Status.Hash, common.InputHashName, hashOfHashes); changed { + // Hash changed and instance status should be updated (which will be done by main defer func), + // so update all the input hashes and return to reconcile again + instance.Status.Hash = hashMap + for k, s := range inputHashEnv { + var envVar corev1.EnvVar + s(&envVar) + instance.Status.Hash[k] = envVar.Value + } + util.LogForObject(helper, fmt.Sprintf("Input hash changed %s", hashOfHashes), instance) + return ctrl.Result{}, nil + } + // the headless service provides DNS entries for pods // the name of the resource must match the name of the app selector - pkghl := redis.HeadlessService(instance) - headless := &corev1.Service{ObjectMeta: pkghl.ObjectMeta} - _, err = controllerutil.CreateOrPatch(ctx, r.Client, headless, func() error { - headless.Spec = pkghl.Spec - err := controllerutil.SetOwnerReference(instance, headless, r.Client.Scheme()) - if err != nil { - return err - } - return nil - }) + headless, err := commonservice.NewService(redis.HeadlessService(instance), time.Duration(5)*time.Second, nil) if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) return ctrl.Result{}, err } + hlres, hlerr := headless.CreateOrPatch(ctx, helper) + if hlerr != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error())) + return hlres, hlerr + } // Service to expose Redis pods commonsvc, err := commonservice.NewService(redis.Service(instance), time.Duration(5)*time.Second, nil) @@ -252,11 +329,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct return ctrl.Result{}, nil } -// generateConfigMaps returns the config map resource for a galera instance +// generateConfigMaps returns the config map resource for a redis instance func (r *Reconciler) generateConfigMaps( ctx context.Context, h *helper.Helper, - instance *redisv1beta1.Redis, + instance *redisv1.Redis, envVars *map[string]env.Setter, ) error { templateParameters := make(map[string]interface{}) @@ -294,8 +371,37 @@ func (r *Reconciler) generateConfigMaps( // SetupWithManager sets up the controller with the Manager. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + r.config = mgr.GetConfig() + + // Various CR fields need to be indexed to filter watch events + // for the secret changes we want to be notified of + // index caBundleSecretName + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &redisv1.Redis{}, caSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*redisv1.Redis) + tls := &cr.Spec.TLS + if tls.Ca.CaBundleSecretName != "" { + return []string{tls.Ca.CaBundleSecretName} + } + return nil + }); err != nil { + return err + } + // index secretName + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &redisv1.Redis{}, serviceSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*redisv1.Redis) + tls := &cr.Spec.TLS + if tls.Enabled() { + return []string{*tls.GenericService.SecretName} + } + return nil + }); err != nil { + return err + } + return ctrl.NewControllerManagedBy(mgr). - For(&redisv1beta1.Redis{}). + For(&redisv1.Redis{}). Owns(&appsv1.StatefulSet{}). Owns(&corev1.Service{}). Owns(&corev1.ServiceAccount{}). @@ -303,3 +409,33 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&rbacv1.RoleBinding{}). Complete(r) } + +// findObjectsForSrc - returns a reconcile request if the object is referenced by a Redis CR +func (r *Reconciler) findObjectsForSrc(src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + for _, field := range allWatchFields { + crList := &redisv1.RedisList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(context.TODO(), crList, listOps) + if err != nil { + return []reconcile.Request{} + } + + for _, item := range crList.Items { + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} diff --git a/controllers/redis/suite_test.go b/controllers/redis/suite_test.go index 2d1cb320..5b6edbf7 100644 --- a/controllers/redis/suite_test.go +++ b/controllers/redis/suite_test.go @@ -30,7 +30,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - redisv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" + redisv1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" //+kubebuilder:scaffold:imports ) @@ -62,7 +62,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - err = redisv1beta1.AddToScheme(scheme.Scheme) + err = redisv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:scheme diff --git a/pkg/redis/deployment.go b/pkg/redis/deployment.go index 945b7810..7fcde5e9 100644 --- a/pkg/redis/deployment.go +++ b/pkg/redis/deployment.go @@ -1,7 +1,7 @@ package redis import ( - redisv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" + redisv1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -10,7 +10,7 @@ import ( ) // Deployment returns a Deployment resource for the Redis CR -func Deployment(r *redisv1beta1.Redis) *appsv1.Deployment { +func Deployment(r *redisv1.Redis) *appsv1.Deployment { matchls := map[string]string{ "app": "redis", "cr": "redis-" + r.Name, diff --git a/pkg/redis/service.go b/pkg/redis/service.go index c072ac80..0315531a 100644 --- a/pkg/redis/service.go +++ b/pkg/redis/service.go @@ -1,7 +1,7 @@ package redis import ( - redisv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" + redisv1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" common "github.com/openstack-k8s-operators/lib-common/modules/common" labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" service "github.com/openstack-k8s-operators/lib-common/modules/common/service" @@ -9,7 +9,7 @@ import ( ) // Service exposes redis pods for a redis CR -func Service(instance *redisv1beta1.Redis) *corev1.Service { +func Service(instance *redisv1.Redis) *corev1.Service { labels := labels.GetLabels(instance, "redis", map[string]string{ common.AppSelector: "redis", common.OwnerSelector: instance.Name, @@ -35,7 +35,7 @@ func Service(instance *redisv1beta1.Redis) *corev1.Service { } // HeadlessService - service to give redis pods connectivity via DNS -func HeadlessService(instance *redisv1beta1.Redis) *corev1.Service { +func HeadlessService(instance *redisv1.Redis) *corev1.Service { labels := labels.GetLabels(instance, "redis", map[string]string{ common.AppSelector: "redis", common.OwnerSelector: instance.Name, diff --git a/pkg/redis/statefulset.go b/pkg/redis/statefulset.go index ef8f194d..613e3f6d 100644 --- a/pkg/redis/statefulset.go +++ b/pkg/redis/statefulset.go @@ -3,7 +3,7 @@ package redis import ( "strconv" - redisv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" + redisv1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" common "github.com/openstack-k8s-operators/lib-common/modules/common" labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels" appsv1 "k8s.io/api/apps/v1" @@ -13,7 +13,7 @@ import ( ) // Deployment returns a Deployment resource for the Redis CR -func StatefulSet(r *redisv1beta1.Redis) *appsv1.StatefulSet { +func StatefulSet(r *redisv1.Redis) *appsv1.StatefulSet { matchls := map[string]string{ common.AppSelector: "redis", common.OwnerSelector: r.Name, @@ -93,7 +93,7 @@ func StatefulSet(r *redisv1beta1.Redis) *appsv1.StatefulSet { Command: []string{"/var/lib/operator-scripts/start_redis_replication.sh"}, Name: "redis", Env: commonEnvVars, - VolumeMounts: getRedisVolumeMounts(), + VolumeMounts: getRedisVolumeMounts(r), Ports: []corev1.ContainerPort{{ ContainerPort: 6379, Name: "redis", @@ -121,7 +121,7 @@ func StatefulSet(r *redisv1beta1.Redis) *appsv1.StatefulSet { Name: "SENTINEL_QUORUM", Value: strconv.Itoa((int(*r.Spec.Replicas) / 2) + 1), }), - VolumeMounts: getSentinelVolumeMounts(), + VolumeMounts: getSentinelVolumeMounts(r), Ports: []corev1.ContainerPort{{ ContainerPort: 26379, Name: "sentinel", diff --git a/pkg/redis/volumes.go b/pkg/redis/volumes.go index 993e8ab5..101535b4 100644 --- a/pkg/redis/volumes.go +++ b/pkg/redis/volumes.go @@ -3,12 +3,40 @@ package redis import ( "fmt" - redisv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" + redisv1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" corev1 "k8s.io/api/core/v1" ) -func getVolumes(r *redisv1beta1.Redis) []corev1.Volume { +const ( + RedisCertPrefix = "redis" +) + +func getVolumes(r *redisv1.Redis) []corev1.Volume { scriptsPerms := int32(0755) + configDataFiles := []corev1.KeyToPath{ + { + Key: "sentinel.conf.in", + Path: "var/lib/redis/sentinel.conf.in", + }, + { + Key: "redis.conf.in", + Path: "var/lib/redis/redis.conf.in", + }, + } + if r.Spec.TLS.Enabled() { + configDataFiles = append(configDataFiles, []corev1.KeyToPath{ + { + Key: "redis-tls.conf.in", + Path: "var/lib/redis/redis-tls.conf.in", + }, + { + Key: "sentinel-tls.conf.in", + Path: "var/lib/redis/sentinel-tls.conf.in", + }, + }...) + } + vols := []corev1.Volume{ { Name: "kolla-config", @@ -55,16 +83,7 @@ func getVolumes(r *redisv1beta1.Redis) []corev1.Volume { LocalObjectReference: corev1.LocalObjectReference{ Name: fmt.Sprintf("%s-config-data", r.Name), }, - Items: []corev1.KeyToPath{ - { - Key: "sentinel.conf.in", - Path: "var/lib/redis/sentinel.conf.in", - }, - { - Key: "redis.conf.in", - Path: "var/lib/redis/redis.conf.in", - }, - }, + Items: configDataFiles, }, }, }, @@ -102,10 +121,45 @@ func getVolumes(r *redisv1beta1.Redis) []corev1.Volume { }, }, } + + if r.Spec.TLS.Enabled() { + svc := tls.Service{ + SecretName: *r.Spec.TLS.GenericService.SecretName, + CertMount: nil, + KeyMount: nil, + CaMount: nil, + } + serviceVolume := svc.CreateVolume(RedisCertPrefix) + vols = append(vols, serviceVolume) + if r.Spec.TLS.Ca.CaBundleSecretName != "" { + caVolume := r.Spec.TLS.Ca.CreateVolume() + vols = append(vols, caVolume) + } + } + + return vols +} + +func getTLSVolumeMounts(r *redisv1.Redis) []corev1.VolumeMount { + var vols []corev1.VolumeMount + if r.Spec.TLS.Enabled() { + svc := tls.Service{ + SecretName: *r.Spec.TLS.GenericService.SecretName, + CertMount: nil, + KeyMount: nil, + CaMount: nil, + } + serviceVolumeMounts := svc.CreateVolumeMounts(RedisCertPrefix) + vols = serviceVolumeMounts + if r.Spec.TLS.Ca.CaBundleSecretName != "" { + caVolumeMounts := r.Spec.TLS.Ca.CreateVolumeMounts(nil) + vols = append(vols, caVolumeMounts...) + } + } return vols } -func getRedisVolumeMounts() []corev1.VolumeMount { +func getRedisVolumeMounts(r *redisv1.Redis) []corev1.VolumeMount { vm := []corev1.VolumeMount{{ MountPath: "/var/lib/config-data/default", ReadOnly: true, @@ -122,10 +176,11 @@ func getRedisVolumeMounts() []corev1.VolumeMount { ReadOnly: true, Name: "kolla-config", }} + vm = append(vm, getTLSVolumeMounts(r)...) return vm } -func getSentinelVolumeMounts() []corev1.VolumeMount { +func getSentinelVolumeMounts(r *redisv1.Redis) []corev1.VolumeMount { vm := []corev1.VolumeMount{{ MountPath: "/var/lib/config-data/default", ReadOnly: true, @@ -142,5 +197,6 @@ func getSentinelVolumeMounts() []corev1.VolumeMount { ReadOnly: true, Name: "kolla-config-sentinel", }} + vm = append(vm, getTLSVolumeMounts(r)...) return vm } diff --git a/templates/redis/bin/common.sh b/templates/redis/bin/common.sh index 8f287926..805f21c6 100644 --- a/templates/redis/bin/common.sh +++ b/templates/redis/bin/common.sh @@ -10,6 +10,16 @@ TIMEOUT=3 POD_NAME=$HOSTNAME POD_FQDN=$HOSTNAME.$SVC_FQDN +if test -d /var/lib/config-data/tls; then + REDIS_CLI_CMD="redis-cli --tls" + REDIS_CONFIG=/var/lib/redis/redis-tls.conf + SENTINEL_CONFIG=/var/lib/redis/sentinel-tls.conf +else + REDIS_CLI_CMD=redis-cli + REDIS_CONFIG=/var/lib/redis/redis.conf + SENTINEL_CONFIG=/var/lib/redis/sentinel.conf +fi + function log() { echo "$(date +%F_%H_%M_%S) $*" } diff --git a/templates/redis/bin/redis_probe.sh b/templates/redis/bin/redis_probe.sh index c199a250..94390351 100755 --- a/templates/redis/bin/redis_probe.sh +++ b/templates/redis/bin/redis_probe.sh @@ -1,16 +1,18 @@ #!/bin/bash set -eux +. /var/lib/operator-scripts/common.sh + case "$1" in readiness) # ready if we're the master or if we're a slave connected to the current master - output=$(redis-cli info replication | tr -d '\r') + output=$($REDIS_CLI_CMD info replication | tr -d '\r') declare -A state while IFS=: read -r key value; do state[$key]=$value; done < <(echo "$output") [[ "${state[role]}" == "master" ]] || [[ "${state[role]}" == "slave" && "${state[master_link_status]}" == "up" ]] ;; liveness) - redis-cli -e ping >/dev/null;; + $REDIS_CLI_CMD -e ping >/dev/null;; *) echo "Invalid probe option '$1'" exit 1;; diff --git a/templates/redis/bin/start_redis_replication.sh b/templates/redis/bin/start_redis_replication.sh index a24443c4..edad01da 100755 --- a/templates/redis/bin/start_redis_replication.sh +++ b/templates/redis/bin/start_redis_replication.sh @@ -6,19 +6,19 @@ generate_configs sudo -E kolla_set_configs # 1. check if a redis cluster is already running by contacting sentinel -output=$(timeout ${TIMEOUT} redis-cli -h ${SVC_FQDN} -p 26379 sentinel master redis) +output=$(timeout ${TIMEOUT} $REDIS_CLI_CMD -h ${SVC_FQDN} -p 26379 sentinel master redis) if [ $? -eq 0 ]; then master=$(echo "$output" | awk '/^ip$/ {getline; print $0; exit}') # TODO skip if no master was found log "Connecting to the existing Redis cluster (master: ${master})" - exec redis-server /var/lib/redis/redis.conf --protected-mode no --replicaof "$master" 6379 + exec redis-server $REDIS_CONFIG --protected-mode no --replicaof "$master" 6379 fi # 2. else bootstrap a new cluster (assume we should be the first redis pod) if is_bootstrap_pod $POD_NAME; then log "Bootstrapping a new Redis cluster from ${POD_NAME}" set_pod_label $POD_NAME redis~1master - exec redis-server /var/lib/redis/redis.conf --protected-mode no + exec redis-server $REDIS_CONFIG --protected-mode no fi # 3. else this is an error, exit and let the pod restart and try again diff --git a/templates/redis/bin/start_sentinel.sh b/templates/redis/bin/start_sentinel.sh index ffcaeb90..91cebd1f 100755 --- a/templates/redis/bin/start_sentinel.sh +++ b/templates/redis/bin/start_sentinel.sh @@ -6,21 +6,21 @@ generate_configs sudo -E kolla_set_configs # 1. check if a redis cluster is already running by contacting sentinel -output=$(timeout ${TIMEOUT} redis-cli -h ${SVC_FQDN} -p 26379 sentinel master redis) +output=$(timeout ${TIMEOUT} $REDIS_CLI_CMD -h ${SVC_FQDN} -p 26379 sentinel master redis) if [ $? -eq 0 ]; then master=$(echo "$output" | awk '/^ip$/ {getline; print $0; exit}') # TODO skip if no master was found log "Connecting to the existing sentinel cluster (master: $master)" - echo "sentinel monitor redis ${master} 6379 ${SENTINEL_QUORUM}" >> /var/lib/redis/sentinel.conf - exec redis-sentinel /var/lib/redis/sentinel.conf + echo "sentinel monitor redis ${master} 6379 ${SENTINEL_QUORUM}" >> $SENTINEL_CONFIG + exec redis-sentinel $SENTINEL_CONFIG fi # 2. else let the pod's redis server bootstrap a new cluster and monitor it # (assume we should be the first redis pod) if is_bootstrap_pod $POD_NAME; then log "Bootstrapping a new sentinel cluster" - echo "sentinel monitor redis ${POD_FQDN} 6379 ${SENTINEL_QUORUM}" >> /var/lib/redis/sentinel.conf - exec redis-sentinel /var/lib/redis/sentinel.conf + echo "sentinel monitor redis ${POD_FQDN} 6379 ${SENTINEL_QUORUM}" >> $SENTINEL_CONFIG + exec redis-sentinel $SENTINEL_CONFIG fi # 3. else this is an error, exit and let the pod restart and try again diff --git a/templates/redis/config/config-sentinel.json b/templates/redis/config/config-sentinel.json index 3bf685c3..3b750fd7 100644 --- a/templates/redis/config/config-sentinel.json +++ b/templates/redis/config/config-sentinel.json @@ -7,6 +7,20 @@ "preserve_properties": true, "optional": true, "source": "/var/lib/config-data/generated/*" + }, + { + "source": "/var/lib/config-data/tls/private/redis.key", + "dest": "/etc/pki/tls/private/redis.key", + "owner": "redis", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/config-data/tls/certs/redis.crt", + "dest": "/etc/pki/tls/certs/redis.crt", + "owner": "redis", + "perm": "0755", + "optional": true } ], "permissions": [ diff --git a/templates/redis/config/config.json b/templates/redis/config/config.json index f5e32411..e92c06b0 100644 --- a/templates/redis/config/config.json +++ b/templates/redis/config/config.json @@ -7,6 +7,20 @@ "preserve_properties": true, "optional": true, "source": "/var/lib/config-data/generated/*" + }, + { + "source": "/var/lib/config-data/tls/private/redis.key", + "dest": "/etc/pki/tls/private/redis.key", + "owner": "redis", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/config-data/tls/certs/redis.crt", + "dest": "/etc/pki/tls/certs/redis.crt", + "owner": "redis", + "perm": "0755", + "optional": true } ], "permissions": [ diff --git a/templates/redis/config/redis-tls.conf.in b/templates/redis/config/redis-tls.conf.in new file mode 100644 index 00000000..d39f1532 --- /dev/null +++ b/templates/redis/config/redis-tls.conf.in @@ -0,0 +1,9 @@ +include /var/lib/redis/redis.conf + +port 0 +tls-port 6379 +tls-cert-file /etc/pki/tls/certs/redis.crt +tls-key-file /etc/pki/tls/private/redis.key +tls-ca-cert-file /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +tls-replication yes +tls-auth-clients optional diff --git a/templates/redis/config/sentinel-tls.conf.in b/templates/redis/config/sentinel-tls.conf.in new file mode 100644 index 00000000..b68ecca1 --- /dev/null +++ b/templates/redis/config/sentinel-tls.conf.in @@ -0,0 +1,9 @@ +include /var/lib/redis/sentinel.conf + +port 0 +tls-port 26379 +tls-cert-file /etc/pki/tls/certs/redis.crt +tls-key-file /etc/pki/tls/private/redis.key +tls-ca-cert-file /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +tls-replication yes +tls-auth-clients optional diff --git a/tests/kuttl/tests/redis/01-assert.yaml b/tests/kuttl/tests/redis/01-assert.yaml index 801a13d9..69989fa6 100644 --- a/tests/kuttl/tests/redis/01-assert.yaml +++ b/tests/kuttl/tests/redis/01-assert.yaml @@ -46,6 +46,10 @@ status: reason: Ready status: "True" type: ServiceConfigReady + - message: Input data complete + reason: Ready + status: "True" + type: TLSInputReady --- apiVersion: apps/v1 kind: StatefulSet diff --git a/tests/kuttl/tests/redis/02-assert.yaml b/tests/kuttl/tests/redis/02-assert.yaml index 09d039dc..9e4d0517 100644 --- a/tests/kuttl/tests/redis/02-assert.yaml +++ b/tests/kuttl/tests/redis/02-assert.yaml @@ -46,6 +46,10 @@ status: reason: Ready status: "True" type: ServiceConfigReady + - message: Input data complete + reason: Ready + status: "True" + type: TLSInputReady --- apiVersion: apps/v1 kind: StatefulSet diff --git a/tests/kuttl/tests/redis/06-assert.yaml b/tests/kuttl/tests/redis/06-assert.yaml new file mode 100644 index 00000000..2c976ecc --- /dev/null +++ b/tests/kuttl/tests/redis/06-assert.yaml @@ -0,0 +1,71 @@ +apiVersion: redis.openstack.org/v1beta1 +kind: Redis +metadata: + name: redis +spec: + replicas: 3 +status: + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Deployment completed + reason: Ready + status: "True" + type: DeploymentReady + - message: Exposing service completed + reason: Ready + status: "True" + type: ExposeServiceReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady + - message: Service config create completed + reason: Ready + status: "True" + type: ServiceConfigReady + - message: Input data complete + reason: Ready + status: "True" + type: TLSInputReady +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis-redis +spec: + replicas: 3 +status: + availableReplicas: 3 + readyReplicas: 3 + replicas: 3 +--- +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: 6379 + selector: + service: redis + owner: redis + redis/master: "true" +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: redis diff --git a/tests/kuttl/tests/redis/06-redis-tls.yaml b/tests/kuttl/tests/redis/06-redis-tls.yaml new file mode 100644 index 00000000..9828f30e --- /dev/null +++ b/tests/kuttl/tests/redis/06-redis-tls.yaml @@ -0,0 +1,17 @@ +# delete the previous 2-node redis if it exists +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: redis.openstack.org/v1beta1 + kind: Redis + name: redis +--- +apiVersion: redis.openstack.org/v1beta1 +kind: Redis +metadata: + name: redis +spec: + replicas: 3 + tls: + secretName: kuttl-redis-tls + caBundleSecretName: kuttl-redis-tls diff --git a/tests/kuttl/tests/redis/06-tls-certificate.yaml b/tests/kuttl/tests/redis/06-tls-certificate.yaml new file mode 100644 index 00000000..c96fd813 --- /dev/null +++ b/tests/kuttl/tests/redis/06-tls-certificate.yaml @@ -0,0 +1,45 @@ +# hardcode the secret generated by the certificate CR below, +# to avoid kuttl from depending on cert-manager at runtime +# the ca.crt key has been renamed to tls-ca-bundle.pem +# --- +# apiVersion: cert-manager.io/v1 +# kind: Certificate +# metadata: +# name: kuttl-redis-cert +# spec: +# secretName: kuttl-redis-tls +# duration: 12720h +# renewBefore: 1h +# subject: +# organizations: +# - cluster.local +# commonName: openstack-redis +# isCA: false +# privateKey: +# algorithm: RSA +# encoding: PKCS8 +# size: 2048 +# usages: +# - server auth +# - client auth +# dnsNames: +# - "redis.openstack.svc" +# - "redis.openstack.svc.cluster.local" +# - "*.redis-redis" +# - "*.redis-redis.openstack" +# - "*.redis-redis.openstack.svc" +# - "*.redis-redis.openstack.svc.cluster" +# - "*.redis-redis.openstack.svc.cluster.local" +# issuerRef: +# name: kuttl-ca-issuer +# group: cert-manager.io +# kind: Issuer +# --- +apiVersion: v1 +kind: Secret +metadata: + name: kuttl-redis-tls +data: + tls-ca-bundle.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJmRENDQVNLZ0F3SUJBZ0lRSlUrQTRBYUpDRFowRDYzbDF5UXNFVEFLQmdncWhrak9QUVFEQWpBZU1Sd3cKR2dZRFZRUURFeE5yZFhSMGJDMXpaV3htYzJsbmJtVmtMV05oTUI0WERUSTBNREV4T0RFek1UazBNbG9YRFRJMQpNRGN3TVRFek1UazBNbG93SGpFY01Cb0dBMVVFQXhNVGEzVjBkR3d0YzJWc1puTnBaMjVsWkMxallUQlpNQk1HCkJ5cUdTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCSTJKUWdYME1oZUNHSjQ2OHFSNE9wMGJYWGFuTWZSMWRpd3EKR1VtcXlrM20vdHVNZ2hxZlJNNmdWYXFpekNLMjQyNjJUL2dIamdsaDNJTEQ4UnByQXFlalFqQkFNQTRHQTFVZApEd0VCL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlFMbThjamY3dmc5ZjRxCjdsMzVmN1YxUXNsRDlqQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpQUZkdEhUbkdiMWtQVlJlZmcvbmNHaThoR2UKVlh5UVZycFJjRStNSXZMeUpRSWhBS2VLZHNleE9LUElQSDVOT0VBUHNxOTQ5cWlFVHU4ZlJEVUdkanozSkZSKwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURTVENDQXUrZ0F3SUJBZ0lSQVBDL0JSbFZNQ1VKTW52WFZRVittc0V3Q2dZSUtvWkl6ajBFQXdJd0hqRWMKTUJvR0ExVUVBeE1UYTNWMGRHd3RjMlZzWm5OcFoyNWxaQzFqWVRBZUZ3MHlOREF4TXpFd09USTFNRFZhRncweQpOVEEzTVRRd09USTFNRFZhTURJeEZqQVVCZ05WQkFvVERXTnNkWE4wWlhJdWJHOWpZV3d4R0RBV0JnTlZCQU1UCkQyOXdaVzV6ZEdGamF5MXlaV1JwY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUIKQUwrU29pVWZ5eldmRjhxdURGcGFqS1NVR2hNS0I3dGJkRU9tNWpPWWtLL0VZV3d0K3dTN3ZJbU0rZUVBS3FvcApoQVFkOUQrL0I0ZFY0QkhjUjBGdVg0eXhBZWxEc05hNWN0UEswUGcwdkpqZ1JJeUZ5a29MSEtrQVIvM1RqYUR3CmxuUk5ZRDVjSUc0YUxkdkcwSkF4elhDZ1k0bkFPdVJObzQ0MXNsTDJtZkRXdi9FbFdTZFlvU0lQUk40Ly83K0YKcEdOcHkxUzkyY0ZxNVRaRFgwSlFFU20vcmRNWko5TGwrdlBJd0RFU2VpQm85bE5MWkhlVnEwcVRHM1h6QVVWUQp2aHY1ekNIc052bklxZ1BlNC9RQlk3VFFSUDNML215bUNSNEo1ZUFoUDE0N2RxRUVaY0hrQWhIdkFvUVcvTXNRCnhCMFVDMzRBUGdZZ3F0bGdPSGxuSTM4Q0F3RUFBYU9DQVMwd2dnRXBNQjBHQTFVZEpRUVdNQlFHQ0NzR0FRVUYKQndNQkJnZ3JCZ0VGQlFjREFqQU1CZ05WSFJNQkFmOEVBakFBTUI4R0ExVWRJd1FZTUJhQUZBdWJ4eU4vdStEMQovaXJ1WGZsL3RYVkN5VVAyTUlIWUJnTlZIUkVFZ2RBd2djMkNFM0psWkdsekxtOXdaVzV6ZEdGamF5NXpkbU9DCklYSmxaR2x6TG05d1pXNXpkR0ZqYXk1emRtTXVZMngxYzNSbGNpNXNiMk5oYklJTktpNXlaV1JwY3kxeVpXUnAKYzRJWEtpNXlaV1JwY3kxeVpXUnBjeTV2Y0dWdWMzUmhZMnVDR3lvdWNtVmthWE10Y21Wa2FYTXViM0JsYm5OMApZV05yTG5OMlk0SWpLaTV5WldScGN5MXlaV1JwY3k1dmNHVnVjM1JoWTJzdWMzWmpMbU5zZFhOMFpYS0NLU291CmNtVmthWE10Y21Wa2FYTXViM0JsYm5OMFlXTnJMbk4yWXk1amJIVnpkR1Z5TG14dlkyRnNNQW9HQ0NxR1NNNDkKQkFNQ0EwZ0FNRVVDSVFDbXQ1R2VuWVNWZllxYUE5dVkrUjZkbi9GUG83N0x6ZmhJaE5TMS9HbzB6d0lnTG5FaQpOQkpwRmg3YkZkcmpnY0xETE5SRm1RTkllaGR6Qm5NRnVSa0ZXMHc9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRQy9rcUlsSDhzMW54ZksKcmd4YVdveWtsQm9UQ2dlN1czUkRwdVl6bUpDdnhHRnNMZnNFdTd5SmpQbmhBQ3FxS1lRRUhmUS92d2VIVmVBUgozRWRCYmwrTXNRSHBRN0RXdVhMVHl0RDROTHlZNEVTTWhjcEtDeHlwQUVmOTA0Mmc4SlowVFdBK1hDQnVHaTNiCnh0Q1FNYzF3b0dPSndEcmtUYU9PTmJKUzlwbncxci94SlZrbldLRWlEMFRlUC8rL2hhUmphY3RVdmRuQmF1VTIKUTE5Q1VCRXB2NjNUR1NmUzVmcnp5TUF4RW5vZ2FQWlRTMlIzbGF0S2t4dDE4d0ZGVUw0Yitjd2g3RGI1eUtvRAozdVAwQVdPMDBFVDl5LzVzcGdrZUNlWGdJVDllTzNhaEJHWEI1QUlSN3dLRUZ2ekxFTVFkRkF0K0FENEdJS3JaCllEaDVaeU4vQWdNQkFBRUNnZ0VBVkwzS1Y5MnVpRE90MUl3VkRzckxOK29EZHJTVEl2K2JlR21WbnZFMzMyaGgKSi9kVytJc0xKVlZsRzNCMWJ2d2FWNi9nWVdwaExDNkNoYVFKS1JwbnpkWm00QVovYlJ4dmZOeFVmOWJrTGtQVwpUc3JINXVUdmNwcWJQZDZjNGJwSzgzdGV2WGNIS1cwUEtBN1VKMVRBYWJlcEVFQW1UT05ESEI4SW1NWlk1ajF6CithNXVocm91akROSG16Q3lna3JKQWVNRjgyZkh2RStFOFBxM3JxTjg2YWxuSkZhNTgyRmZsbmhya3hhcG9XQmEKT1VIR0hWRFlpSkJlbm52K0ZqZWVvZDhYbEk3VlhnVlpuRjNWeHBrQThwRmF1SER5R2xKU2lKMUMzbHRpc01xcQpvSmJvSFBsNWZGRGtJSDVNQzA1NkxIVEp1bTVGM3gzSGllVG81Ym01RVFLQmdRRE0yRU1TeGd4bVRpRmxsR3FxCjJ3MHdwQVFEVWUyVCtEQUNYK01vQVNMaFNZeGMxR01KZGl3WWQvM2h5cE5Iamh5Uy9aZnVGQy85WlFRREZ6dkIKU0lXb3NQYlhEazNnNGYraklhTDJXTlMzcTVkd3N2dGFId0VvWWVyVWtCUHJ3VUhNODZVQ05RUnprM0N2TVJUOAozLzdmQmRwdm0vVXJwWHBZKzdubUViWXRXd0tCZ1FEdmFlUjdFaS9OTTNYaUlMS1RyK3ExZ3lNWUEwSHBxZ0x4CkduSURObWtVajdLWUVQQmhhOHdSTjNZQmlIQ1JVdVVGS3hvaUIwa0VNZUlLS2F4ME1UdHY1cklnWng0SEk3ZksKOGFPOVl4ZlY4ekliU3RYcTcrbDNkeWZycFFTbFM0SnZ6czg3UDVMVS8rMzFXcWJCd1F5VlhBcmdKcEdsMzBRdwpJcnlSYnJjSHJRS0JnUURKOTAvcWFxby9GRG1KVmRQMXNSUklLTzVyOTVNdW1UMThtZDMxeTJrQWh1dUZlMEpLClNlRWdIdTZLZUpqTnJDZ3dKU2h1N0NpRXhkdzJ6K2x5b0habjVGTDdwbmJTaXdEcGJuaW1PdDlBV0Vad0w0ZnoKU3k1eENsbm1Ta2ZaNGlsbVViTVhnZjVwbGEwOGprQUxNeTZ2NWEyQTdWdkZOTnAwY1h6UWdoWUVrUUtCZ0JTbQpxS3VvL1BDUlVNakprejNEL2RYY1V1bWVWbEFtZHd4L0FIaWdkOTNyS3plTXRuOWd3Z0IyTFRxaW56c1owR0ZxCnYxMVNEWTFNRkRoV01lYnYzRFdoeTVtWjYzQW9ONUZNMkpmY2RWRGlJbDlTVEROd3NFMjZ2SG5LQ1NXTTV0cG8KRjEzLzlOVmtvZ3o0M2N0MnNIUXR0VTV5WlR2T2oxNHJrT0ptajJrZEFvR0JBS05tK2RjZUlGWDI4OHpzc1FGZgpWK1d3c0RDUzVKOGs2eEFlNVJ6OXNQZ2hXbzVvYUJDRkNZQVZ2dkhiZm1tTmMrQVBOSjlMeTI5R1c1N0ZtN0haCmF5YkkrY3dPS2hOQXJlWGlnQTBTVmx2SUNSeDhyYWpXYVFMY3JBMFR0ODl2aEg1SjcvVVJOcFVwcXpzZDhySFUKZTR5RHFTdHhraXJUa2ZZY3lZSlFjUDRaCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K diff --git a/tests/kuttl/tests/redis/07-assert.yaml b/tests/kuttl/tests/redis/07-assert.yaml new file mode 100644 index 00000000..82c9a03a --- /dev/null +++ b/tests/kuttl/tests/redis/07-assert.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: | + set -e + # ensure redis endpoint is exposed over TLS + TLSDATA=$(oc rsh -n ${NAMESPACE} -c redis redis-redis-0 /bin/sh -c 'echo | openssl s_client -connect redis.'${NAMESPACE}'.svc.cluster.local --port 6379 2>&1') + echo "$TLSDATA" | grep -w CONNECTED + # ensure redis is properly clustered + SENTINELDATA=$(oc rsh -n ${NAMESPACE} -c sentinel redis-redis-0 redis-cli --tls -p 26379 info | grep master | tr ',' '\n') + # there should be 1 master + echo "$SENTINELDATA" | grep -w sentinel_masters:1 + # there should be 2 slaves + echo "$SENTINELDATA" | grep -w slaves=2 + # there should be 3 connected sentinels for quorum + echo "$SENTINELDATA" | grep -w sentinels=3