Add support for clustered Redis

Redis 6 comes with sentinel, which allows Redis to run as a
A/P service with automatic failover.

Update the redis controller to support deploying Redis as a
standalone 1-pod Redis service, or a 3-pods A/P clustered
service. Each redis pod hosts a container that runs the redis
server, and another container that runs a sentinel for the
quorated cluster management.

Also changed the volume mounts to prepare for TLS support.

apis/redis/v1beta1/redis_types.go

@@ -26,7 +26,7 @@ const (
 	// Container image fall-back defaults
 
 	// RedisContainerImage is the fall-back container image for Redis
-	RedisContainerImage = "registry.redhat.io/rhel9/redis-6:latest"
+	RedisContainerImage = "quay.io/podified-antelope-centos9/openstack-redis:current-podified"
 )
 
 // RedisSpec defines the desired state of Redis

config/default/manager_default_images.yaml

@@ -14,7 +14,7 @@ spec:
         - name: RELATED_IMAGE_INFRA_MEMCACHED_IMAGE_URL_DEFAULT
           value: quay.io/podified-antelope-centos9/openstack-memcached:current-podified
         - name: RELATED_IMAGE_INFRA_REDIS_IMAGE_URL_DEFAULT
-          value: registry.redhat.io/rhel9/redis-6:latest
+          value: quay.io/podified-antelope-centos9/openstack-redis:current-podified
         # TODO create its own container image, instead of using neutron one
         - name: RELATED_IMAGE_INFRA_DNSMASQ_IMAGE_URL_DEFAULT
           value: quay.io/podified-antelope-centos9/openstack-neutron-server:current-podified

controllers/redis/redis_controller.go

@@ -18,6 +18,7 @@ package redis
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"k8s.io/apimachinery/pkg/runtime"
@@ -28,7 +29,10 @@ import (
 
 	"github.com/go-logr/logr"
 	redisv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/configmap"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/util"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -37,14 +41,16 @@ import (
 
 	redis "github.com/openstack-k8s-operators/infra-operator/pkg/redis"
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
-	commondeployment "github.com/openstack-k8s-operators/lib-common/modules/common/deployment"
+
 	common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac"
 	commonservice "github.com/openstack-k8s-operators/lib-common/modules/common/service"
+	commonstatefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
 // GetLogger returns a logger object with a prefix of "controller.name" and additional controller context fields
 func (r *Reconciler) GetLogger(ctx context.Context) logr.Logger {
-	return log.FromContext(ctx).WithName("Controllers").WithName("DNSData")
+	return log.FromContext(ctx).WithName("Controllers").WithName("Redis")
 }
 
 // Reconciler reconciles a Redis object
@@ -133,6 +139,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct
 		cl := condition.CreateList(
 			// endpoint for adoption redirect
 			condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage),
+			// configmap generation
+			condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage),
 			// redis pods ready
 			condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage),
 			// service account, role, rolebinding conditions
@@ -172,6 +180,36 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct
 		return rbacResult, nil
 	}
 
+	// Redis config maps
+	configMapVars := make(map[string]env.Setter)
+	err = r.generateConfigMaps(ctx, helper, instance, &configMapVars)
+	if err != nil {
+		instance.Status.Conditions.Set(condition.FalseCondition(
+			condition.ServiceConfigReadyCondition,
+			condition.ErrorReason,
+			condition.SeverityWarning,
+			condition.ServiceConfigReadyErrorMessage,
+			err.Error()))
+		return ctrl.Result{}, fmt.Errorf("error calculating configmap hash: %w", err)
+	}
+	instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage)
+
+	// the headless service provides DNS entries for pods
+	// the name of the resource must match the name of the app selector
+	pkghl := redis.HeadlessService(instance)
+	headless := &corev1.Service{ObjectMeta: pkghl.ObjectMeta}
+	_, err = controllerutil.CreateOrPatch(ctx, r.Client, headless, func() error {
+		headless.Spec = pkghl.Spec
+		err := controllerutil.SetOwnerReference(instance, headless, r.Client.Scheme())
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
 	// Service to expose Redis pods
 	commonsvc, err := commonservice.NewService(redis.Service(instance), time.Duration(5)*time.Second, nil)
 	if err != nil {
@@ -195,30 +233,70 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct
 	}
 	instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage)
 
-	// Deployment
-	commondeployment := commondeployment.NewDeployment(redis.Deployment(instance), time.Duration(5)*time.Second)
-	sfres, sferr := commondeployment.CreateOrPatch(ctx, helper)
-	if sferr != nil {
-		return sfres, sferr
-	}
-	deployment := commondeployment.GetDeployment()
-
 	//
 	// Reconstruct the state of the redis resource based on the deployment and its pods
 	//
 
-	if deployment.Status.ReadyReplicas > 0 {
+	// Statefulset
+	commonstatefulset := commonstatefulset.NewStatefulSet(redis.StatefulSet(instance), 5)
+	sfres, sferr := commonstatefulset.CreateOrPatch(ctx, helper)
+	if sferr != nil {
+		return sfres, sferr
+	}
+	statefulset := commonstatefulset.GetStatefulSet()
+
+	if statefulset.Status.ReadyReplicas > 0 {
 		instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage)
 	}
 
 	return ctrl.Result{}, nil
 }
 
+// generateConfigMaps returns the config map resource for a galera instance
+func (r *Reconciler) generateConfigMaps(
+	ctx context.Context,
+	h *helper.Helper,
+	instance *redisv1beta1.Redis,
+	envVars *map[string]env.Setter,
+) error {
+	templateParameters := make(map[string]interface{})
+	customData := make(map[string]string)
+
+	cms := []util.Template{
+		// ScriptsConfigMap
+		{
+			Name:         fmt.Sprintf("%s-scripts", instance.Name),
+			Namespace:    instance.Namespace,
+			Type:         util.TemplateTypeScripts,
+			InstanceType: instance.Kind,
+			Labels:       map[string]string{},
+		},
+		// ConfigMap
+		{
+			Name:          fmt.Sprintf("%s-config-data", instance.Name),
+			Namespace:     instance.Namespace,
+			Type:          util.TemplateTypeConfig,
+			InstanceType:  instance.Kind,
+			CustomData:    customData,
+			ConfigOptions: templateParameters,
+			Labels:        map[string]string{},
+		},
+	}
+
+	err := configmap.EnsureConfigMaps(ctx, h, instance, cms, envVars)
+	if err != nil {
+		util.LogErrorForObject(h, err, "Unable to retrieve or create config maps", instance)
+		return err
+	}
+
+	return nil
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&redisv1beta1.Redis{}).
-		Owns(&appsv1.Deployment{}).
+		Owns(&appsv1.StatefulSet{}).
 		Owns(&corev1.Service{}).
 		Owns(&corev1.ServiceAccount{}).
 		Owns(&rbacv1.Role{}).

go.mod

@@ -87,3 +87,6 @@ replace github.com/openstack-k8s-operators/infra-operator/apis => ./apis
 // mschuppert: map to latest commit from release-4.13 tag
 // must consistent within modules and service operators
 replace github.com/openshift/api => github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 //allow-merging
+
+// WIP: PublishNotReadyAddresses in service
+replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/dciabrin/lib-common/modules/common v0.0.0-20240126141905-5a6e5db15cc0

go.sum

@@ -63,6 +63,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dciabrin/lib-common/modules/common v0.0.0-20240126141905-5a6e5db15cc0 h1:0U0+bTHiZGMGPBqwgJ9TGG3DrmhncvL7gtI3iDeXOqY=
+github.com/dciabrin/lib-common/modules/common v0.0.0-20240126141905-5a6e5db15cc0/go.mod h1:F2490pi067Cc3tU3b1nCJPfZ5bLpm+rwldEdMUPA0d4=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ=
 github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
@@ -232,8 +234,6 @@ github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
 github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
 github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7 h1:rncLxJBpFGqBztyxCMwNRnMjhhIDOWHJowi6q8G6koI=
 github.com/openshift/api v0.0.0-20230414143018-3367bc7e6ac7/go.mod h1:ctXNyWanKEjGj8sss1KjjHQ3ENKFm33FFnS5BKaIPh4=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240124141114-55d029e4658b h1:8tPUN0Aj4MKEltI2pv3vjy2HyxPEAYXcs6UNrz2vzm8=
-github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20240124141114-55d029e4658b/go.mod h1:F2490pi067Cc3tU3b1nCJPfZ5bLpm+rwldEdMUPA0d4=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240124141114-55d029e4658b h1:Jr6BWxwT6zCNC6TPxrKzO99te7v6phhmMRGVC9LD+nM=
 github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20240124141114-55d029e4658b/go.mod h1:ni4mvKeubWsTjKmcToJ+hIo7pJipM9hwiUv8qhm1R6Y=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

pkg/redis/service.go

@@ -2,31 +2,58 @@ package redis
 
 import (
 	redisv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1"
+	common "github.com/openstack-k8s-operators/lib-common/modules/common"
 	labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels"
 	service "github.com/openstack-k8s-operators/lib-common/modules/common/service"
 	corev1 "k8s.io/api/core/v1"
 )
 
 // Service exposes redis pods for a redis CR
-func Service(m *redisv1beta1.Redis) *corev1.Service {
-	labels := labels.GetLabels(m, "redis", map[string]string{
-		"owner": "infra-operator",
-		"cr":    m.GetName(),
-		"app":   "redis",
+func Service(instance *redisv1beta1.Redis) *corev1.Service {
+	labels := labels.GetLabels(instance, "redis", map[string]string{
+		common.AppSelector:   "redis",
+		common.OwnerSelector: instance.Name,
 	})
 	details := &service.GenericServiceDetails{
-		Name:      m.GetName(),
-		Namespace: m.GetNamespace(),
+		Name:      instance.GetName(),
+		Namespace: instance.GetNamespace(),
 		Labels:    labels,
 		Selector: map[string]string{
-			"app": "redis",
+			common.AppSelector:   "redis",
+			common.OwnerSelector: instance.Name,
+			"redis/master":       "true",
 		},
 		Port: service.GenericServicePort{
 			Name:     "redis",
 			Port:     6379,
 			Protocol: "TCP",
 		},
-		ClusterIP: "None",
+	}
+
+	svc := service.GenericService(details)
+	return svc
+}
+
+// HeadlessService - service to give redis pods connectivity via DNS
+func HeadlessService(instance *redisv1beta1.Redis) *corev1.Service {
+	labels := labels.GetLabels(instance, "redis", map[string]string{
+		common.AppSelector:   "redis",
+		common.OwnerSelector: instance.Name,
+	})
+	details := &service.GenericServiceDetails{
+		Name:      instance.GetName() + "-" + "redis",
+		Namespace: instance.GetNamespace(),
+		Labels:    labels,
+		Selector: map[string]string{
+			common.AppSelector:   "redis",
+			common.OwnerSelector: instance.Name,
+		},
+		Ports: []corev1.ServicePort{
+			{Name: "redis", Protocol: "TCP", Port: 6379},
+			{Name: "sentinel", Protocol: "TCP", Port: 26379},
+		},
+		ClusterIP:                "None",
+		PublishNotReadyAddresses: true,
 	}
 
 	svc := service.GenericService(details)