Skip to content

Commit

Permalink
Add support for clustered Redis
Browse files Browse the repository at this point in the history
Redis 6 comes with sentinel, which allows Redis to run as a
A/P service with automatic failover.

Update the redis controller to support deploying Redis as a
standalone 1-pod Redis service, or a 3-pods A/P clustered
service. Each redis pod hosts a container that runs the redis
server, and another container that runs a sentinel for the
quorated cluster management.
  • Loading branch information
dciabrin committed Jan 4, 2024
1 parent f37affc commit 85228ed
Show file tree
Hide file tree
Showing 21 changed files with 1,044 additions and 13 deletions.
100 changes: 89 additions & 11 deletions controllers/redis/redis_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package redis

import (
"context"
"fmt"
"time"

"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -28,7 +29,10 @@ import (

"github.com/go-logr/logr"
redisv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1"
"github.com/openstack-k8s-operators/lib-common/modules/common/configmap"
"github.com/openstack-k8s-operators/lib-common/modules/common/env"
"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
"github.com/openstack-k8s-operators/lib-common/modules/common/util"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand All @@ -37,9 +41,11 @@ import (

redis "github.com/openstack-k8s-operators/infra-operator/pkg/redis"
condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
commondeployment "github.com/openstack-k8s-operators/lib-common/modules/common/deployment"

common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac"
commonservice "github.com/openstack-k8s-operators/lib-common/modules/common/service"
commonstatefulset "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

// GetLogger returns a logger object with a prefix of "controller.name" and additional controller context fields
Expand Down Expand Up @@ -133,6 +139,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct
cl := condition.CreateList(
// endpoint for adoption redirect
condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage),
// configmap generation
condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage),
// redis pods ready
condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage),
// service account, role, rolebinding conditions
Expand Down Expand Up @@ -172,6 +180,36 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct
return rbacResult, nil
}

// Redis config maps
configMapVars := make(map[string]env.Setter)
err = r.generateConfigMaps(ctx, helper, instance, &configMapVars)
if err != nil {
instance.Status.Conditions.Set(condition.FalseCondition(
condition.ServiceConfigReadyCondition,
condition.ErrorReason,
condition.SeverityWarning,
condition.ServiceConfigReadyErrorMessage,
err.Error()))
return ctrl.Result{}, fmt.Errorf("error calculating configmap hash: %w", err)
}
instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage)

// the headless service provides DNS entries for pods
// the name of the resource must match the name of the app selector
pkghl := redis.HeadlessService(instance)
headless := &corev1.Service{ObjectMeta: pkghl.ObjectMeta}
_, err = controllerutil.CreateOrPatch(ctx, r.Client, headless, func() error {
headless.Spec = pkghl.Spec
err := controllerutil.SetOwnerReference(instance, headless, r.Client.Scheme())
if err != nil {
return err
}
return nil
})
if err != nil {
return ctrl.Result{}, err
}

// Service to expose Redis pods
commonsvc, err := commonservice.NewService(redis.Service(instance), time.Duration(5)*time.Second, nil)
if err != nil {
Expand All @@ -195,30 +233,70 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct
}
instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage)

// Deployment
commondeployment := commondeployment.NewDeployment(redis.Deployment(instance), time.Duration(5)*time.Second)
sfres, sferr := commondeployment.CreateOrPatch(ctx, helper)
if sferr != nil {
return sfres, sferr
}
deployment := commondeployment.GetDeployment()

//
// Reconstruct the state of the redis resource based on the deployment and its pods
//

if deployment.Status.ReadyReplicas > 0 {
// Statefulset
commonstatefulset := commonstatefulset.NewStatefulSet(redis.StatefulSet(instance), 5)
sfres, sferr := commonstatefulset.CreateOrPatch(ctx, helper)
if sferr != nil {
return sfres, sferr
}
statefulset := commonstatefulset.GetStatefulSet()

if statefulset.Status.ReadyReplicas > 0 {
instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage)
}

return ctrl.Result{}, nil
}

// generateConfigMaps returns the config map resource for a galera instance
func (r *Reconciler) generateConfigMaps(
ctx context.Context,
h *helper.Helper,
instance *redisv1beta1.Redis,
envVars *map[string]env.Setter,
) error {
templateParameters := make(map[string]interface{})
customData := make(map[string]string)

cms := []util.Template{
// ScriptsConfigMap
{
Name: fmt.Sprintf("%s-scripts", instance.Name),
Namespace: instance.Namespace,
Type: util.TemplateTypeScripts,
InstanceType: instance.Kind,
Labels: map[string]string{},
},
// ConfigMap
{
Name: fmt.Sprintf("%s-config-data", instance.Name),
Namespace: instance.Namespace,
Type: util.TemplateTypeConfig,
InstanceType: instance.Kind,
CustomData: customData,
ConfigOptions: templateParameters,
Labels: map[string]string{},
},
}

err := configmap.EnsureConfigMaps(ctx, h, instance, cms, envVars)
if err != nil {
util.LogErrorForObject(h, err, "Unable to retrieve or create config maps", instance)
return err
}

return nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&redisv1beta1.Redis{}).
Owns(&appsv1.Deployment{}).
Owns(&appsv1.StatefulSet{}).
Owns(&corev1.Service{}).
Owns(&corev1.ServiceAccount{}).
Owns(&rbacv1.Role{}).
Expand Down
32 changes: 30 additions & 2 deletions pkg/redis/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels"
service "github.com/openstack-k8s-operators/lib-common/modules/common/service"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Service exposes redis pods for a redis CR
Expand All @@ -19,16 +20,43 @@ func Service(m *redisv1beta1.Redis) *corev1.Service {
Namespace: m.GetNamespace(),
Labels: labels,
Selector: map[string]string{
"app": "redis",
"app": "redis",
"cr": "redis-redis",
"redis/master": "true",
},
Port: service.GenericServicePort{
Name: "redis",
Port: 6379,
Protocol: "TCP",
},
ClusterIP: "None",
}

svc := service.GenericService(details)
return svc
}

// HeadlessService - service to give redis pods connectivity via DNS
func HeadlessService(m *redisv1beta1.Redis) *corev1.Service {
dep := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: m.GetName() + "-" + "redis",
Namespace: m.GetNamespace(),
},
Spec: corev1.ServiceSpec{
Type: "ClusterIP",
ClusterIP: "None",
Ports: []corev1.ServicePort{
{Name: "redis", Protocol: "TCP", Port: 6379},
{Name: "sentinel", Protocol: "TCP", Port: 26379},
},
Selector: map[string]string{
"app": "redis",
"cr": "redis-redis",
},
// This is required to let pod communicate when
// they are still in Starting state
PublishNotReadyAddresses: true,
},
}
return dep
}
134 changes: 134 additions & 0 deletions pkg/redis/statefulset.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package redis

import (
"strconv"

redisv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/redis/v1beta1"
labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)

// Deployment returns a Deployment resource for the Redis CR
func StatefulSet(r *redisv1beta1.Redis) *appsv1.StatefulSet {
matchls := map[string]string{
"app": "redis",
"cr": "redis-" + r.Name,
"owner": "infra-operator",
}
ls := labels.GetLabels(r, "redis", matchls)

livenessProbe := &corev1.Probe{
// TODO might need tuning
TimeoutSeconds: 5,
PeriodSeconds: 3,
InitialDelaySeconds: 3,
}
readinessProbe := &corev1.Probe{
// TODO might need tuning
TimeoutSeconds: 5,
PeriodSeconds: 5,
InitialDelaySeconds: 5,
}
sentinelLivenessProbe := &corev1.Probe{
// TODO might need tuning
TimeoutSeconds: 5,
PeriodSeconds: 3,
InitialDelaySeconds: 3,
}
sentinelReadinessProbe := &corev1.Probe{
// TODO might need tuning
TimeoutSeconds: 5,
PeriodSeconds: 5,
InitialDelaySeconds: 5,
}

// TODO might want to disable probes in 'Debug' mode
livenessProbe.TCPSocket = &corev1.TCPSocketAction{
Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(6379)},
}
readinessProbe.TCPSocket = &corev1.TCPSocketAction{
Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(6379)},
}
sentinelLivenessProbe.TCPSocket = &corev1.TCPSocketAction{
Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(26379)},
}
sentinelReadinessProbe.TCPSocket = &corev1.TCPSocketAction{
Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(26379)},
}
name := r.Name + "-" + "redis"
sts := &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: r.Namespace,
},
Spec: appsv1.StatefulSetSpec{
ServiceName: name,
Replicas: r.Spec.Replicas,
Selector: &metav1.LabelSelector{
MatchLabels: ls,
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: ls,
},
Spec: corev1.PodSpec{
ServiceAccountName: r.RbacResourceName(),
Containers: []corev1.Container{{
Image: r.Spec.ContainerImage,
Command: []string{"/var/lib/operator-scripts/start_redis_replication.sh"},
Name: "redis",
Env: []corev1.EnvVar{{
Name: "KOLLA_CONFIG_STRATEGY",
Value: "COPY_ALWAYS",
}},
VolumeMounts: getRedisVolumeMounts(),
Ports: []corev1.ContainerPort{{
ContainerPort: 6379,
Name: "redis",
}},
LivenessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
Exec: &corev1.ExecAction{
Command: []string{"/var/lib/operator-scripts/redis_probe.sh", "liveness"},
},
},
},
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
Exec: &corev1.ExecAction{
Command: []string{"/var/lib/operator-scripts/redis_probe.sh", "readiness"},
},
},
},
}, {
Image: r.Spec.ContainerImage,
Command: []string{"/var/lib/operator-scripts/start_sentinel.sh"},

Name: "sentinel",
Env: []corev1.EnvVar{{
Name: "SENTINEL_QUORUM",
Value: strconv.Itoa((int(*r.Spec.Replicas) / 2) + 1),
}, {
Name: "KOLLA_CONFIG_STRATEGY",
Value: "COPY_ALWAYS",
}},
VolumeMounts: getSentinelVolumeMounts(),
Ports: []corev1.ContainerPort{{
ContainerPort: 26379,
Name: "sentinel",
}},
ReadinessProbe: sentinelReadinessProbe,
LivenessProbe: sentinelLivenessProbe,
},
},
Volumes: getVolumes(r),
},
},
},
}

return sts
}
Loading

0 comments on commit 85228ed

Please sign in to comment.