Skip to content

Commit

Permalink
feat: add watch on resolver to update endpointslice
Browse files Browse the repository at this point in the history
  • Loading branch information
ramantehlan committed Jun 6, 2024
1 parent cda02ea commit 5eebffb
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 25 deletions.
74 changes: 57 additions & 17 deletions operator/internal/controller/elastiservice_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,22 @@ type (
)

const (
ServeMode = "serve"
ProxyMode = "proxy"
NullMode = ""
ServeMode = "serve"
ProxyMode = "proxy"
NullMode = ""
resolverNamespace = "elasti"
resolverDeploymentName = "elasti-resolver"
)

var locks sync.Map

func getMutexForRequest(key string) *sync.Mutex {
func getMutexForRunReconcile(key string) *sync.Mutex {
l, _ := locks.LoadOrStore(key, &sync.Mutex{})
return l.(*sync.Mutex)
}

var watcherStartLock sync.Once

//+kubebuilder:rbac:groups=elasti.truefoundry.com,resources=elastiservices,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=elasti.truefoundry.com,resources=elastiservices/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=elasti.truefoundry.com,resources=elastiservices/finalizers,verbs=update
Expand Down Expand Up @@ -97,27 +101,63 @@ func (r *ElastiServiceReconciler) Reconcile(ctx context.Context, req ctrl.Reques
}
}

go r.Watcher.Add(es.Spec.DeploymentName, req.Namespace, cache.ResourceEventHandlerFuncs{
UpdateFunc: func(old, new interface{}) {
newDeployment := new.(*appsv1.Deployment)
condition := newDeployment.Status.Conditions
if newDeployment.Status.Replicas == 0 {
r.Logger.Debug("Deployment has 0 replicas", zap.String("deployment_name", es.Spec.DeploymentName))
r.runReconcile(context.Background(), req, ProxyMode)
} else if newDeployment.Status.Replicas > 0 && condition[1].Status == "True" {
r.Logger.Debug("Deployment has replicas", zap.String("deployment_name", es.Spec.DeploymentName))
r.runReconcile(context.Background(), req, ServeMode)
}
},
// These steps happen once once, when the ElastiService is created.
watcherStartLock.Do(func() {
go r.Watcher.AddDeploymentWatch(es.Spec.DeploymentName, req.Namespace, cache.ResourceEventHandlerFuncs{
UpdateFunc: func(old, new interface{}) {
newDeployment := new.(*appsv1.Deployment)
condition := newDeployment.Status.Conditions
if newDeployment.Status.Replicas == 0 {
r.Logger.Debug("Deployment has 0 replicas", zap.String("deployment_name", es.Spec.DeploymentName))
r.runReconcile(context.Background(), req, ProxyMode)
} else if newDeployment.Status.Replicas > 0 && condition[1].Status == "True" {
r.Logger.Debug("Deployment has replicas", zap.String("deployment_name", es.Spec.DeploymentName))
r.runReconcile(context.Background(), req, ServeMode)
}
},
})

// Watch for changes in activator deployment, we must update the service endpointslice to resolver
go r.Watcher.AddDeploymentWatch(resolverDeploymentName, resolverNamespace, cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
r.handleResolverChanges(ctx, obj, es.Spec.Service, req.Namespace)
},
UpdateFunc: func(old, new interface{}) {
r.handleResolverChanges(ctx, new, es.Spec.Service, req.Namespace)
},
DeleteFunc: func(obj interface{}) {
// TODO: Handle deletion of resolver deployment
r.Logger.Debug("Resolver deployment deleted", zap.String("deployment_name", resolverDeploymentName))
},
})
})

return r.runReconcile(ctx, req, NullMode)
}

func (r *ElastiServiceReconciler) handleResolverChanges(ctx context.Context, obj interface{}, serviceName, namespace string) {
resolverDeployment := obj.(*appsv1.Deployment)
if resolverDeployment.Name == resolverDeploymentName {
targetNamespacedName := types.NamespacedName{
Name: serviceName,
Namespace: namespace,
}
targetSVC := &v1.Service{}
if err := r.Get(ctx, targetNamespacedName, targetSVC); err != nil {
r.Logger.Error("Failed to get target service", zap.Error(err))
return
}
if err := r.CreateOrUpdateEndpointsliceToResolver(ctx, targetSVC); err != nil {
r.Logger.Error("Failed to create or update endpointslice to resolver", zap.Error(err))
return
}
}
}

func (r *ElastiServiceReconciler) runReconcile(ctx context.Context, req ctrl.Request, mode string) (res ctrl.Result, err error) {
r.Logger.Debug("- In RunReconcile", zap.String("key", req.NamespacedName.String()))
// Only 1 reconcile should run at a time for a given ElastiService. This prevents conflicts when updating different objects.
mutex := getMutexForRequest(req.NamespacedName.String())
mutex := getMutexForRunReconcile(req.NamespacedName.String())
mutex.Lock()
defer r.Logger.Debug("- Out of RunReconcile", zap.String("key", req.NamespacedName.String()))
defer mutex.Unlock()
Expand Down
11 changes: 6 additions & 5 deletions operator/internal/controller/watcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,24 +65,25 @@ func (m *InformerManager) monitorInformers() {
})
}

func (m *InformerManager) Add(deploymentName, namespace string, handlers cache.ResourceEventHandlerFuncs) {
func (m *InformerManager) AddDeploymentWatch(deploymentName, namespace string, handlers cache.ResourceEventHandlerFuncs) {
m.logger.Info("Adding Deployment informer", zap.String("deployment_name", deploymentName))
m.enableInformer(deploymentName, namespace, handlers)
}

func (m *InformerManager) enableInformer(deploymentName, namespace string, handlers cache.ResourceEventHandlerFuncs) {
key := getKey(namespace, deploymentName)
if i, ok := m.informers.Load(key); ok && i.(InformerInfo).Active {
m.logger.Info("Informer already running", zap.String("key", key))
return
}

defer func() {
if r := recover(); r != nil {
m.logger.Error("Recovered from panic", zap.Any("error", r))
m.StopInformer(deploymentName, namespace)
m.enableInformer(deploymentName, namespace, handlers)
}
}()
if i, ok := m.informers.Load(key); ok && i.(InformerInfo).Active {
m.logger.Info("Informer already running", zap.String("key", key))
return
}
informer := cache.NewSharedIndexInformer(
&cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
Expand Down
6 changes: 3 additions & 3 deletions resolver/config/deployment.yaml
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: resolver
name: elasti-resolver
namespace: elasti
spec:
replicas: 1
selector:
matchLabels:
app: resolver
app: elasti-resolver
template:
metadata:
labels:
app: resolver
app: elasti-resolver
spec:
containers:
- name: playground
Expand Down

0 comments on commit 5eebffb

Please sign in to comment.