Skip to content

Commit

Permalink
Merge pull request #1757 from rksharma95/fix-controller
Browse files Browse the repository at this point in the history 
fix(operator): fix tls rotation logic for controller
  • Loading branch information
@DelusionalOptimist
DelusionalOptimist authored May 29, 2024
2 parents 0eaec66 + 69ce963 commit bbdc04d
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 102 deletions.
2 changes: 1 addition & 1 deletion pkg/KubeArmorOperator/internal/controller/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -701,7 +701,7 @@ func (clusterWatcher *ClusterWatcher) DeleteAllTlsSecrets() error {
for _, secret := range tlsSecrets {
err := clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Delete(context.Background(), secret, v1.DeleteOptions{})
if err != nil {
clusterWatcher.Log.Errorf("error while deleing secret: %s", secret)
clusterWatcher.Log.Errorf("error while deleing secret: %s, error=%s", secret, err.Error())
return err
}
}
Expand Down
147 changes: 46 additions & 101 deletions pkg/KubeArmorOperator/internal/controller/resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs,
vols = append(vols, common.KubeArmorCaVolume...)
volMnts = append(volMnts, common.KubeArmorCaVolumeMount...)
common.AddOrReplaceArg("-tlsEnabled=true", "-tlsEnabled=false", &daemonset.Spec.Template.Spec.Containers[0].Args)
} else {
common.AddOrReplaceArg("-tlsEnabled=false", "-tlsEnabled=true", &daemonset.Spec.Template.Spec.Containers[0].Args)
}
daemonset.Spec.Template.Spec.Volumes = vols
daemonset.Spec.Template.Spec.InitContainers[0].VolumeMounts = commonVolMnts
Expand Down Expand Up @@ -426,7 +428,6 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
var caCert, tlsCrt, tlsKey *bytes.Buffer
var kGenErr, err, installErr error
RotateTls := false
FirstRun := true
srvAccs := []*corev1.ServiceAccount{
addOwnership(deployments.GetServiceAccount(common.Namespace)).(*corev1.ServiceAccount),
addOwnership(deployments.GetRelayServiceAccount(common.Namespace)).(*corev1.ServiceAccount),
Expand Down Expand Up @@ -486,6 +487,8 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
relayServer.Spec.Template.Spec.Volumes =
append(relayServer.Spec.Template.Spec.Volumes, common.KubeArmorRelayTlsVolume...)
common.AddOrReplaceArg("-tlsEnabled=true", "-tlsEnabled=false", &relayServer.Spec.Template.Spec.Containers[0].Args)
} else {
common.AddOrReplaceArg("-tlsEnabled=false", "-tlsEnabled=true", &relayServer.Spec.Template.Spec.Containers[0].Args)
}

// update images
Expand Down Expand Up @@ -631,10 +634,7 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
installErr = err
clusterWatcher.Log.Warnf("Cannot create secret %s, error=%s", secret.Name, err.Error())
} else {
RotateTls = true && !FirstRun
if !FirstRun {
clusterWatcher.Log.Warnf("Secret was created, rotating TLS secrets")
}
caInK8sSecret = secret.Data["ca.crt"]
}
} else if err == nil {
caInK8sSecret = s.Data["ca.crt"]
Expand Down Expand Up @@ -672,10 +672,7 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
} else if err == nil {
if !bytes.Equal(hook.Webhooks[0].ClientConfig.CABundle, caInK8sSecret) {
// rotate
RotateTls = true && !FirstRun
if !FirstRun {
clusterWatcher.Log.Warnf("mutation CA cert does not match secret CA cert, rotating tls secrets")
}
RotateTls = true
}
} else {
installErr = err
Expand All @@ -698,107 +695,55 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
clusterWatcher.RotateTlsCerts()
RotateTls = false
}
if FirstRun {
FirstRun = false
}
time.Sleep(10 * time.Second)
}
}

func (clusterWatcher *ClusterWatcher) RotateTlsCerts() {
var suffix string
var caCert, tlsCrt, tlsKey *bytes.Buffer
var err error
retries := 0
for {
if retries == 3 {
return
}
retries++
suffix, err = common.GetFreeRandSuffix(clusterWatcher.Client, common.Namespace)
if err == nil {
clusterWatcher.Log.Infof("Using suffix %s for all new temorary resources", suffix)
break
}
clusterWatcher.Log.Infof("Cannot find a suffix, err=%s, retrying in 3 seconds...", err.Error())
time.Sleep(3 * time.Second)
}
serviceName := deployments.KubeArmorControllerWebhookServiceName + "-" + suffix
for {
caCert, tlsCrt, tlsKey, err = common.GeneratePki(common.Namespace, serviceName)
if err == nil {
break
}
clusterWatcher.Log.Infof("Couldn't generate TLS secret, retrying in 3 seconds")
time.Sleep(3 * time.Second)
}
tmpsecret := deployments.GetKubeArmorControllerTLSSecret(common.Namespace, caCert.String(), tlsCrt.String(), tlsKey.String())
tmpsecret = addOwnership(tmpsecret).(*corev1.Secret)
tmpsecret.Name = tmpsecret.GetName() + "-" + suffix
_, err = clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Create(context.Background(), tmpsecret, metav1.CreateOptions{})

origdeploy, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.KubeArmorControllerDeploymentName, metav1.GetOptions{})
if err != nil {
clusterWatcher.Log.Warnf("Cannot create secret %s, error=%s", tmpsecret.Name, err.Error())
}
tmpdeploy := deployments.GetKubeArmorControllerDeployment(common.Namespace)
tmpdeploy = addOwnership(tmpdeploy).(*appsv1.Deployment)
tmpdeploy.Name = tmpdeploy.GetName() + "-" + suffix
for i, s := range tmpdeploy.Spec.Template.Spec.Volumes {
if s.Name == "cert" {
s.Secret.SecretName = tmpsecret.GetName()
tmpdeploy.Spec.Template.Spec.Volumes[i] = s
break
}
clusterWatcher.Log.Warnf("cannot get controller deployment, error=%s", err.Error())
}
selectLabels := tmpdeploy.Spec.Selector.MatchLabels
selectLabels["kubearmor-app"] = suffix
tmpdeploy.Spec.Selector.MatchLabels = selectLabels
origdeploy, _ := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.KubeArmorControllerDeploymentName, metav1.GetOptions{})
origdeploy = addOwnership(origdeploy).(*appsv1.Deployment)
tmpdeploy.Spec.Replicas = origdeploy.Spec.Replicas
if _, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Create(context.Background(), tmpdeploy, metav1.CreateOptions{}); err != nil {
clusterWatcher.Log.Warnf("Cannot create deployment %s, error=%s", tmpdeploy.Name, err.Error())
}

time.Sleep(10 * time.Second)

tmpservice := deployments.GetKubeArmorControllerWebhookService(common.Namespace)
tmpservice = addOwnership(tmpservice).(*corev1.Service)
tmpservice.Name = serviceName
tmpservice.Spec.Selector = selectLabels
if _, err := clusterWatcher.Client.CoreV1().Services(common.Namespace).Create(context.Background(), tmpservice, metav1.CreateOptions{}); err != nil {
clusterWatcher.Log.Warnf("Cannot create deployment %s, error=%s", tmpservice.Name, err.Error())
}
tmpmutation := deployments.GetKubeArmorControllerMutationAdmissionConfiguration(common.Namespace, caCert.Bytes())
mutationName := tmpmutation.Name
tmpmutation = addOwnership(tmpmutation).(*v1.MutatingWebhookConfiguration)
tmpmutation.Name = tmpmutation.Name + "-" + suffix
tmpmutation.Webhooks[0].ClientConfig.Service.Name = tmpservice.GetName()
if _, err := clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), tmpmutation, metav1.CreateOptions{}); err != nil {
clusterWatcher.Log.Warnf("Cannot create mutation webhook %s, error=%s", tmpmutation.Name, err.Error())
}
clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), mutationName, metav1.DeleteOptions{})

caCert, tlsCrt, tlsKey, _ = common.GeneratePki(common.Namespace, deployments.KubeArmorControllerWebhookServiceName)
secret := deployments.GetKubeArmorControllerTLSSecret(common.Namespace, caCert.String(), tlsCrt.String(), tlsKey.String())
secret = addOwnership(secret).(*corev1.Secret)
clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Update(context.Background(), secret, metav1.UpdateOptions{})

replicas := int32(0)
origdeploy.Spec.Replicas = &replicas
clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), origdeploy, metav1.UpdateOptions{})
time.Sleep(10 * time.Second)
origdeploy, _ = clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.KubeArmorControllerDeploymentName, metav1.GetOptions{})
origdeploy = addOwnership(origdeploy).(*appsv1.Deployment)
origdeploy.Spec.Replicas = tmpdeploy.Spec.Replicas

clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), origdeploy, metav1.UpdateOptions{})
mutation := deployments.GetKubeArmorControllerMutationAdmissionConfiguration(common.Namespace, caCert.Bytes())
mutation = addOwnership(mutation).(*v1.MutatingWebhookConfiguration)

clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), mutation, metav1.CreateOptions{})

clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), tmpmutation.Name, metav1.DeleteOptions{})
clusterWatcher.Client.CoreV1().Services(common.Namespace).Delete(context.Background(), tmpservice.Name, metav1.DeleteOptions{})
clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Delete(context.Background(), tmpdeploy.Name, metav1.DeleteOptions{})
clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Delete(context.Background(), tmpsecret.Name, metav1.DeleteOptions{})
replicas := origdeploy.Spec.Replicas

// TODO: Keep CA certificate in k8s secret

// == CLEANUP ==
// scale down controller deployment to 0
controllerDeployment := deployments.GetKubeArmorControllerDeployment(common.Namespace)
controllerDeployment = addOwnership(controllerDeployment).(*appsv1.Deployment)
zeroReplicas := int32(0)
controllerDeployment.Spec.Replicas = &zeroReplicas
if _, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), controllerDeployment, metav1.UpdateOptions{}); err != nil {
clusterWatcher.Log.Warnf("cannot scale down controller %s, error=%s", controllerDeployment.Name, err.Error())
}
// delete mutation webhook configuration
mutationWebhook := deployments.GetKubeArmorControllerMutationAdmissionConfiguration(common.Namespace, caCert.Bytes())
mutationWebhook = addOwnership(mutationWebhook).(*v1.MutatingWebhookConfiguration)
if err := clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), mutationWebhook.Name, metav1.DeleteOptions{}); err != nil {
clusterWatcher.Log.Warnf("cannot delete mutation webhook %s, error=%s", mutationWebhook.Name, err.Error())
}
// == ROTATE ==
// update controller tls secret
controllerSecret := deployments.GetKubeArmorControllerTLSSecret(common.Namespace, caCert.String(), tlsCrt.String(), tlsKey.String())
controllerSecret = addOwnership(controllerSecret).(*corev1.Secret)
if _, err := clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Update(context.Background(), controllerSecret, metav1.UpdateOptions{}); err != nil {
clusterWatcher.Log.Warnf("cannot update controller tls secret %s, error=%s", controllerSecret.Name, err.Error())
}
// == ROLLOUT ==
// create mutation webhook configuration
if _, err := clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), mutationWebhook, metav1.CreateOptions{}); err != nil {
clusterWatcher.Log.Warnf("Cannot create mutation webhook %s, error=%s", mutationWebhook.Name, err.Error())
}
// scale up controller deployment to previous settings
controllerDeployment.Spec.Replicas = replicas
if _, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), controllerDeployment, metav1.UpdateOptions{}); err != nil {
clusterWatcher.Log.Warnf("cannot scale down controller %s, error=%s", controllerDeployment.Name, err.Error())
}
clusterWatcher.Log.Info("Tls rotation completed")
}

0 comments on commit bbdc04d

Please sign in to comment.