add annotate existing resources flag

Signed-off-by: Aryan-sharma11 <[email protected]>

.github/workflows/ci-test-ginkgo.yml

@@ -133,7 +133,7 @@ jobs:
           fi
           docker system prune -a -f 
           docker buildx prune -a -f
-          helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest 
+          helm upgrade --install kubearmor-operator ./deployments/helm/KubeArmorOperator -n kubearmor --create-namespace --set kubearmorOperator.image.tag=latest  --set kubearmorOperator.annotateExisting=true
           kubectl wait --for=condition=ready --timeout=5m -n kubearmor pod -l kubearmor-app=kubearmor-operator
           kubectl get pods -A
           if [[ ${{ steps.filter.outputs.controller }} == 'true' ]]; then

deployments/get/objects.go

@@ -526,6 +526,7 @@ func GetKubeArmorControllerDeployment(namespace string) *appsv1.Deployment {
 							Args: []string{
 								"--leader-elect",
 								"--health-probe-bind-address=:8081",
+								"--annotateExisting=false",
 							},
 							Command: []string{"/manager"},
 							Ports: []corev1.ContainerPort{

deployments/helm/KubeArmor/templates/deployment.yaml

@@ -82,6 +82,7 @@ spec:
       - args:
         - --health-probe-bind-address=:8081
         - --leader-elect
+        - --anotateExisting=false
         command:
         - /manager
         image: {{printf "%s:%s" .Values.kubearmorController.image.repository .Values.kubearmorController.image.tag}}

deployments/helm/KubeArmorOperator/templates/deployment.yaml

@@ -46,6 +46,8 @@ spec:
         image: {{ include "operatorImage" . }}
         imagePullPolicy: {{ .Values.kubearmorOperator.imagePullPolicy }}
         args:
+        - --annotateExisting={{ .Values.kubearmorOperator.annotateExisting }}
+        - --annotateResource={{ .Values.kubearmorOperator.annotateResource }}
         {{- if .Values.kubearmorOperator.args -}}
           {{- toYaml .Values.kubearmorOperator.args | trim | nindent 8 }}
         {{- end }}

deployments/helm/KubeArmorOperator/values.yaml

@@ -29,6 +29,7 @@ oci_meta:
 # in case if image pinning is disabled
 kubearmorOperator:
   annotateResource: false
+  annotateExisting: false
   name: kubearmor-operator
   image:
     repository: kubearmor/kubearmor-operator

pkg/KubeArmorController/cmd/main.go

@@ -49,6 +49,7 @@ func main() {
 	var probeAddr string
 	var secureMetrics bool
 	var enableHTTP2 bool
+	var annotateExisting bool
 	var tlsOpts []func(*tls.Config)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -60,6 +61,8 @@ func main() {
 		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
+	flag.BoolVar(&annotateExisting, "annotateExisting", false,
+		"If 'true', controller will restart and annotate existing resources with required annotations")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -170,10 +173,11 @@ func main() {
 	})
 	setupLog.Info("Adding pod refresher controller")
 	if err = (&controllers.PodRefresherReconciler{
-		Client:    mgr.GetClient(),
-		Scheme:    mgr.GetScheme(),
-		Cluster:   &cluster,
-		ClientSet: client,
+		Client:           mgr.GetClient(),
+		Scheme:           mgr.GetScheme(),
+		Cluster:          &cluster,
+		ClientSet:        client,
+		AnnotateExisting: annotateExisting,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Pod")
 		os.Exit(1)

pkg/KubeArmorController/internal/controller/podrefresh_controller.go

@@ -22,9 +22,10 @@ import (
 
 type PodRefresherReconciler struct {
 	client.Client
-	Scheme    *runtime.Scheme
-	Cluster   *types.Cluster
-	ClientSet *kubernetes.Clientset
+	Scheme           *runtime.Scheme
+	Cluster          *types.Cluster
+	ClientSet        *kubernetes.Clientset
+	AnnotateExisting bool
 }
 type ResourceInfo struct {
 	kind          string
@@ -34,7 +35,12 @@ type ResourceInfo struct {
 // +kubebuilder:rbac:groups="",resources=pods,verbs=get;watch;list;create;update;delete
 
 func (r *PodRefresherReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+
 	log := log.FromContext(ctx)
+	if !r.AnnotateExisting {
+		log.Info(fmt.Sprintf("Not annotating existing resources as annotate existing is set to false"))
+		return ctrl.Result{}, nil
+	}
 	var podList corev1.PodList
 
 	if err := r.List(ctx, &podList); err != nil {
@@ -72,8 +78,6 @@ func (r *PodRefresherReconciler) Reconcile(ctx context.Context, req ctrl.Request
 			orginalPod := pod.DeepCopy()
 			common.AddCommonAnnotations(&pod.ObjectMeta)
 			patch := client.MergeFrom(orginalPod)
-			fmt.Println("patch:", patch)
-			fmt.Println("pod", pod)
 			err := r.Patch(ctx, &pod, patch)
 			if err != nil {
 				if !errors.IsNotFound(err) {

pkg/KubeArmorOperator/cmd/operator/main.go

@@ -31,6 +31,7 @@ var ExtClient *apiextensionsclientset.Clientset
 var Opv1Client *opv1client.Clientset
 var Secv1Client *secv1client.Clientset
 var AnnotateResource bool
+var AnnotateExisting bool
 var InitDeploy bool
 var LogLevel string
 var ProviderHostname, ProviderEndpoint string
@@ -57,7 +58,7 @@ var Cmd = &cobra.Command{
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		nodeWatcher := controllers.NewClusterWatcher(K8sClient, Logger, ExtClient, Opv1Client, Secv1Client, PathPrefix, DeploymentName, ProviderHostname, ProviderEndpoint, InitDeploy, AnnotateResource)
+		nodeWatcher := controllers.NewClusterWatcher(K8sClient, Logger, ExtClient, Opv1Client, Secv1Client, PathPrefix, DeploymentName, ProviderHostname, ProviderEndpoint, InitDeploy, AnnotateResource, AnnotateExisting)
 		go nodeWatcher.WatchConfigCrd()
 		nodeWatcher.WatchNodes()
 
@@ -89,6 +90,7 @@ func init() {
 	Cmd.PersistentFlags().BoolVar(&InitDeploy, "initDeploy", true, "Init container deployment")
 	Cmd.PersistentFlags().StringVar(&LogLevel, "loglevel", "info", "log level, e.g., debug, info, warn, error")
 	Cmd.PersistentFlags().BoolVar(&AnnotateResource, "annotateResource", false, "when true kubearmor annotate k8s resources with apparmor annotation")
+	Cmd.PersistentFlags().BoolVar(&AnnotateExisting, "annotateExisting", false, "when true kubearmor-controller restarts and annotates existing resources, with required annotations")
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.

pkg/KubeArmorOperator/common/defaults.go

@@ -128,6 +128,7 @@ var (
 	KubeArmorControllerArgs []string = []string{
 		"--leader-elect",
 		"--health-probe-bind-address=:8081",
+		"--annotateExisting=false",
 	}
 	KubeArmorControllerImage            string                        = "kubearmor/kubearmor-controller:latest"
 	KubeArmorControllerImagePullPolicy  string                        = "Always"

pkg/KubeArmorOperator/internal/controller/cluster.go

@@ -46,7 +46,7 @@ var informer informers.SharedInformerFactory
 var deployment_uuid types.UID
 var deployment_name string = "kubearmor-operator"
 var PathPrefix string
-var initDeploy, annotateResource bool
+var initDeploy, annotateResource, annotateExisting bool
 var ProviderHostname, ProviderEndpoint string
 
 type ClusterWatcher struct {
@@ -72,7 +72,7 @@ type Node struct {
 	Seccomp          string
 }
 
-func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, extClient *apiextensionsclientset.Clientset, opv1Client *opv1client.Clientset, secv1Client *secv1client.Clientset, pathPrefix, deploy_name, providerHostname, providerEndpoint string, initdeploy, annotateresource bool) *ClusterWatcher {
+func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, extClient *apiextensionsclientset.Clientset, opv1Client *opv1client.Clientset, secv1Client *secv1client.Clientset, pathPrefix, deploy_name, providerHostname, providerEndpoint string, initdeploy, annotateresource, annotateexisting bool) *ClusterWatcher {
 	if informer == nil {
 		informer = informers.NewSharedInformerFactory(client, 0)
 	}
@@ -90,6 +90,7 @@ func NewClusterWatcher(client *kubernetes.Clientset, log *zap.SugaredLogger, ext
 	deployment_name = deploy_name
 	initDeploy = initdeploy
 	annotateResource = annotateresource
+	annotateExisting = annotateexisting
 	ProviderHostname = providerHostname
 	ProviderEndpoint = providerEndpoint
 

pkg/KubeArmorOperator/internal/controller/resources.go

@@ -767,6 +767,9 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
 	relayServer := deployments.GetRelayDeployment(common.Namespace)
 	// update args, imagePullSecrets and tolerations
 	UpdateArgsIfDefinedAndUpdated(&controller.Spec.Template.Spec.Containers[0].Args, common.KubeArmorControllerArgs)
+	if annotateExisting {
+		UpdateArgsIfDefinedAndUpdated(&controller.Spec.Template.Spec.Containers[0].Args, []string{"annotateExisting=true"})
+	}
 	UpdateImagePullSecretsIfDefinedAndUpdated(&controller.Spec.Template.Spec.ImagePullSecrets, common.KubeArmorControllerImagePullSecrets)
 	UpdateTolerationsIfDefinedAndUpdated(&controller.Spec.Template.Spec.Tolerations, common.KubeArmorControllerTolerations)
 	if len(controller.Spec.Template.Spec.ImagePullSecrets) < 1 {