adding kwok cm controller retries

internal/kwok-gpu-device-plugin/controllers/configmap/controller.go

@@ -2,6 +2,7 @@ package configmamp
 
 import (
 	"log"
+	"time"
 
 	"github.com/run-ai/fake-gpu-operator/internal/common/constants"
 	"github.com/run-ai/fake-gpu-operator/internal/common/topology"
@@ -61,7 +62,7 @@ func NewConfigMapController(
 		Handler: cache.ResourceEventHandlerFuncs{
 			AddFunc: func(obj interface{}) {
 				go func() {
-					c.callConfigMapHandler(obj.(*v1.ConfigMap))
+					c.callConfigMapHandler(obj.(*v1.ConfigMap), 0)
 				}()
 			},
 		},
@@ -86,11 +87,16 @@ func (c *ConfigMapController) isFakeGpuKWOKNodeConfigMap(cm *v1.ConfigMap) bool
 	return cm.Annotations[constants.AnnotationKwokNode] == "fake"
 }
 
-func (c *ConfigMapController) callConfigMapHandler(cm *v1.ConfigMap) {
+func (c *ConfigMapController) callConfigMapHandler(cm *v1.ConfigMap, retryCount int) {
 	nodeName := cm.Labels[constants.LabelTopologyCMNodeName]
 	node, err := c.nodeLister.Get(nodeName)
 	if err != nil {
-		log.Printf("Failed to get node %s: %v", nodeName, err)
+		delay := time.Millisecond * 100 * (1 << retryCount)
+		log.Printf("Failed to get node %s: %v. retry in %v", nodeName, err, delay)
+		time.Sleep(delay)
+		if retryCount < 5 {
+			c.callConfigMapHandler(cm, retryCount+1)
+		}
 		return
 	}
 	util.LogErrorIfExist(c.handler.HandleAdd(cm, node), "Failed to handle cm addition")