Merged PR 1379: added retry for creatingAzureManager in case of throt…

…tled requests

added retry for forceRefresh in case of throttled requests
ran tests
MallocNanoZone=0 go test -race k8s.io/autoscaler/cluster-autoscaler/cloudprovider/azure -- passed

and commented out unit test -- commented out as it takes 10 minutes to complete

func TestCreateAzureManagerWithRetryError(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	mockVMClient := mockvmclient.NewMockInterface(ctrl)
	mockVMSSClient := mockvmssclient.NewMockInterface(ctrl)
	mockVMSSClient.EXPECT().List(gomock.Any(), "fakeId").Return([]compute.VirtualMachineScaleSet{}, retry.NewError(true, errors.New("test"))).AnyTimes()
	mockAzClient := &azClient{
		virtualMachinesClient:         mockVMClient,
		virtualMachineScaleSetsClient: mockVMSSClient,
	}
	manager, err := createAzureManagerInternal(strings.NewReader(validAzureCfg), cloudprovider.NodeGroupDiscoveryOptions{}, config.AutoscalingOptions{}, mockAzClient)
	assert.Nil(t, manager)
	assert.NotNil(t, err)
}

cluster-autoscaler/cloudprovider/azure/azure_manager.go

@@ -26,10 +26,13 @@ import (
 	"time"
 
 	"github.com/Azure/go-autorest/autorest/azure"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
 	"k8s.io/autoscaler/cluster-autoscaler/config"
 	"k8s.io/autoscaler/cluster-autoscaler/config/dynamic"
+	kretry "k8s.io/client-go/util/retry"
 	klog "k8s.io/klog/v2"
+	"sigs.k8s.io/cloud-provider-azure/pkg/retry"
 )
 
 const (
@@ -106,8 +109,22 @@ func createAzureManagerInternal(configReader io.Reader, discoveryOpts cloudprovi
 		return nil, err
 	}
 
+	retryBackoff := wait.Backoff{
+		Duration: 2 * time.Minute,
+		Factor:   1.0,
+		Jitter:   0.1,
+		Steps:    6,
+		Cap:      10 * time.Minute,
+	}
+
 	if err := manager.forceRefresh(); err != nil {
-		return nil, err
+		err = kretry.OnError(retryBackoff, retry.IsErrorRetriable, func() (err error) {
+			return manager.forceRefresh()
+		})
+		if err != nil {
+			return nil, err
+		}
+		return manager, nil
 	}
 
 	return manager, nil