add retry logic

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
+	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect

go.sum

@@ -30,6 +30,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1/go.mod h1:0wEl7vrAD8mehJyohS9HZy+WyEOaQO2mJx86Cvh93kM=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE=
+github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M=
+github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g=
 github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=

pkg/common/utils/retry.go

@@ -0,0 +1,20 @@
+package utils
+
+import (
+	"github.com/cenkalti/backoff/v3"
+	"log"
+	"time"
+)
+
+// WithRetry retries a given function with an exponential backoff until maxTime is reached
+func WithRetry(do func() error, interval, maxInterval, maxTime time.Duration) error {
+	exponential := backoff.NewExponentialBackOff()
+	exponential.MaxElapsedTime = maxTime
+	exponential.InitialInterval = interval
+	exponential.MaxInterval = maxInterval
+
+	err := backoff.RetryNotify(do, exponential, func(err error, d time.Duration) {
+		log.Printf("call failed, retrying in %d. Error: %v", d.Round(time.Second), err)
+	})
+	return err
+}

pkg/crawlers/azure/azure.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"log"
 	"os/exec"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/stackrox/external-network-pusher/pkg/common"
@@ -316,9 +317,14 @@ func (c *azureNetworkCrawler) redirectToJSONURL(rawURL string) (string, error) {
 			// Trim trailing newline char
 			"tr -d '\n'",
 		rawURL)
-	out, err := exec.Command("/bin/sh", "-c", cmd).Output()
+	var out []byte
+	err := utils.WithRetry(func() error {
+		var err error
+		out, err = exec.Command("/bin/sh", "-c", cmd).Output()
+		return err
+	}, 2*time.Second, 10*time.Second, 5*time.Minute)
 	if err != nil {
-		errors.Wrapf(err, "failed to redirect to JSON URL while trying to crawl Azure with URL: %s", rawURL)
+		err = errors.Wrapf(err, "failed to redirect to JSON URL while trying to crawl Azure with URL: %s", rawURL)
 		return "", err
 	}
 	return string(out), nil