From 23567047dcd54dee6c907dcb6d608d0959776a55 Mon Sep 17 00:00:00 2001 From: lvalerom Date: Tue, 10 Dec 2024 20:07:22 +0100 Subject: [PATCH] add retry logic --- go.mod | 1 + go.sum | 2 ++ pkg/common/utils/retry.go | 20 ++++++++++++++++++++ pkg/crawlers/azure/azure.go | 10 ++++++++-- 4 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 pkg/common/utils/retry.go diff --git a/go.mod b/go.mod index bd8c339..a66e755 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect + github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect diff --git a/go.sum b/go.sum index d4935ad..1ad61b7 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1/go.mod h1:0wEl7vrAD8mehJyohS9HZy+WyEOaQO2mJx86Cvh93kM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE= +github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= +github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= diff --git a/pkg/common/utils/retry.go b/pkg/common/utils/retry.go new file mode 100644 index 0000000..fe6e897 --- /dev/null +++ b/pkg/common/utils/retry.go @@ -0,0 +1,20 @@ +package utils + +import ( + "github.com/cenkalti/backoff/v3" + "log" + "time" +) + +// WithRetry retries a given function with an exponential backoff until maxTime is reached +func WithRetry(do func() error, interval, maxInterval, maxTime time.Duration) error { + exponential := backoff.NewExponentialBackOff() + exponential.MaxElapsedTime = maxTime + exponential.InitialInterval = interval + exponential.MaxInterval = maxInterval + + err := backoff.RetryNotify(do, exponential, func(err error, d time.Duration) { + log.Printf("call failed, retrying in %d. Error: %v", d.Round(time.Second), err) + }) + return err +} diff --git a/pkg/crawlers/azure/azure.go b/pkg/crawlers/azure/azure.go index 1ae0759..e2f9786 100644 --- a/pkg/crawlers/azure/azure.go +++ b/pkg/crawlers/azure/azure.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "os/exec" + "time" "github.com/pkg/errors" "github.com/stackrox/external-network-pusher/pkg/common" @@ -316,9 +317,14 @@ func (c *azureNetworkCrawler) redirectToJSONURL(rawURL string) (string, error) { // Trim trailing newline char "tr -d '\n'", rawURL) - out, err := exec.Command("/bin/sh", "-c", cmd).Output() + var out []byte + err := utils.WithRetry(func() error { + var err error + out, err = exec.Command("/bin/sh", "-c", cmd).Output() + return err + }, 2*time.Second, 10*time.Second, 5*time.Minute) if err != nil { - errors.Wrapf(err, "failed to redirect to JSON URL while trying to crawl Azure with URL: %s", rawURL) + err = errors.Wrapf(err, "failed to redirect to JSON URL while trying to crawl Azure with URL: %s", rawURL) return "", err } return string(out), nil