From 863134738e0bb2656a6c0b8103e639caaff1437e Mon Sep 17 00:00:00 2001 From: lvalerom Date: Tue, 10 Dec 2024 20:07:22 +0100 Subject: [PATCH] add retry logic --- go.mod | 1 + go.sum | 2 ++ pkg/common/utils/http.go | 17 +++++++++++++++ pkg/common/utils/retry.go | 25 +++++++++++++++++++++ pkg/crawlers/aws/aws.go | 6 +----- pkg/crawlers/azure/azure.go | 31 ++++++++++++++++----------- pkg/crawlers/cloudflare/cloudflare.go | 6 +----- pkg/crawlers/gcp/gcp.go | 6 +----- pkg/crawlers/oracle/oracle.go | 6 +----- 9 files changed, 68 insertions(+), 32 deletions(-) create mode 100644 pkg/common/utils/retry.go diff --git a/go.mod b/go.mod index bd8c339..a66e755 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect + github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect diff --git a/go.sum b/go.sum index d4935ad..1ad61b7 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1/go.mod h1:0wEl7vrAD8mehJyohS9HZy+WyEOaQO2mJx86Cvh93kM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 h1:8nn+rsCvTq9axyEh382S0PFLBeaFwNsT43IrPWzctRU= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1/go.mod h1:viRWSEhtMZqz1rhwmOVKkWl6SwmVowfL9O2YR5gI2PE= +github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= +github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= diff --git a/pkg/common/utils/http.go b/pkg/common/utils/http.go index 9b2c1c5..16a35a1 100644 --- a/pkg/common/utils/http.go +++ b/pkg/common/utils/http.go @@ -12,6 +12,23 @@ import ( const httpGetTimeout = 60 * time.Second +// HTTPGetWithRetry returns the body of the HTTP Get response. Retries if the call fails. +func HTTPGetWithRetry(provider, url string) ([]byte, error) { + var body []byte + retryErr := WithDefaultRetry(func() error { + var err error + body, err = HTTPGet(url) + if err != nil { + return errors.Wrapf(err, "failed to fetch networks from %s with URL: %s", provider, url) + } + return nil + }) + if retryErr != nil { + return nil, retryErr + } + return body, nil +} + // HTTPGet returns the body of the HTTP GET response func HTTPGet(url string) ([]byte, error) { log.Printf("Getting from URL: %s...", url) diff --git a/pkg/common/utils/retry.go b/pkg/common/utils/retry.go new file mode 100644 index 0000000..1ddc913 --- /dev/null +++ b/pkg/common/utils/retry.go @@ -0,0 +1,25 @@ +package utils + +import ( + "github.com/cenkalti/backoff/v3" + "log" + "time" +) + +// WithDefaultRetry retries a given function with a default exponential backoff +func WithDefaultRetry(do func() error) error { + return WithRetry(do, 2*time.Second, 10*time.Second, 5*time.Minute) +} + +// WithRetry retries a given function with an exponential backoff until maxTime is reached +func WithRetry(do func() error, interval, maxInterval, maxTime time.Duration) error { + exponential := backoff.NewExponentialBackOff() + exponential.MaxElapsedTime = maxTime + exponential.InitialInterval = interval + exponential.MaxInterval = maxInterval + + err := backoff.RetryNotify(do, exponential, func(err error, d time.Duration) { + log.Printf("call failed, retrying in %s. Error: %v", d.Round(time.Second), err) + }) + return err +} diff --git a/pkg/crawlers/aws/aws.go b/pkg/crawlers/aws/aws.go index 492a637..010a47e 100644 --- a/pkg/crawlers/aws/aws.go +++ b/pkg/crawlers/aws/aws.go @@ -67,11 +67,7 @@ func (c *awsNetworkCrawler) CrawlPublicNetworkRanges() (*common.ProviderNetworkR } func (c *awsNetworkCrawler) fetch() ([]byte, error) { - body, err := utils.HTTPGet(c.url) - if err != nil { - return nil, errors.Wrap(err, "failed to fetch networks from Google") - } - return body, nil + return utils.HTTPGetWithRetry("Amazon", c.url) } func (c *awsNetworkCrawler) parseNetworks(data []byte) (*common.ProviderNetworkRanges, error) { diff --git a/pkg/crawlers/azure/azure.go b/pkg/crawlers/azure/azure.go index 1ae0759..a3ccba0 100644 --- a/pkg/crawlers/azure/azure.go +++ b/pkg/crawlers/azure/azure.go @@ -3,12 +3,11 @@ package azure import ( "encoding/json" "fmt" - "log" - "os/exec" - "github.com/pkg/errors" "github.com/stackrox/external-network-pusher/pkg/common" "github.com/stackrox/external-network-pusher/pkg/common/utils" + "log" + "os/exec" ) // With Microsoft, it is a little different in a sense that: it has four @@ -278,12 +277,20 @@ func (c *azureNetworkCrawler) fetchAll() ([][]byte, error) { // and the page then renders generated URLs to json files. jsonURLs := make([]string, 0, len(c.urls)) for _, url := range c.urls { - jsonURL, err := c.redirectToJSONURL(url) - if err != nil { - return nil, errors.Wrapf(err, "failed to crawl Azure with URL: %s. Error: %v. JSON URL: %s", url, err, jsonURL) - } - if jsonURL == "" { - return nil, errors.Errorf("failed to crawl Azure with URL: %s, empty JSON URL returned. This could indicate Azure's services are unavailable", url) + var jsonURL string + retryErr := utils.WithDefaultRetry(func() error { + var err error + jsonURL, err = c.redirectToJSONURL(url) + if err != nil { + return errors.Wrapf(err, "failed to crawl Azure with URL: %s. Error: %v. JSON URL: %s", url, err, jsonURL) + } + if jsonURL == "" { + return errors.Errorf("failed to crawl Azure with URL: %s, empty JSON URL returned. This could indicate Azure's services are unavailable", url) + } + return nil + }) + if retryErr != nil { + return nil, retryErr } log.Printf("Received Azure network JSON URL: %s", jsonURL) jsonURLs = append(jsonURLs, jsonURL) @@ -292,9 +299,9 @@ func (c *azureNetworkCrawler) fetchAll() ([][]byte, error) { contents := make([][]byte, 0, len(jsonURLs)) for _, jsonURL := range jsonURLs { log.Printf("Current URL is: %s", jsonURL) - body, err := utils.HTTPGet(jsonURL) + body, err := utils.HTTPGetWithRetry("Azure", jsonURL) if err != nil { - return nil, errors.Wrapf(err, "Failed to fetch networks from Azure with URL: %s. Error: %v", jsonURL, err) + return nil, err } contents = append(contents, body) } @@ -318,7 +325,7 @@ func (c *azureNetworkCrawler) redirectToJSONURL(rawURL string) (string, error) { rawURL) out, err := exec.Command("/bin/sh", "-c", cmd).Output() if err != nil { - errors.Wrapf(err, "failed to redirect to JSON URL while trying to crawl Azure with URL: %s", rawURL) + err = errors.Wrapf(err, "failed to redirect to JSON URL while trying to crawl Azure with URL: %s", rawURL) return "", err } return string(out), nil diff --git a/pkg/crawlers/cloudflare/cloudflare.go b/pkg/crawlers/cloudflare/cloudflare.go index f2e219b..ec2e417 100644 --- a/pkg/crawlers/cloudflare/cloudflare.go +++ b/pkg/crawlers/cloudflare/cloudflare.go @@ -60,11 +60,7 @@ func (c *cloudflareNetworkCrawler) CrawlPublicNetworkRanges() (*common.ProviderN } func (c *cloudflareNetworkCrawler) fetch() ([]byte, error) { - body, err := utils.HTTPGet(c.url) - if err != nil { - return nil, errors.Wrapf(err, "failed to fetch networks from Cloudflare with URL: %s", c.url) - } - return body, nil + return utils.HTTPGetWithRetry("Cloudflare", c.url) } func (c *cloudflareNetworkCrawler) parseNetworks(networks []byte) (*common.ProviderNetworkRanges, error) { diff --git a/pkg/crawlers/gcp/gcp.go b/pkg/crawlers/gcp/gcp.go index 652e097..f664ce2 100644 --- a/pkg/crawlers/gcp/gcp.go +++ b/pkg/crawlers/gcp/gcp.go @@ -58,11 +58,7 @@ func (c *gcpNetworkCrawler) CrawlPublicNetworkRanges() (*common.ProviderNetworkR } func (c *gcpNetworkCrawler) fetch() ([]byte, error) { - body, err := utils.HTTPGet(c.url) - if err != nil { - return nil, errors.Wrap(err, "failed to fetch networks from Google") - } - return body, nil + return utils.HTTPGetWithRetry("Google", c.url) } func (c *gcpNetworkCrawler) parseNetworks(data []byte) (*common.ProviderNetworkRanges, error) { diff --git a/pkg/crawlers/oracle/oracle.go b/pkg/crawlers/oracle/oracle.go index bf75c1a..43f05a2 100644 --- a/pkg/crawlers/oracle/oracle.go +++ b/pkg/crawlers/oracle/oracle.go @@ -61,11 +61,7 @@ func (c *ociNetworkCrawler) CrawlPublicNetworkRanges() (*common.ProviderNetworkR } func (c *ociNetworkCrawler) fetch() ([]byte, error) { - body, err := utils.HTTPGet(c.url) - if err != nil { - return nil, errors.Wrap(err, "failed to fetch networks from Oracle") - } - return body, nil + return utils.HTTPGetWithRetry("Oracle", c.url) } func (c *ociNetworkCrawler) parseNetworks(data []byte) (*common.ProviderNetworkRanges, error) {