From 3ca3d2598bc2a016c71a4a26c7c2d6949b9a6baf Mon Sep 17 00:00:00 2001 From: Vanshikav123 Date: Mon, 20 May 2024 00:33:33 +0530 Subject: [PATCH 1/7] hedged_request Signed-off-by: Vanshikav123 --- client/factory.go | 35 +++++++++++++++++++++++++++-------- go.mod | 1 + go.sum | 2 ++ providers/azure/azure.go | 5 +++-- providers/cos/cos.go | 6 +++--- providers/gcs/gcs.go | 4 ++-- providers/oci/oci.go | 14 ++++++++------ providers/s3/s3.go | 4 ++-- 8 files changed, 48 insertions(+), 23 deletions(-) diff --git a/client/factory.go b/client/factory.go index 9e1adec9..8f9afa17 100644 --- a/client/factory.go +++ b/client/factory.go @@ -6,8 +6,11 @@ package client import ( "context" "fmt" + "net/http" "strings" + "time" + "github.com/cristalhq/hedgedhttp" "github.com/thanos-io/objstore" "github.com/thanos-io/objstore/providers/azure" "github.com/thanos-io/objstore/providers/bos" @@ -42,9 +45,10 @@ const ( ) type BucketConfig struct { - Type ObjProvider `yaml:"type"` - Config interface{} `yaml:"config"` - Prefix string `yaml:"prefix" default:""` + Type ObjProvider `yaml:"type"` + Config interface{} `yaml:"config"` + Prefix string `yaml:"prefix" default:""` + UseHedgedHTTP bool `yaml:"use_hedged_http" default:"false"` } // NewBucket initializes and returns new object storage clients. @@ -60,19 +64,34 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string) (obj if err != nil { return nil, errors.Wrap(err, "marshal content of bucket configuration") } + var httpClient *http.Client = http.DefaultClient + // If UseHedgedHTTP is true, create a hedged HTTP client + if bucketConf.UseHedgedHTTP { + hedgedClient, err := hedgedhttp.New(hedgedhttp.Config{ + Transport: http.DefaultTransport, + Upto: 3, + Delay: 100 * time.Millisecond, + }) + if err != nil { + return nil, errors.Wrap(err, "creating hedged HTTP client") + } + httpClient = &http.Client{ + Transport: hedgedClient, + } + } var bucket objstore.Bucket switch strings.ToUpper(string(bucketConf.Type)) { case string(GCS): - bucket, err = gcs.NewBucket(context.Background(), logger, config, component) + bucket, err = gcs.NewBucket(context.Background(), logger, config, component, httpClient) case string(S3): - bucket, err = s3.NewBucket(logger, config, component) + bucket, err = s3.NewBucket(logger, config, component, httpClient) case string(AZURE): - bucket, err = azure.NewBucket(logger, config, component) + bucket, err = azure.NewBucket(logger, config, component, httpClient) case string(SWIFT): bucket, err = swift.NewContainer(logger, config) case string(COS): - bucket, err = cos.NewBucket(logger, config, component) + bucket, err = cos.NewBucket(logger, config, component, httpClient) case string(ALIYUNOSS): bucket, err = oss.NewBucket(logger, config, component) case string(FILESYSTEM): @@ -80,7 +99,7 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string) (obj case string(BOS): bucket, err = bos.NewBucket(logger, config, component) case string(OCI): - bucket, err = oci.NewBucket(logger, config) + bucket, err = oci.NewBucket(logger, config, httpClient) case string(OBS): bucket, err = obs.NewBucket(logger, config) default: diff --git a/go.mod b/go.mod index 8887b088..026b9bb5 100644 --- a/go.mod +++ b/go.mod @@ -111,6 +111,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 + github.com/cristalhq/hedgedhttp v0.9.1 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect golang.org/x/crypto v0.21.0 // indirect diff --git a/go.sum b/go.sum index 8e01ea18..a1a2fb10 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,8 @@ github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5P github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cristalhq/hedgedhttp v0.9.1 h1:g68L9cf8uUyQKQJwciD0A1Vgbsz+QgCjuB1I8FAsCDs= +github.com/cristalhq/hedgedhttp v0.9.1/go.mod h1:XkqWU6qVMutbhW68NnzjWrGtH8NUx1UfYqGYtHVKIsI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/providers/azure/azure.go b/providers/azure/azure.go index a72f3052..db337043 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -6,6 +6,7 @@ package azure import ( "context" "io" + "net/http" "os" "strings" "testing" @@ -143,7 +144,7 @@ type Bucket struct { } // NewBucket returns a new Bucket using the provided Azure config. -func NewBucket(logger log.Logger, azureConfig []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, azureConfig []byte, component string, httpClient *http.Client) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new Azure bucket connection", "component", component) conf, err := parseConfig(azureConfig) if err != nil { @@ -352,7 +353,7 @@ func NewTestBucket(t testing.TB, component string) (objstore.Bucket, func(), err if err != nil { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), bc, component) + bkt, err := NewBucket(log.NewNopLogger(), bc, component, http.DefaultClient) if err != nil { t.Errorf("Cannot create Azure storage container:") return nil, nil, err diff --git a/providers/cos/cos.go b/providers/cos/cos.go index e518cae2..e0e3aabf 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -95,7 +95,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided cos configuration. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, httpClient *http.Client) (*Bucket, error) { if logger == nil { logger = log.NewNopLogger() } @@ -485,7 +485,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultClient) if err != nil { return nil, nil, err } @@ -506,7 +506,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultClient) if err != nil { return nil, nil, err } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index c4743630..54ef6c39 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -72,7 +72,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, httpClient *http.Client) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err @@ -302,7 +302,7 @@ func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error return nil, nil, err } - b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultClient) if err != nil { return nil, nil, err } diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 2db35461..c73f93f1 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -288,7 +288,7 @@ func (b *Bucket) deleteBucket(ctx context.Context) (err error) { } // NewBucket returns a new Bucket using the provided oci config values. -func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { +func NewBucket(logger log.Logger, ociConfig []byte, httpClient *http.Client) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new oci bucket connection") var config = DefaultConfig var configurationProvider common.ConfigurationProvider @@ -335,11 +335,13 @@ func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { return nil, errors.Wrapf(err, "unable to create ObjectStorage client with the given oci configurations") } - httpClient := http.Client{ - Transport: CustomTransport(config), - Timeout: config.HTTPConfig.ClientTimeout, + if httpClient == nil { + httpClient = &http.Client{ + Transport: CustomTransport(config), + Timeout: config.HTTPConfig.ClientTimeout, + } } - client.HTTPClient = &httpClient + client.HTTPClient = httpClient requestMetadata := getRequestMetadata(config.MaxRequestRetries, config.RequestRetryInterval) @@ -375,7 +377,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), ociConfig) + bkt, err := NewBucket(log.NewNopLogger(), ociConfig, http.DefaultClient) if err != nil { return nil, nil, err } diff --git a/providers/s3/s3.go b/providers/s3/s3.go index fd98bd85..54a4c7c3 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -175,7 +175,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, httpClient *http.Client) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err @@ -605,7 +605,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultClient) if err != nil { return nil, nil, err } From e606f9d23d97b26848886097c445452cd716383f Mon Sep 17 00:00:00 2001 From: Vanshikav123 Date: Mon, 20 May 2024 23:31:44 +0530 Subject: [PATCH 2/7] http custom transport parameter Signed-off-by: Vanshikav123 --- README.md | 4 +++- client/factory.go | 36 +++++++++--------------------------- client/factory_test.go | 7 ++++--- providers/azure/azure.go | 5 ++--- providers/cos/cos.go | 6 +++--- providers/gcs/gcs.go | 4 ++-- providers/oci/oci.go | 15 +++++++-------- providers/s3/s3.go | 4 ++-- 8 files changed, 32 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 949c2716..d2e9e50d 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ Generally, you have two ways of using `objstore` module: First is to import the provider you want e.g. [`github.com/thanos-io/objstore/providers/s3`](providers/s3) and instantiate it with available constructor (e.g. `NewBucket`). -The second option is to use the factory `NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registerer, component string)` that will instantiate the object storage client based on YAML file provided. The YAML file has generally the format like this: +The second option is to use the factory `NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registerer, component string,rt http.RoundTripper)` that will instantiate the object storage client based on YAML file provided. The YAML file has generally the format like this: ```yaml type: @@ -114,6 +114,8 @@ config: The exact option depends on provider and are in sections below. + `NewBucket` function now accepts an `http.RoundTripper` parameter allows clients to provide a custom transport for HTTP requests, making the function more flexible , this change facilitates the use of various HTTP client configurations, including hedged HTTP transports. + > NOTE: All code snippets are auto-generated from code and up-to-date. Check out the [Thanos documentation](https://thanos.io/tip/thanos/storage.md/) to see how Thanos uses this module. diff --git a/client/factory.go b/client/factory.go index 8f9afa17..fe1f5260 100644 --- a/client/factory.go +++ b/client/factory.go @@ -8,9 +8,7 @@ import ( "fmt" "net/http" "strings" - "time" - "github.com/cristalhq/hedgedhttp" "github.com/thanos-io/objstore" "github.com/thanos-io/objstore/providers/azure" "github.com/thanos-io/objstore/providers/bos" @@ -45,15 +43,14 @@ const ( ) type BucketConfig struct { - Type ObjProvider `yaml:"type"` - Config interface{} `yaml:"config"` - Prefix string `yaml:"prefix" default:""` - UseHedgedHTTP bool `yaml:"use_hedged_http" default:"false"` + Type ObjProvider `yaml:"type"` + Config interface{} `yaml:"config"` + Prefix string `yaml:"prefix" default:""` } // NewBucket initializes and returns new object storage clients. // NOTE: confContentYaml can contain secrets. -func NewBucket(logger log.Logger, confContentYaml []byte, component string) (objstore.Bucket, error) { +func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt http.RoundTripper) (objstore.Bucket, error) { level.Info(logger).Log("msg", "loading bucket configuration") bucketConf := &BucketConfig{} if err := yaml.UnmarshalStrict(confContentYaml, bucketConf); err != nil { @@ -64,34 +61,19 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string) (obj if err != nil { return nil, errors.Wrap(err, "marshal content of bucket configuration") } - var httpClient *http.Client = http.DefaultClient - // If UseHedgedHTTP is true, create a hedged HTTP client - if bucketConf.UseHedgedHTTP { - hedgedClient, err := hedgedhttp.New(hedgedhttp.Config{ - Transport: http.DefaultTransport, - Upto: 3, - Delay: 100 * time.Millisecond, - }) - if err != nil { - return nil, errors.Wrap(err, "creating hedged HTTP client") - } - httpClient = &http.Client{ - Transport: hedgedClient, - } - } var bucket objstore.Bucket switch strings.ToUpper(string(bucketConf.Type)) { case string(GCS): - bucket, err = gcs.NewBucket(context.Background(), logger, config, component, httpClient) + bucket, err = gcs.NewBucket(context.Background(), logger, config, component) case string(S3): - bucket, err = s3.NewBucket(logger, config, component, httpClient) + bucket, err = s3.NewBucket(logger, config, component) case string(AZURE): - bucket, err = azure.NewBucket(logger, config, component, httpClient) + bucket, err = azure.NewBucket(logger, config, component) case string(SWIFT): bucket, err = swift.NewContainer(logger, config) case string(COS): - bucket, err = cos.NewBucket(logger, config, component, httpClient) + bucket, err = cos.NewBucket(logger, config, component) case string(ALIYUNOSS): bucket, err = oss.NewBucket(logger, config, component) case string(FILESYSTEM): @@ -99,7 +81,7 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string) (obj case string(BOS): bucket, err = bos.NewBucket(logger, config, component) case string(OCI): - bucket, err = oci.NewBucket(logger, config, httpClient) + bucket, err = oci.NewBucket(logger, config) case string(OBS): bucket, err = obs.NewBucket(logger, config) default: diff --git a/client/factory_test.go b/client/factory_test.go index 4a9cf879..4d442397 100644 --- a/client/factory_test.go +++ b/client/factory_test.go @@ -6,6 +6,7 @@ package client import ( "context" "fmt" + "net/http" "os" "github.com/go-kit/log" @@ -23,7 +24,7 @@ func ExampleBucket() { } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", http.DefaultTransport) if err != nil { panic(err) } @@ -46,7 +47,7 @@ func ExampleTracingBucketUsingOpenTracing() { //nolint:govet } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", http.DefaultTransport) if err != nil { panic(err) } @@ -72,7 +73,7 @@ func ExampleTracingBucketUsingOpenTelemetry() { //nolint:govet } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", http.DefaultTransport) if err != nil { panic(err) } diff --git a/providers/azure/azure.go b/providers/azure/azure.go index db337043..a72f3052 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -6,7 +6,6 @@ package azure import ( "context" "io" - "net/http" "os" "strings" "testing" @@ -144,7 +143,7 @@ type Bucket struct { } // NewBucket returns a new Bucket using the provided Azure config. -func NewBucket(logger log.Logger, azureConfig []byte, component string, httpClient *http.Client) (*Bucket, error) { +func NewBucket(logger log.Logger, azureConfig []byte, component string) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new Azure bucket connection", "component", component) conf, err := parseConfig(azureConfig) if err != nil { @@ -353,7 +352,7 @@ func NewTestBucket(t testing.TB, component string) (objstore.Bucket, func(), err if err != nil { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), bc, component, http.DefaultClient) + bkt, err := NewBucket(log.NewNopLogger(), bc, component) if err != nil { t.Errorf("Cannot create Azure storage container:") return nil, nil, err diff --git a/providers/cos/cos.go b/providers/cos/cos.go index e0e3aabf..e518cae2 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -95,7 +95,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided cos configuration. -func NewBucket(logger log.Logger, conf []byte, component string, httpClient *http.Client) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { if logger == nil { logger = log.NewNopLogger() } @@ -485,7 +485,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultClient) + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") if err != nil { return nil, nil, err } @@ -506,7 +506,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultClient) + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") if err != nil { return nil, nil, err } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 54ef6c39..c4743630 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -72,7 +72,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, httpClient *http.Client) (*Bucket, error) { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err @@ -302,7 +302,7 @@ func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error return nil, nil, err } - b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultClient) + b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test") if err != nil { return nil, nil, err } diff --git a/providers/oci/oci.go b/providers/oci/oci.go index c73f93f1..3f7af45e 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -288,7 +288,7 @@ func (b *Bucket) deleteBucket(ctx context.Context) (err error) { } // NewBucket returns a new Bucket using the provided oci config values. -func NewBucket(logger log.Logger, ociConfig []byte, httpClient *http.Client) (*Bucket, error) { +func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new oci bucket connection") var config = DefaultConfig var configurationProvider common.ConfigurationProvider @@ -335,13 +335,12 @@ func NewBucket(logger log.Logger, ociConfig []byte, httpClient *http.Client) (*B return nil, errors.Wrapf(err, "unable to create ObjectStorage client with the given oci configurations") } - if httpClient == nil { - httpClient = &http.Client{ - Transport: CustomTransport(config), - Timeout: config.HTTPConfig.ClientTimeout, - } + httpClient := http.Client{ + Transport: CustomTransport(config), + Timeout: config.HTTPConfig.ClientTimeout, } - client.HTTPClient = httpClient + + client.HTTPClient = &httpClient requestMetadata := getRequestMetadata(config.MaxRequestRetries, config.RequestRetryInterval) @@ -377,7 +376,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), ociConfig, http.DefaultClient) + bkt, err := NewBucket(log.NewNopLogger(), ociConfig) if err != nil { return nil, nil, err } diff --git a/providers/s3/s3.go b/providers/s3/s3.go index 54a4c7c3..fd98bd85 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -175,7 +175,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf []byte, component string, httpClient *http.Client) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err @@ -605,7 +605,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultClient) + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") if err != nil { return nil, nil, err } From 659a3bcd40132b1e49b7202fdeb869bb9b55a738 Mon Sep 17 00:00:00 2001 From: Vanshikav123 Date: Mon, 20 May 2024 23:35:36 +0530 Subject: [PATCH 3/7] module changes Signed-off-by: Vanshikav123 --- go.mod | 1 - go.sum | 2 -- providers/oci/oci.go | 1 - 3 files changed, 4 deletions(-) diff --git a/go.mod b/go.mod index 026b9bb5..8887b088 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 - github.com/cristalhq/hedgedhttp v0.9.1 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect golang.org/x/crypto v0.21.0 // indirect diff --git a/go.sum b/go.sum index a1a2fb10..8e01ea18 100644 --- a/go.sum +++ b/go.sum @@ -66,8 +66,6 @@ github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5P github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cristalhq/hedgedhttp v0.9.1 h1:g68L9cf8uUyQKQJwciD0A1Vgbsz+QgCjuB1I8FAsCDs= -github.com/cristalhq/hedgedhttp v0.9.1/go.mod h1:XkqWU6qVMutbhW68NnzjWrGtH8NUx1UfYqGYtHVKIsI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 3f7af45e..2db35461 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -339,7 +339,6 @@ func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { Transport: CustomTransport(config), Timeout: config.HTTPConfig.ClientTimeout, } - client.HTTPClient = &httpClient requestMetadata := getRequestMetadata(config.MaxRequestRetries, config.RequestRetryInterval) From 1cffe47c0bbe3429c50c0137c303dabf7b659208 Mon Sep 17 00:00:00 2001 From: Vanshikav123 Date: Tue, 21 May 2024 23:56:06 +0530 Subject: [PATCH 4/7] applied logic Signed-off-by: Vanshikav123 --- client/factory.go | 10 +++++----- providers/azure/azure.go | 5 +++-- providers/cos/cos.go | 6 +++--- providers/gcs/gcs.go | 4 ++-- providers/oci/oci.go | 4 ++-- providers/s3/s3.go | 4 ++-- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/client/factory.go b/client/factory.go index fe1f5260..8b17156f 100644 --- a/client/factory.go +++ b/client/factory.go @@ -65,15 +65,15 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt h var bucket objstore.Bucket switch strings.ToUpper(string(bucketConf.Type)) { case string(GCS): - bucket, err = gcs.NewBucket(context.Background(), logger, config, component) + bucket, err = gcs.NewBucket(context.Background(), logger, config, component, rt) case string(S3): - bucket, err = s3.NewBucket(logger, config, component) + bucket, err = s3.NewBucket(logger, config, component, rt) case string(AZURE): - bucket, err = azure.NewBucket(logger, config, component) + bucket, err = azure.NewBucket(logger, config, component, rt) case string(SWIFT): bucket, err = swift.NewContainer(logger, config) case string(COS): - bucket, err = cos.NewBucket(logger, config, component) + bucket, err = cos.NewBucket(logger, config, component, rt) case string(ALIYUNOSS): bucket, err = oss.NewBucket(logger, config, component) case string(FILESYSTEM): @@ -81,7 +81,7 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt h case string(BOS): bucket, err = bos.NewBucket(logger, config, component) case string(OCI): - bucket, err = oci.NewBucket(logger, config) + bucket, err = oci.NewBucket(logger, config, rt) case string(OBS): bucket, err = obs.NewBucket(logger, config) default: diff --git a/providers/azure/azure.go b/providers/azure/azure.go index a72f3052..b6209428 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -6,6 +6,7 @@ package azure import ( "context" "io" + "net/http" "os" "strings" "testing" @@ -143,7 +144,7 @@ type Bucket struct { } // NewBucket returns a new Bucket using the provided Azure config. -func NewBucket(logger log.Logger, azureConfig []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, azureConfig []byte, component string, rt http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new Azure bucket connection", "component", component) conf, err := parseConfig(azureConfig) if err != nil { @@ -352,7 +353,7 @@ func NewTestBucket(t testing.TB, component string) (objstore.Bucket, func(), err if err != nil { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), bc, component) + bkt, err := NewBucket(log.NewNopLogger(), bc, component, http.DefaultTransport) if err != nil { t.Errorf("Cannot create Azure storage container:") return nil, nil, err diff --git a/providers/cos/cos.go b/providers/cos/cos.go index e518cae2..daa47aca 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -95,7 +95,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided cos configuration. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { if logger == nil { logger = log.NewNopLogger() } @@ -485,7 +485,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultTransport) if err != nil { return nil, nil, err } @@ -506,7 +506,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultTransport) if err != nil { return nil, nil, err } diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index c4743630..42e0545e 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -72,7 +72,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err @@ -302,7 +302,7 @@ func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error return nil, nil, err } - b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultTransport) if err != nil { return nil, nil, err } diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 2db35461..01862fcb 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -288,7 +288,7 @@ func (b *Bucket) deleteBucket(ctx context.Context) (err error) { } // NewBucket returns a new Bucket using the provided oci config values. -func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { +func NewBucket(logger log.Logger, ociConfig []byte, rt http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new oci bucket connection") var config = DefaultConfig var configurationProvider common.ConfigurationProvider @@ -375,7 +375,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), ociConfig) + bkt, err := NewBucket(log.NewNopLogger(), ociConfig, http.DefaultTransport) if err != nil { return nil, nil, err } diff --git a/providers/s3/s3.go b/providers/s3/s3.go index fd98bd85..fc1d92f3 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -175,7 +175,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err @@ -605,7 +605,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", http.DefaultTransport) if err != nil { return nil, nil, err } From 540c1e5acf89f1b14117f0a4104c38d479474722 Mon Sep 17 00:00:00 2001 From: Vanshikav123 Date: Fri, 24 May 2024 21:32:25 +0530 Subject: [PATCH 5/7] added rt Signed-off-by: Vanshikav123 --- providers/azure/azure.go | 6 +++--- providers/azure/helpers.go | 16 ++++++++++++---- providers/cos/cos.go | 11 ++++++++--- providers/gcs/gcs.go | 19 +++++++++---------- providers/gcs/gcs_test.go | 2 +- providers/oci/oci.go | 9 ++++++++- providers/s3/s3.go | 14 +++++++------- providers/s3/s3_e2e_test.go | 2 ++ providers/s3/s3_test.go | 16 ++++++++-------- 9 files changed, 58 insertions(+), 37 deletions(-) diff --git a/providers/azure/azure.go b/providers/azure/azure.go index b6209428..38b0a8ed 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -153,16 +153,16 @@ func NewBucket(logger log.Logger, azureConfig []byte, component string, rt http. if conf.MSIResource != "" { level.Warn(logger).Log("msg", "The field msi_resource has been deprecated and should no longer be set") } - return NewBucketWithConfig(logger, conf, component) + return NewBucketWithConfig(logger, conf, component, rt) } // NewBucketWithConfig returns a new Bucket using the provided Azure config struct. -func NewBucketWithConfig(logger log.Logger, conf Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, conf Config, component string, rt http.RoundTripper) (*Bucket, error) { if err := conf.validate(); err != nil { return nil, err } - containerClient, err := getContainerClient(conf) + containerClient, err := getContainerClient(conf, rt) if err != nil { return nil, err } diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 846394a0..53b2ac19 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -19,11 +19,19 @@ import ( // DirDelim is the delimiter used to model a directory structure in an object store bucket. const DirDelim = "/" -func getContainerClient(conf Config) (*container.Client, error) { - dt, err := exthttp.DefaultTransport(conf.HTTPConfig) - if err != nil { - return nil, err +func getContainerClient(conf Config, rt http.RoundTripper) (*container.Client, error) { + var dt http.RoundTripper + var err error + + if rt != nil { + dt = rt + } else { + dt, err = exthttp.DefaultTransport(conf.HTTPConfig) + if err != nil { + return nil, err + } } + opt := &container.ClientOptions{ ClientOptions: azcore.ClientOptions{ Retry: policy.RetryOptions{ diff --git a/providers/cos/cos.go b/providers/cos/cos.go index daa47aca..2cb0aa9b 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -105,11 +105,11 @@ func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTr return nil, errors.Wrap(err, "parsing cos configuration") } - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, rt) } // NewBucketWithConfig returns a new Bucket using the provided cos config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate cos configuration") } @@ -128,7 +128,12 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B } } b := &cos.BaseURL{BucketURL: bucketURL} - tpt, _ := exthttp.DefaultTransport(config.HTTPConfig) + var tpt http.RoundTripper + if rt != nil { + tpt = rt + } else { + tpt, _ = exthttp.DefaultTransport(config.HTTPConfig) + } client := cos.NewClient(b, &http.Client{ Transport: &cos.AuthorizationTransport{ SecretID: config.SecretId, diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 42e0545e..597b530a 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -77,11 +77,11 @@ func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component st if err != nil { return nil, err } - return NewBucketWithConfig(ctx, logger, config, component) + return NewBucketWithConfig(ctx, logger, config, component, rt) } // NewBucketWithConfig returns a new Bucket with gcs Config struct. -func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string) (*Bucket, error) { +func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, rt http.RoundTripper) (*Bucket, error) { if gc.Bucket == "" { return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") } @@ -103,7 +103,7 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp if !gc.UseGRPC { var err error - opts, err = appendHttpOptions(gc, opts) + opts, err = appendHttpOptions(gc, opts, rt) if err != nil { return nil, err } @@ -112,25 +112,24 @@ func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, comp return newBucket(ctx, logger, gc, opts) } -func appendHttpOptions(gc Config, opts []option.ClientOption) ([]option.ClientOption, error) { +func appendHttpOptions(gc Config, opts []option.ClientOption, rt http.RoundTripper) ([]option.ClientOption, error) { // Check if a roundtripper has been set in the config // otherwise build the default transport. - var rt http.RoundTripper - if gc.HTTPConfig.Transport != nil { - rt = gc.HTTPConfig.Transport + var tpt http.RoundTripper + if rt != nil { + tpt = rt } else { var err error - rt, err = exthttp.DefaultTransport(gc.HTTPConfig) + tpt, err = exthttp.DefaultTransport(gc.HTTPConfig) if err != nil { return nil, err } } - // GCS uses some defaults when "options.WithHTTPClient" is not used that are important when we call // htransport.NewTransport namely the scopes that are then used for OAth authentication. So to build our own // http client we need to se those defaults opts = append(opts, option.WithScopes(storage.ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")) - gRT, err := htransport.NewTransport(context.Background(), rt, opts...) + gRT, err := htransport.NewTransport(context.Background(), tpt, opts...) if err != nil { return nil, err } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index c35b4f50..86058b33 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -66,7 +66,7 @@ func TestNewBucketWithConfig_ShouldCreateGRPC(t *testing.T) { err = os.Setenv("STORAGE_EMULATOR_HOST_GRPC", svr.Addr) testutil.Ok(t, err) - bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket") + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", http.DefaultTransport) testutil.Ok(t, err) // Check if the bucket is created. diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 01862fcb..9f0cea53 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -335,8 +335,15 @@ func NewBucket(logger log.Logger, ociConfig []byte, rt http.RoundTripper) (*Buck return nil, errors.Wrapf(err, "unable to create ObjectStorage client with the given oci configurations") } + var tpt http.RoundTripper + if rt != nil { + tpt = rt + } else { + tpt = CustomTransport(config) + } + httpClient := http.Client{ - Transport: CustomTransport(config), + Transport: tpt, Timeout: config.HTTPConfig.ClientTimeout, } client.HTTPClient = &httpClient diff --git a/providers/s3/s3.go b/providers/s3/s3.go index fc1d92f3..43d4d36a 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -181,7 +181,7 @@ func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTr return nil, err } - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, rt) } type overrideSignerType struct { @@ -201,7 +201,7 @@ func (s *overrideSignerType) Retrieve() (credentials.Value, error) { } // NewBucketWithConfig returns a new Bucket using the provided s3 config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { var chain []credentials.Provider // TODO(bwplotka): Don't do flags as they won't scale, use actual params like v2, v4 instead @@ -244,12 +244,12 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B // Check if a roundtripper has been set in the config // otherwise build the default transport. - var rt http.RoundTripper - if config.HTTPConfig.Transport != nil { - rt = config.HTTPConfig.Transport + var tpt http.RoundTripper + if rt != nil { + tpt = rt } else { var err error - rt, err = exthttp.DefaultTransport(config.HTTPConfig) + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) if err != nil { return nil, err } @@ -259,7 +259,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B Creds: credentials.NewChainCredentials(chain), Secure: !config.Insecure, Region: config.Region, - Transport: rt, + Transport: tpt, BucketLookup: config.BucketLookupType.MinioType(), }) if err != nil { diff --git a/providers/s3/s3_e2e_test.go b/providers/s3/s3_e2e_test.go index 4b75a014..a3fd7d62 100644 --- a/providers/s3/s3_e2e_test.go +++ b/providers/s3/s3_e2e_test.go @@ -6,6 +6,7 @@ package s3_test import ( "bytes" "context" + "net/http" "strings" "testing" @@ -37,6 +38,7 @@ func BenchmarkUpload(b *testing.B) { log.NewNopLogger(), e2ethanos.NewS3Config(bucket, m.Endpoint("https"), m.Dir()), "test-feed", + http.DefaultTransport, ) testutil.Ok(b, err) diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index cdab39c3..306718e2 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -324,7 +324,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { // Default config should return no SSE config. cfg := DefaultConfig cfg.Endpoint = endpoint - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", http.DefaultTransport) testutil.Ok(t, err) sse, err := bkt.getServerSideEncryption(context.Background()) @@ -335,7 +335,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { cfg = DefaultConfig cfg.Endpoint = endpoint cfg.SSEConfig = SSEConfig{Type: SSES3} - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", http.DefaultTransport) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -351,7 +351,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { Type: SSEKMS, KMSKeyID: "key", } - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", http.DefaultTransport) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -375,7 +375,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { KMSKeyID: "key", KMSEncryptionContext: map[string]string{"foo": "bar"}, } - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", http.DefaultTransport) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -396,7 +396,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { override, err := encrypt.NewSSEKMS("test", nil) testutil.Ok(t, err) - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", http.DefaultTransport) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.WithValue(context.Background(), sseConfigKey, override)) @@ -423,7 +423,7 @@ func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) { cfg.AccessKey = "test" cfg.SecretKey = "test" - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", http.DefaultTransport) testutil.Ok(t, err) reader, err := bkt.Get(context.Background(), "test") @@ -448,7 +448,7 @@ func TestParseConfig_CustomStorageClass(t *testing.T) { cfg.Endpoint = endpoint storageClass := "STANDARD_IA" cfg.PutUserMetadata[testCase.storageClassKey] = storageClass - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", http.DefaultTransport) testutil.Ok(t, err) testutil.Equals(t, storageClass, bkt.storageClass) }) @@ -458,7 +458,7 @@ func TestParseConfig_CustomStorageClass(t *testing.T) { func TestParseConfig_DefaultStorageClassIsZero(t *testing.T) { cfg := DefaultConfig cfg.Endpoint = endpoint - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", http.DefaultTransport) testutil.Ok(t, err) testutil.Equals(t, "", bkt.storageClass) } From 1b2aa76fbd3b484e2c3596980159eba4b4ad50e4 Mon Sep 17 00:00:00 2001 From: Vanshikav123 Date: Fri, 31 May 2024 19:20:22 +0530 Subject: [PATCH 6/7] readme update Signed-off-by: Vanshikav123 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d2e9e50d..881ea208 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ Generally, you have two ways of using `objstore` module: First is to import the provider you want e.g. [`github.com/thanos-io/objstore/providers/s3`](providers/s3) and instantiate it with available constructor (e.g. `NewBucket`). -The second option is to use the factory `NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registerer, component string,rt http.RoundTripper)` that will instantiate the object storage client based on YAML file provided. The YAML file has generally the format like this: +The second option is to use the factory `NewBucket(logger log.Logger, confContentYaml []byte, reg prometheus.Registerer, component string, rt http.RoundTripper)` that will instantiate the object storage client based on YAML file provided. The YAML file has generally the format like this: ```yaml type: @@ -114,7 +114,7 @@ config: The exact option depends on provider and are in sections below. - `NewBucket` function now accepts an `http.RoundTripper` parameter allows clients to provide a custom transport for HTTP requests, making the function more flexible , this change facilitates the use of various HTTP client configurations, including hedged HTTP transports. + `NewBucket` function now accepts an `http.RoundTripper` parameter allows clients to provide a custom transport for HTTP requests. This change allows the use of various HTTP clients, including hedged HTTP transports. > NOTE: All code snippets are auto-generated from code and up-to-date. From bd0b7b37261f1b311463825cd3af287516dbeb5c Mon Sep 17 00:00:00 2001 From: Vanshikav123 Date: Sun, 2 Jun 2024 19:11:22 +0530 Subject: [PATCH 7/7] added for other providers Signed-off-by: Vanshikav123 --- client/factory.go | 2 +- providers/obs/obs.go | 27 +++++++++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/client/factory.go b/client/factory.go index 8b17156f..6c49f60e 100644 --- a/client/factory.go +++ b/client/factory.go @@ -83,7 +83,7 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt h case string(OCI): bucket, err = oci.NewBucket(logger, config, rt) case string(OBS): - bucket, err = obs.NewBucket(logger, config) + bucket, err = obs.NewBucket(logger, config, rt) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 7bd9666b..e0a60fd2 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -7,6 +7,7 @@ import ( "context" "io" "math" + "net/http" "os" "strings" "testing" @@ -74,13 +75,13 @@ type Bucket struct { name string } -func NewBucket(logger log.Logger, conf []byte) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - return NewBucketWithConfig(logger, config) + return NewBucketWithConfig(logger, config, rt) } func parseConfig(conf []byte) (Config, error) { @@ -92,17 +93,27 @@ func parseConfig(conf []byte) (Config, error) { return config, nil } -func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, rt http.RoundTripper) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate obs config err") } + var tpt *http.Transport + var err error - rt, err := exthttp.DefaultTransport(config.HTTPConfig) - if err != nil { - return nil, errors.Wrap(err, "get http transport err") + if rt != nil { + var ok bool + tpt, ok = rt.(*http.Transport) + if !ok { + return nil, errors.New("provided RoundTripper is not an *http.Transport") + } + } else { + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, errors.Wrap(err, "get http transport err") + } } - client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt)) + client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(tpt)) if err != nil { return nil, errors.Wrap(err, "initialize obs client err") } @@ -369,7 +380,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool, location if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc) + b, err := NewBucket(log.NewNopLogger(), bc, http.DefaultTransport) if err != nil { return nil, nil, err }