From af4c9dde8e54283ba7556ce8bb91042272064c7a Mon Sep 17 00:00:00 2001 From: Khurram Date: Wed, 9 Oct 2024 23:21:45 +0530 Subject: [PATCH] blob/s3blob: Add disable_https and use_path_style query param options --- blob/s3blob/s3blob.go | 38 ++++++++++++++++++++++++++++++++------ blob/s3blob/s3blob_test.go | 8 ++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/blob/s3blob/s3blob.go b/blob/s3blob/s3blob.go index 7ed47e444e..f575649b7e 100644 --- a/blob/s3blob/s3blob.go +++ b/blob/s3blob/s3blob.go @@ -150,9 +150,11 @@ type URLOpener struct { } const ( - sseTypeParamKey = "ssetype" - kmsKeyIdParamKey = "kmskeyid" - accelerateParamKey = "accelerate" + sseTypeParamKey = "ssetype" + kmsKeyIdParamKey = "kmskeyid" + accelerateParamKey = "accelerate" + usePathStyleParamkey = "use_path_style" + disableHTTPSParamKey = "disable_https" ) func toServerSideEncryptionType(value string) (typesv2.ServerSideEncryption, error) { @@ -195,13 +197,37 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket } if o.UseV2 { + opts := []func(*s3v2.Options){ + func(o *s3v2.Options) { + o.UseAccelerate = accelerate + }, + } + if disableHTTPSParam := q.Get(disableHTTPSParamKey); disableHTTPSParam != "" { + q.Del(disableHTTPSParamKey) + value, err := strconv.ParseBool(disableHTTPSParam) + if err != nil { + return nil, fmt.Errorf("invalid value for %q: %v", disableHTTPSParamKey, err) + } + opts = append(opts, func(o *s3v2.Options) { + o.EndpointOptions.DisableHTTPS = value + }) + } + if usePathStyleParam := q.Get(usePathStyleParamkey); usePathStyleParam != "" { + q.Del(usePathStyleParamkey) + value, err := strconv.ParseBool(usePathStyleParam) + if err != nil { + return nil, fmt.Errorf("invalid value for %q: %v", usePathStyleParamkey, err) + } + opts = append(opts, func(o *s3v2.Options) { + o.UsePathStyle = value + }) + } + cfg, err := gcaws.V2ConfigFromURLParams(ctx, q) if err != nil { return nil, fmt.Errorf("open bucket %v: %v", u, err) } - clientV2 := s3v2.NewFromConfig(cfg, func(o *s3v2.Options) { - o.UseAccelerate = accelerate - }) + clientV2 := s3v2.NewFromConfig(cfg, opts...) return OpenBucketV2(ctx, clientV2, u.Host, &o.Options) } diff --git a/blob/s3blob/s3blob_test.go b/blob/s3blob/s3blob_test.go index 8d54c521ca..2639b74d70 100644 --- a/blob/s3blob/s3blob_test.go +++ b/blob/s3blob/s3blob_test.go @@ -482,6 +482,10 @@ func TestOpenBucketFromURL(t *testing.T) { {"s3://mybucket?fips=true", false}, // OK, use S3 Transfer accleration and dual stack endpoints (v1) {"s3://mybucket?awssdk=v1&accelerate=true&dualstack=true", false}, + // OK, use use_path_style + {"s3://mybucket?use_path_style=true", false}, + // OK, use disable_https + {"s3://mybucket?disable_https=true", false}, // OK, use FIPS endpoints (v1) {"s3://mybucket?awssdk=v1&fips=true", false}, // Invalid accelerate (v1) @@ -500,6 +504,10 @@ func TestOpenBucketFromURL(t *testing.T) { {"s3://mybucket?ssetype=aws:notkmsoraes&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", true}, // Invalid parameter together with a valid one. {"s3://mybucket?profile=main¶m=value", true}, + // Invalid use_path_style (v1) + {"s3://mybucket?awssdk=v1&usePathStyle=bad", true}, + // Invalid disable_https (v2) + {"s3://mybucket?usePathStyle=bad", true}, // Invalid parameter. {"s3://mybucket?param=value", true}, }