Skip to content

Commit

Permalink
Add disableHTTPS and usePathStyle s3v2.Options as query param
Browse files Browse the repository at this point in the history
This ensures that we can use blob/blob with s3 compatible storage like minio.
Pass use_path_style=true to force PathStyle for s3.
Pass disable_https=true to disable tls for endpoint.
  • Loading branch information
khrm committed Oct 9, 2024
1 parent e5b1bc6 commit 588ba91
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 7 deletions.
40 changes: 33 additions & 7 deletions blob/s3blob/s3blob.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,11 @@ type URLOpener struct {
}

const (
sseTypeParamKey = "ssetype"
kmsKeyIdParamKey = "kmskeyid"
accelerateParamKey = "accelerate"
sseTypeParamKey = "ssetype"
kmsKeyIdParamKey = "kmskeyid"
accelerateParamKey = "accelerate"
usePathStyleParamkey = "use_path_style"
disableHTTPSParamKey = "disable_https"
)

func toServerSideEncryptionType(value string) (typesv2.ServerSideEncryption, error) {
Expand Down Expand Up @@ -184,7 +186,7 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
o.Options.KMSEncryptionID = kmsKeyID
}

accelerate := false
var accelerate bool
if accelerateParam := q.Get(accelerateParamKey); accelerateParam != "" {
q.Del(accelerateParamKey)
var err error
Expand All @@ -195,13 +197,37 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
}

if o.UseV2 {
opts := []func(*s3v2.Options){
func(o *s3v2.Options) {
o.UseAccelerate = accelerate
},
}
if disableHTTPSParam := q.Get(disableHTTPSParamKey); disableHTTPSParam != "" {
q.Del(disableHTTPSParamKey)
value, err := strconv.ParseBool(disableHTTPSParam)
if err != nil {
return nil, fmt.Errorf("invalid value for %q: %v", disableHTTPSParamKey, err)
}
opts = append(opts, func(o *s3v2.Options) {
o.EndpointOptions.DisableHTTPS = value
})
}
if usePathStyleParam := q.Get(usePathStyleParamkey); usePathStyleParam != "" {
q.Del(usePathStyleParamkey)
value, err := strconv.ParseBool(usePathStyleParam)
if err != nil {
return nil, fmt.Errorf("invalid value for %q: %v", usePathStyleParamkey, err)
}
opts = append(opts, func(o *s3v2.Options) {
o.UsePathStyle = value
})
}

cfg, err := gcaws.V2ConfigFromURLParams(ctx, q)
if err != nil {
return nil, fmt.Errorf("open bucket %v: %v", u, err)
}
clientV2 := s3v2.NewFromConfig(cfg, func(o *s3v2.Options) {
o.UseAccelerate = accelerate
})
clientV2 := s3v2.NewFromConfig(cfg, opts...)

return OpenBucketV2(ctx, clientV2, u.Host, &o.Options)
}
Expand Down
8 changes: 8 additions & 0 deletions blob/s3blob/s3blob_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,10 @@ func TestOpenBucketFromURL(t *testing.T) {
{"s3://mybucket?fips=true", false},
// OK, use S3 Transfer accleration and dual stack endpoints (v1)
{"s3://mybucket?awssdk=v1&accelerate=true&dualstack=true", false},
// OK, use use_path_style
{"s3://mybucket?use_path_style=true", false},
// OK, use disable_https
{"s3://mybucket?disable_https=true", false},
// OK, use FIPS endpoints (v1)
{"s3://mybucket?awssdk=v1&fips=true", false},
// Invalid accelerate (v1)
Expand All @@ -500,6 +504,10 @@ func TestOpenBucketFromURL(t *testing.T) {
{"s3://mybucket?ssetype=aws:notkmsoraes&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", true},
// Invalid parameter together with a valid one.
{"s3://mybucket?profile=main&param=value", true},
// Invalid use_path_style (v1)
{"s3://mybucket?awssdk=v1&usePathStyle=bad", true},
// Invalid disable_https (v2)
{"s3://mybucket?usePathStyle=bad", true},
// Invalid parameter.
{"s3://mybucket?param=value", true},
}
Expand Down

0 comments on commit 588ba91

Please sign in to comment.