feat: s3 support path-style access

Author: iwilltry42

pkg/cli/workspace.go

@@ -14,6 +14,7 @@ type workspaceProvider struct {
 	DataHome       string `usage:"The data home directory or bucket name" env:"WORKSPACE_PROVIDER_DATA_HOME"`
 	S3Bucket       string `usage:"The S3 bucket name" name:"s3-bucket" env:"WORKSPACE_PROVIDER_S3_BUCKET"`
 	S3BaseEndpoint string `usage:"The S3 base endpoint to use with S3 compatible providers" name:"s3-base-endpoint" env:"WORKSPACE_PROVIDER_S3_BASE_ENDPOINT"`
+	S3UsePathStyle bool   `usage:"Use path style addressing for S3 compatible providers" name:"s3-use-path-style" env:"WORKSPACE_PROVIDER_S3_USE_PATH_STYLE"`
 
 	client *client.Client
 }
@@ -64,6 +65,7 @@ func (w *workspaceProvider) PersistentPre(cmd *cobra.Command, _ []string) error
 		DirectoryDataHome: w.DataHome,
 		S3BucketName:      w.S3Bucket,
 		S3BaseEndpoint:    w.S3BaseEndpoint,
+		S3UsePathStyle:    w.S3UsePathStyle,
 	})
 
 	return err

pkg/client/client.go

@@ -41,6 +41,7 @@ type Options struct {
 	DirectoryDataHome string
 	S3BucketName      string
 	S3BaseEndpoint    string
+	S3UsePathStyle    bool
 }
 
 func complete(opts ...Options) Options {
@@ -56,6 +57,7 @@ func complete(opts ...Options) Options {
 		if o.S3BaseEndpoint != "" {
 			opt.S3BaseEndpoint = o.S3BaseEndpoint
 		}
+		opt.S3UsePathStyle = o.S3UsePathStyle
 	}
 
 	if opt.DirectoryDataHome == "" {
@@ -71,7 +73,7 @@ func New(ctx context.Context, opts ...Options) (*Client, error) {
 	var s3 workspaceFactory
 	if opt.S3BucketName != "" {
 		var err error
-		s3, err = newS3(ctx, opt.S3BucketName, opt.S3BaseEndpoint)
+		s3, err = newS3(ctx, opt.S3BucketName, opt.S3BaseEndpoint, opt.S3UsePathStyle)
 		if err != nil {
 			return nil, err
 		}

pkg/client/directory_test.go

@@ -29,7 +29,7 @@ func TestMain(m *testing.M) {
 	dirPrv, _ = directoryFactory.New(directoryTestingID)
 
 	if !skipS3Tests {
-		s3Factory, _ = newS3(context.Background(), os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET"), os.Getenv("WORKSPACE_PROVIDER_S3_BASE_ENDPOINT"))
+		s3Factory, _ = newS3(context.Background(), os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET"), os.Getenv("WORKSPACE_PROVIDER_S3_BASE_ENDPOINT"), os.Getenv("WORKSPACE_PROVIDER_S3_USE_PATH_STYLE") == "true")
 		// This won't ever error because it doesn't create anything.
 		s3TestingID = s3Factory.Create()
 

pkg/client/s3.go

@@ -21,7 +21,7 @@ import (
 	"github.com/gabriel-vasile/mimetype"
 )
 
-func newS3(ctx context.Context, bucket, baseEndpoint string) (workspaceFactory, error) {
+func newS3(ctx context.Context, bucket string, baseEndpoint string, usePathStyle bool) (workspaceFactory, error) {
 	cfg, err := config.LoadDefaultConfig(ctx)
 	if err != nil {
 		return nil, err
@@ -31,6 +31,7 @@ func newS3(ctx context.Context, bucket, baseEndpoint string) (workspaceFactory,
 		if baseEndpoint != "" {
 			o.BaseEndpoint = aws.String(baseEndpoint)
 		}
+		o.UsePathStyle = usePathStyle // often required e.g. for MinIO which requires extra configuration for virtual hosted-style requests
 	})
 	return &s3Provider{
 		bucket: bucket,