feat: s3 support path-style access

pkg/cli/workspace.go

@@ -14,6 +14,7 @@ type workspaceProvider struct {
 	DataHome              string `usage:"The data home directory or bucket name" env:"WORKSPACE_PROVIDER_DATA_HOME"`
 	S3Bucket              string `usage:"The S3 bucket name" name:"s3-bucket" env:"WORKSPACE_PROVIDER_S3_BUCKET"`
 	S3BaseEndpoint        string `usage:"The S3 base endpoint to use with S3 compatible providers" name:"s3-base-endpoint" env:"WORKSPACE_PROVIDER_S3_BASE_ENDPOINT"`
+	S3UsePathStyle        bool   `usage:"Use path style addressing for S3 compatible providers" name:"s3-use-path-style" env:"WORKSPACE_PROVIDER_S3_USE_PATH_STYLE"`
 	AzureContainer        string `usage:"The Azure container name" name:"azure-container" env:"WORKSPACE_PROVIDER_AZURE_CONTAINER"`
 	AzureConnectionString string `usage:"The Azure connection string" name:"azure-connection-string" env:"WORKSPACE_PROVIDER_AZURE_CONNECTION_STRING"`
 
@@ -73,6 +74,7 @@ func (w *workspaceProvider) PersistentPre(cmd *cobra.Command, _ []string) error
 		DirectoryDataHome:     w.DataHome,
 		S3BucketName:          w.S3Bucket,
 		S3BaseEndpoint:        w.S3BaseEndpoint,
+		S3UsePathStyle:        w.S3UsePathStyle,
 		AzureContainerName:    w.AzureContainer,
 		AzureConnectionString: w.AzureConnectionString,
 	})

pkg/client/client.go

@@ -42,6 +42,7 @@ type Options struct {
 	DirectoryDataHome     string
 	S3BucketName          string
 	S3BaseEndpoint        string
+	S3UsePathStyle        bool
 	AzureContainerName    string
 	AzureConnectionString string
 }
@@ -59,6 +60,7 @@ func complete(opts ...Options) Options {
 		if o.S3BaseEndpoint != "" {
 			opt.S3BaseEndpoint = o.S3BaseEndpoint
 		}
+		opt.S3UsePathStyle = o.S3UsePathStyle
 		if o.AzureContainerName != "" {
 			opt.AzureContainerName = o.AzureContainerName
 		}
@@ -82,7 +84,7 @@ func New(ctx context.Context, opts ...Options) (*Client, error) {
 	}
 
 	if opt.S3BucketName != "" {
-		factory, err := newS3(ctx, opt.S3BucketName, opt.S3BaseEndpoint)
+		factory, err := newS3(ctx, opt.S3BucketName, opt.S3BaseEndpoint, opt.S3UsePathStyle)
 		if err != nil {
 			return nil, err
 		}

pkg/client/client_test.go

@@ -15,6 +15,7 @@ import (
 var c, _ = New(context.Background(), Options{
 	S3BucketName:          os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET"),
 	S3BaseEndpoint:        os.Getenv("WORKSPACE_PROVIDER_S3_BASE_ENDPOINT"),
+	S3UsePathStyle:        os.Getenv("WORKSPACE_PROVIDER_S3_USE_PATH_STYLE") == "true",
 	AzureContainerName:    os.Getenv("WORKSPACE_PROVIDER_AZURE_CONTAINER"),
 	AzureConnectionString: os.Getenv("WORKSPACE_PROVIDER_AZURE_CONNECTION_STRING"),
 })
@@ -83,7 +84,7 @@ func TestCreateAndRmS3Provider(t *testing.T) {
 	bucket, dir, _ := strings.Cut(strings.TrimPrefix(id, S3Provider+"://"), "/")
 	testS3Provider := &s3Provider{
 		bucket: bucket,
-		client: s3Prv.client,
+		client: s3TestSetups[0].provider.client,
 	}
 
 	// Nothing should be created
@@ -318,6 +319,10 @@ func TestCreateAndRmS3ProviderFromDirectoryProvider(t *testing.T) {
 		t.Errorf("unexpected error when listing workspaceContent: %v", err)
 	}
 
+	if len(workspaceContent) == 0 {
+		t.Fatalf("workspaceContent is empty")
+	}
+
 	if len(workspaceContent) != 1 {
 		t.Errorf("unexpected number of workspaceContent: %d", len(workspaceContent))
 	}

pkg/client/directory_test.go

@@ -13,17 +13,22 @@ import (
 	"testing"
 )
 
+type s3TestSetup struct {
+	name      string
+	factory   workspaceFactory
+	testingID string
+	provider  *s3Provider
+}
+
 var (
 	directoryFactory   workspaceFactory
-	s3Factory          workspaceFactory
-	azureFactory       workspaceFactory
 	directoryTestingID string
-	s3TestingID        string
-	azureTestingID     string
 	dirPrv             workspaceClient
-	s3Prv              *s3Provider
-	azurePrv           *azureProvider
+	s3TestSetups       []s3TestSetup
 	skipS3Tests        = os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == ""
+	azureFactory       workspaceFactory
+	azureTestingID     string
+	azurePrv           *azureProvider
 	skipAzureTests     = os.Getenv("WORKSPACE_PROVIDER_AZURE_CONNECTION_STRING") == "" || os.Getenv("WORKSPACE_PROVIDER_AZURE_CONTAINER") == ""
 )
 
@@ -33,12 +38,30 @@ func TestMain(m *testing.M) {
 	dirPrv, _ = directoryFactory.New(directoryTestingID)
 
 	if !skipS3Tests {
-		s3Factory, _ = newS3(context.Background(), os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET"), os.Getenv("WORKSPACE_PROVIDER_S3_BASE_ENDPOINT"))
-		// This won't ever error because it doesn't create anything.
-		s3TestingID = s3Factory.Create()
+		if os.Getenv("WORKSPACE_PROVIDER_S3_USE_PATH_STYLE") != "true" {
+			s3Factory, _ := newS3(context.Background(), os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET"), os.Getenv("WORKSPACE_PROVIDER_S3_BASE_ENDPOINT"), false)
+			// This won't ever error because it doesn't create anything.
+			s3TestingID := s3Factory.Create()
+
+			s3Client, _ := s3Factory.New(s3TestingID)
+			s3Prv := s3Client.(*s3Provider)
+			s3TestSetups = append(s3TestSetups, s3TestSetup{
+				name:      "default",
+				factory:   s3Factory,
+				testingID: s3TestingID,
+				provider:  s3Prv,
+			})
+		}
 
-		s3Client, _ := s3Factory.New(s3TestingID)
-		s3Prv = s3Client.(*s3Provider)
+		s3PathStyleFactory, _ := newS3(context.Background(), os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET"), os.Getenv("WORKSPACE_PROVIDER_S3_BASE_ENDPOINT"), true)
+		s3PathStyleTestingID := s3PathStyleFactory.Create()
+		s3PathStyleClient, _ := s3PathStyleFactory.New(s3PathStyleTestingID)
+		s3TestSetups = append(s3TestSetups, s3TestSetup{
+			name:      "use-path-style",
+			factory:   s3PathStyleFactory,
+			testingID: s3PathStyleTestingID,
+			provider:  s3PathStyleClient.(*s3Provider),
+		})
 	}
 
 	if !skipAzureTests {
@@ -58,8 +81,10 @@ func TestMain(m *testing.M) {
 	}
 
 	if !skipS3Tests {
-		if err := s3Factory.Rm(context.Background(), s3TestingID); err != nil {
-			errs = append(errs, fmt.Errorf("error removing s3 workspace: %v", err))
+		for _, s3TS := range s3TestSetups {
+			if err := s3TS.factory.Rm(context.Background(), s3TS.testingID); err != nil {
+				errs = append(errs, fmt.Errorf("error removing s3 workspace: %v", err))
+			}
 		}
 	}
 

pkg/client/s3.go

@@ -21,7 +21,7 @@ import (
 	"github.com/gabriel-vasile/mimetype"
 )
 
-func newS3(ctx context.Context, bucket, baseEndpoint string) (workspaceFactory, error) {
+func newS3(ctx context.Context, bucket string, baseEndpoint string, usePathStyle bool) (workspaceFactory, error) {
 	cfg, err := config.LoadDefaultConfig(ctx)
 	if err != nil {
 		return nil, err
@@ -31,6 +31,7 @@ func newS3(ctx context.Context, bucket, baseEndpoint string) (workspaceFactory,
 		if baseEndpoint != "" {
 			o.BaseEndpoint = aws.String(baseEndpoint)
 		}
+		o.UsePathStyle = usePathStyle // often required e.g. for MinIO which requires extra configuration for virtual hosted-style requests
 	})
 	return &s3Provider{
 		bucket: bucket,