From 1b92340bbd66330c1bbbac6b656255cc6ccbcad8 Mon Sep 17 00:00:00 2001 From: Zhiyu Wang Date: Thu, 24 Aug 2023 03:12:02 +0000 Subject: [PATCH] Feat: support bucket lookup type Signed-off-by: Zhiyu Wang --- api/v1beta2/bucket_types.go | 17 ++++++ .../source.toolkit.fluxcd.io_buckets.yaml | 10 ++++ docs/api/v1beta2/source.md | 28 ++++++++++ docs/spec/v1beta2/buckets.md | 11 ++++ pkg/minio/minio.go | 18 +++++- pkg/minio/minio_test.go | 56 +++++++++++++++++++ 6 files changed, 139 insertions(+), 1 deletion(-) diff --git a/api/v1beta2/bucket_types.go b/api/v1beta2/bucket_types.go index c9b748a54..d16d493cc 100644 --- a/api/v1beta2/bucket_types.go +++ b/api/v1beta2/bucket_types.go @@ -46,6 +46,15 @@ const ( AzureBucketProvider string = "azure" ) +const ( + // BucketLookupAuto is automatic determine URL access + BucketLookupAuto = "auto" + // BucketLookupDNS is virtual hosted style URL access + BucketLookupDNS = "dns" + // BucketLookupPath is path style URL access + BucketLookupPath = "path" +) + // BucketSpec specifies the required configuration to produce an Artifact for // an object storage bucket. type BucketSpec struct { @@ -109,6 +118,14 @@ type BucketSpec struct { // NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092 // +optional AccessFrom *acl.AccessFrom `json:"accessFrom,omitempty"` + + // LookupType is type of url lookup supported by bucket server, + // only takes effect when the Provider is 'generic'. + // Defaults to 'path'. + // +kubebuilder:validation:Enum=auto;dns;path + // +kubebuilder:default:=path + // +optional + LookupType *string `json:"lookupType,omitempty"` } // BucketStatus records the observed state of a Bucket. diff --git a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml index 57e644a88..d33dd361b 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml @@ -331,6 +331,16 @@ spec: to ensure efficient use of resources. pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$ type: string + lookupType: + default: path + description: LookupType is type of url lookup supported by bucket + server, only takes effect when the Provider is 'generic'. Defaults + to 'path'. + enum: + - auto + - dns + - path + type: string provider: default: generic description: Provider of the object storage bucket. Defaults to 'generic', diff --git a/docs/api/v1beta2/source.md b/docs/api/v1beta2/source.md index 3d58db692..3e3f9e27f 100644 --- a/docs/api/v1beta2/source.md +++ b/docs/api/v1beta2/source.md @@ -223,6 +223,20 @@ references to this object. NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092

+ + +lookupType
+ +string + + + +(Optional) +

LookupType is type of url lookup supported by bucket server, +only takes effect when the Provider is ‘generic’. +Defaults to ‘path’.

+ + @@ -1507,6 +1521,20 @@ references to this object. NOTE: Not implemented, provisional as of https://github.com/fluxcd/flux2/pull/2092

+ + +lookupType
+ +string + + + +(Optional) +

LookupType is type of url lookup supported by bucket server, +only takes effect when the Provider is ‘generic’. +Defaults to ‘path’.

+ + diff --git a/docs/spec/v1beta2/buckets.md b/docs/spec/v1beta2/buckets.md index eb7eb8018..f0a2da496 100644 --- a/docs/spec/v1beta2/buckets.md +++ b/docs/spec/v1beta2/buckets.md @@ -806,6 +806,17 @@ Artifact. When the field is set to `false` or removed, it will resume. For practical information, see [suspending and resuming](#suspending-and-resuming). +### Lookup type + +`.spec.lookupType` is an optional field to select which url lookup type to access bucket. For example, `dns` will use virtual-hosted–style URL access , `path` will use path-style URL access, or you can also use `auto` to automatic. + +Supported options are: +- auto +- dns +- path + +It only takes effect when the [Provider](#provider) is [generic](#generic). If you do not specify `.spec.lookupType`, it defaults to `path`. + ## Working with Buckets ### Excluding files diff --git a/pkg/minio/minio.go b/pkg/minio/minio.go index f1930dbd5..5b40cbf8c 100644 --- a/pkg/minio/minio.go +++ b/pkg/minio/minio.go @@ -40,7 +40,7 @@ func NewClient(bucket *sourcev1.Bucket, secret *corev1.Secret) (*MinioClient, er opt := minio.Options{ Region: bucket.Spec.Region, Secure: !bucket.Spec.Insecure, - BucketLookup: minio.BucketLookupPath, + BucketLookup: getLookupType(bucket), } if secret != nil { @@ -133,3 +133,19 @@ func (c *MinioClient) ObjectIsNotFound(err error) bool { func (c *MinioClient) Close(_ context.Context) { // Minio client does not provide a close method } + +var lookupNameToEnumType = map[string]minio.BucketLookupType{ + sourcev1.BucketLookupAuto: minio.BucketLookupAuto, + sourcev1.BucketLookupDNS: minio.BucketLookupDNS, + sourcev1.BucketLookupPath: minio.BucketLookupPath, +} + +func getLookupType(bucket *sourcev1.Bucket) minio.BucketLookupType { + if bucket.Spec.LookupType != nil { + if t, ok := lookupNameToEnumType[*bucket.Spec.LookupType]; ok { + return t + } + } + // default value + return minio.BucketLookupPath +} diff --git a/pkg/minio/minio_test.go b/pkg/minio/minio_test.go index 3e1598157..1a9009577 100644 --- a/pkg/minio/minio_test.go +++ b/pkg/minio/minio_test.go @@ -28,6 +28,7 @@ import ( "github.com/google/uuid" miniov7 "github.com/minio/minio-go/v7" + "github.com/onsi/gomega" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" "gotest.tools/assert" @@ -349,3 +350,58 @@ func getObjectFile() string { timeout: 30s ` } + +func TestGetLookupType(t *testing.T) { + typeUnexpect := "unexpect" + typeAuto := "auto" + typeDNS := "dns" + typePath := "path" + + tests := []struct { + name string + bucket sourcev1.Bucket + want miniov7.BucketLookupType + }{ + { + name: "lookup type defalut", + bucket: sourcev1.Bucket{Spec: sourcev1.BucketSpec{}}, + want: miniov7.BucketLookupPath, + }, + { + name: "lookup type unexpect string", + bucket: sourcev1.Bucket{ + Spec: sourcev1.BucketSpec{LookupType: &typeUnexpect}, + }, + want: miniov7.BucketLookupPath, + }, + { + name: "lookup type auto", + bucket: sourcev1.Bucket{ + Spec: sourcev1.BucketSpec{LookupType: &typeAuto}, + }, + want: miniov7.BucketLookupAuto, + }, + { + name: "lookup type dns", + bucket: sourcev1.Bucket{ + Spec: sourcev1.BucketSpec{LookupType: &typeDNS}, + }, + want: miniov7.BucketLookupDNS, + }, + { + name: "lookup type path", + bucket: sourcev1.Bucket{ + Spec: sourcev1.BucketSpec{LookupType: &typePath}, + }, + want: miniov7.BucketLookupPath, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gomega.NewWithT(t) + + got := getLookupType(&tt.bucket) + g.Expect(got).To(gomega.Equal(tt.want)) + }) + } +}