+
insecure
bool
diff --git a/docs/spec/v1beta2/buckets.md b/docs/spec/v1beta2/buckets.md
index 630f9f5e5..6f68735f0 100644
--- a/docs/spec/v1beta2/buckets.md
+++ b/docs/spec/v1beta2/buckets.md
@@ -749,6 +749,23 @@ HTTP endpoint requires enabling [`.spec.insecure`](#insecure).
Some endpoints require the specification of a [`.spec.region`](#region),
see [Provider](#provider) for more (provider specific) examples.
+### STS
+
+`.spec.sts` is an optional field for specifying the Security Token Service
+configuration. A Security Token Service (STS) is a web service that issues
+temporary security credentials. By adding this field, one may specify the
+STS endpoint from where temporary credentials will be fetched.
+
+If using `.spec.sts`, the following fields are required:
+
+- `.spec.sts.provider`, the Security Token Service provider. The only supported
+ option is `aws`.
+- `.spec.sts.endpoint`, the HTTP/S endpoint of the Security Token Service. In
+ the case of AWS, this can be `https://sts.amazonaws.com`, or a Regional STS
+ Endpoint, or an Interface Endpoint created inside a VPC.
+
+This field is only supported for the `aws` bucket provider.
+
### Bucket name
`.spec.bucketName` is a required field that specifies which object storage
diff --git a/internal/controller/bucket_controller.go b/internal/controller/bucket_controller.go
index 656e5d704..261eaf3e9 100644
--- a/internal/controller/bucket_controller.go
+++ b/internal/controller/bucket_controller.go
@@ -463,6 +463,11 @@ func (r *BucketReconciler) reconcileSource(ctx context.Context, sp *patch.Serial
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, "%s", e)
return sreconcile.ResultEmpty, e
}
+ if err = minio.ValidateSTSConfig(obj); err != nil {
+ e := serror.NewGeneric(err, sourcev1.AuthenticationFailedReason)
+ conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, "%s", e)
+ return sreconcile.ResultEmpty, e
+ }
tlsConfig, err := r.getTLSConfig(ctx, obj)
if err != nil {
e := serror.NewGeneric(err, sourcev1.AuthenticationFailedReason)
diff --git a/internal/controller/bucket_controller_test.go b/internal/controller/bucket_controller_test.go
index 11c99613f..8b50a1de7 100644
--- a/internal/controller/bucket_controller_test.go
+++ b/internal/controller/bucket_controller_test.go
@@ -608,6 +608,23 @@ func TestBucketReconciler_reconcileSource_generic(t *testing.T) {
*conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
},
},
+ {
+ name: "Observes invalid STS configuration",
+ bucketName: "dummy",
+ beforeFunc: func(obj *bucketv1.Bucket) {
+ obj.Spec.Provider = "some-provider"
+ obj.Spec.STS = &bucketv1.BucketSTSSpec{}
+ conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
+ conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
+ },
+ wantErr: true,
+ assertIndex: index.NewDigester(),
+ assertConditions: []metav1.Condition{
+ *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.AuthenticationFailedReason, "STS configuration is not supported for 'some-provider' bucket provider"),
+ *conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"),
+ *conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
+ },
+ },
{
name: "Transient bucket name API failure",
beforeFunc: func(obj *bucketv1.Bucket) {
diff --git a/pkg/minio/minio.go b/pkg/minio/minio.go
index 8225135fe..dd7c6831f 100644
--- a/pkg/minio/minio.go
+++ b/pkg/minio/minio.go
@@ -23,6 +23,7 @@ import (
"fmt"
"net/http"
"net/url"
+ "regexp"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@@ -71,14 +72,10 @@ func WithProxyURL(proxyURL *url.URL) Option {
// NewClient creates a new Minio storage client.
func NewClient(bucket *sourcev1.Bucket, opts ...Option) (*MinioClient, error) {
-
var o options
for _, opt := range opts {
opt(&o)
}
- secret := o.secret
- tlsConfig := o.tlsConfig
- proxyURL := o.proxyURL
minioOpts := minio.Options{
Region: bucket.Spec.Region,
@@ -88,32 +85,24 @@ func NewClient(bucket *sourcev1.Bucket, opts ...Option) (*MinioClient, error) {
// auto access, which we believe can cover most use cases.
}
- if secret != nil {
- var accessKey, secretKey string
- if k, ok := secret.Data["accesskey"]; ok {
- accessKey = string(k)
- }
- if k, ok := secret.Data["secretkey"]; ok {
- secretKey = string(k)
- }
- if accessKey != "" && secretKey != "" {
- minioOpts.Creds = credentials.NewStaticV4(accessKey, secretKey, "")
- }
- } else if bucket.Spec.Provider == sourcev1.AmazonBucketProvider {
- minioOpts.Creds = credentials.NewIAM("")
+ switch bucketProvider := bucket.Spec.Provider; {
+ case o.secret != nil:
+ minioOpts.Creds = newCredsFromSecret(o.secret)
+ case bucketProvider == sourcev1.AmazonBucketProvider:
+ minioOpts.Creds = newAWSCreds(bucket, o.proxyURL)
}
var transportOpts []func(*http.Transport)
- if minioOpts.Secure && tlsConfig != nil {
+ if minioOpts.Secure && o.tlsConfig != nil {
transportOpts = append(transportOpts, func(t *http.Transport) {
- t.TLSClientConfig = tlsConfig.Clone()
+ t.TLSClientConfig = o.tlsConfig.Clone()
})
}
- if proxyURL != nil {
+ if o.proxyURL != nil {
transportOpts = append(transportOpts, func(t *http.Transport) {
- t.Proxy = http.ProxyURL(proxyURL)
+ t.Proxy = http.ProxyURL(o.proxyURL)
})
}
@@ -135,6 +124,42 @@ func NewClient(bucket *sourcev1.Bucket, opts ...Option) (*MinioClient, error) {
return &MinioClient{Client: client}, nil
}
+// newCredsFromSecret creates a new Minio credentials object from the provided
+// secret.
+func newCredsFromSecret(secret *corev1.Secret) *credentials.Credentials {
+ var accessKey, secretKey string
+ if k, ok := secret.Data["accesskey"]; ok {
+ accessKey = string(k)
+ }
+ if k, ok := secret.Data["secretkey"]; ok {
+ secretKey = string(k)
+ }
+ if accessKey != "" && secretKey != "" {
+ return credentials.NewStaticV4(accessKey, secretKey, "")
+ }
+ return nil
+}
+
+// newAWSCreds creates a new Minio credentials object for `aws` bucket provider.
+func newAWSCreds(bucket *sourcev1.Bucket, proxyURL *url.URL) *credentials.Credentials {
+ stsEndpoint := ""
+ if sts := bucket.Spec.STS; sts != nil {
+ stsEndpoint = sts.Endpoint
+ }
+
+ creds := credentials.NewIAM(stsEndpoint)
+ if proxyURL != nil {
+ transport := http.DefaultTransport.(*http.Transport).Clone()
+ transport.Proxy = http.ProxyURL(proxyURL)
+ client := &http.Client{Transport: transport}
+ creds = credentials.New(&credentials.IAM{
+ Client: client,
+ Endpoint: stsEndpoint,
+ })
+ }
+ return creds
+}
+
// ValidateSecret validates the credential secret. The provided Secret may
// be nil.
func ValidateSecret(secret *corev1.Secret) error {
@@ -151,6 +176,42 @@ func ValidateSecret(secret *corev1.Secret) error {
return nil
}
+const stsEndpointPattern = `^(http|https)://.*$`
+
+var stsEndpointRegex = regexp.MustCompile(stsEndpointPattern)
+
+// ValidateSTSConfig validates the STS provider. The STS configuration for the
+// provided Bucket may be nil.
+func ValidateSTSConfig(bucket *sourcev1.Bucket) error {
+ sts := bucket.Spec.STS
+ if sts == nil {
+ return nil
+ }
+
+ // Validate the STS provider compatibility with the bucket provider.
+ errProviderIncompatbility := fmt.Errorf("STS provider '%s' is not supported for '%s' bucket provider",
+ bucket.Spec.STS.Provider, bucket.Spec.Provider)
+ switch {
+ case bucket.Spec.Provider == sourcev1.AmazonBucketProvider:
+ switch sts.Provider {
+ case sourcev1.STSProviderAmazon:
+ default:
+ return errProviderIncompatbility
+ }
+ default:
+ return fmt.Errorf("STS configuration is not supported for '%s' bucket provider",
+ bucket.Spec.Provider)
+ }
+
+ // Validate the STS endpoint.
+ if !stsEndpointRegex.MatchString(sts.Endpoint) {
+ return fmt.Errorf("STS endpoint '%s' does not match pattern %q",
+ sts.Endpoint, stsEndpointPattern)
+ }
+
+ return nil
+}
+
// FGetObject gets the object from the provided object storage bucket, and
// writes it to targetPath.
// It returns the etag of the successfully fetched file, or any error.
diff --git a/pkg/minio/minio_test.go b/pkg/minio/minio_test.go
index 223a9181b..b3795b280 100644
--- a/pkg/minio/minio_test.go
+++ b/pkg/minio/minio_test.go
@@ -20,10 +20,10 @@ import (
"context"
"crypto/tls"
"crypto/x509"
+ "encoding/json"
"errors"
"fmt"
"log"
- "net"
"net/http"
"net/url"
"os"
@@ -32,9 +32,9 @@ import (
"testing"
"time"
- "github.com/elazarl/goproxy"
"github.com/google/uuid"
miniov7 "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
"gotest.tools/assert"
@@ -45,6 +45,8 @@ import (
"github.com/fluxcd/pkg/sourceignore"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
+ testlistener "github.com/fluxcd/source-controller/tests/listener"
+ testproxy "github.com/fluxcd/source-controller/tests/proxy"
)
const (
@@ -244,34 +246,153 @@ func TestFGetObject(t *testing.T) {
assert.NilError(t, err)
}
-func TestNewClientAndFGetObjectWithProxy(t *testing.T) {
+func TestNewClientAndFGetObjectWithSTSEndpoint(t *testing.T) {
+ // start a mock STS server
+ stsListener, stsAddr, stsPort := testlistener.New(t)
+ stsEndpoint := fmt.Sprintf("http://%s", stsAddr)
+ stsHandler := http.NewServeMux()
+ stsHandler.HandleFunc("PUT "+credentials.TokenPath,
+ func(w http.ResponseWriter, r *http.Request) {
+ _, err := w.Write([]byte("mock-token"))
+ assert.NilError(t, err)
+ })
+ stsHandler.HandleFunc("GET "+credentials.DefaultIAMSecurityCredsPath,
+ func(w http.ResponseWriter, r *http.Request) {
+ token := r.Header.Get(credentials.TokenRequestHeader)
+ assert.Equal(t, token, "mock-token")
+ _, err := w.Write([]byte("mock-role"))
+ assert.NilError(t, err)
+ })
+ var roleCredsRetrieved bool
+ stsHandler.HandleFunc("GET "+credentials.DefaultIAMSecurityCredsPath+"mock-role",
+ func(w http.ResponseWriter, r *http.Request) {
+ token := r.Header.Get(credentials.TokenRequestHeader)
+ assert.Equal(t, token, "mock-token")
+ err := json.NewEncoder(w).Encode(map[string]any{
+ "Code": "Success",
+ "AccessKeyID": testMinioRootUser,
+ "SecretAccessKey": testMinioRootPassword,
+ })
+ assert.NilError(t, err)
+ roleCredsRetrieved = true
+ })
+ stsServer := &http.Server{
+ Addr: stsAddr,
+ Handler: stsHandler,
+ }
+ go stsServer.Serve(stsListener)
+ defer stsServer.Shutdown(context.Background())
+
// start proxy
- proxyListener, err := net.Listen("tcp", ":0")
- assert.NilError(t, err, "could not start proxy server")
- defer proxyListener.Close()
- proxyAddr := proxyListener.Addr().String()
- proxyHandler := goproxy.NewProxyHttpServer()
- proxyHandler.Verbose = true
- proxyServer := &http.Server{
- Addr: proxyAddr,
- Handler: proxyHandler,
+ proxyAddr, proxyPort := testproxy.New(t)
+
+ tests := []struct {
+ name string
+ provider string
+ stsSpec *sourcev1.BucketSTSSpec
+ opts []Option
+ err string
+ }{
+ {
+ name: "with correct endpoint",
+ provider: "aws",
+ stsSpec: &sourcev1.BucketSTSSpec{
+ Provider: "aws",
+ Endpoint: stsEndpoint,
+ },
+ },
+ {
+ name: "with incorrect endpoint",
+ provider: "aws",
+ stsSpec: &sourcev1.BucketSTSSpec{
+ Provider: "aws",
+ Endpoint: fmt.Sprintf("http://localhost:%d", stsPort+1),
+ },
+ err: "connection refused",
+ },
+ {
+ name: "with correct endpoint and proxy",
+ provider: "aws",
+ stsSpec: &sourcev1.BucketSTSSpec{
+ Provider: "aws",
+ Endpoint: stsEndpoint,
+ },
+ opts: []Option{WithProxyURL(&url.URL{Scheme: "http", Host: proxyAddr})},
+ },
+ {
+ name: "with correct endpoint and incorrect proxy",
+ provider: "aws",
+ stsSpec: &sourcev1.BucketSTSSpec{
+ Provider: "aws",
+ Endpoint: stsEndpoint,
+ },
+ opts: []Option{WithProxyURL(&url.URL{Scheme: "http", Host: fmt.Sprintf("localhost:%d", proxyPort+1)})},
+ err: "connection refused",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ roleCredsRetrieved = false
+ bucket := bucketStub(bucket, testMinioAddress)
+ bucket.Spec.Provider = tt.provider
+ bucket.Spec.STS = tt.stsSpec
+ minioClient, err := NewClient(bucket, append(tt.opts, WithTLSConfig(testTLSConfig))...)
+ assert.NilError(t, err)
+ assert.Assert(t, minioClient != nil)
+ ctx := context.Background()
+ tempDir := t.TempDir()
+ path := filepath.Join(tempDir, sourceignore.IgnoreFile)
+ _, err = minioClient.FGetObject(ctx, bucketName, objectName, path)
+ if tt.err != "" {
+ assert.ErrorContains(t, err, tt.err)
+ } else {
+ assert.NilError(t, err)
+ assert.Assert(t, roleCredsRetrieved)
+ }
+ })
+ }
+}
+
+func TestNewClientAndFGetObjectWithProxy(t *testing.T) {
+ proxyAddr, proxyPort := testproxy.New(t)
+
+ tests := []struct {
+ name string
+ proxyURL *url.URL
+ errSubstring string
+ }{
+ {
+ name: "with correct proxy",
+ proxyURL: &url.URL{Scheme: "http", Host: proxyAddr},
+ },
+ {
+ name: "with incorrect proxy",
+ proxyURL: &url.URL{Scheme: "http", Host: fmt.Sprintf("localhost:%d", proxyPort+1)},
+ errSubstring: "connection refused",
+ },
}
- go proxyServer.Serve(proxyListener)
- defer proxyServer.Shutdown(context.Background())
- proxyURL := &url.URL{Scheme: "http", Host: proxyAddr}
// run test
- minioClient, err := NewClient(bucketStub(bucket, testMinioAddress),
- WithSecret(secret.DeepCopy()),
- WithTLSConfig(testTLSConfig),
- WithProxyURL(proxyURL))
- assert.NilError(t, err)
- assert.Assert(t, minioClient != nil)
- ctx := context.Background()
- tempDir := t.TempDir()
- path := filepath.Join(tempDir, sourceignore.IgnoreFile)
- _, err = minioClient.FGetObject(ctx, bucketName, objectName, path)
- assert.NilError(t, err)
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ minioClient, err := NewClient(bucketStub(bucket, testMinioAddress),
+ WithSecret(secret.DeepCopy()),
+ WithTLSConfig(testTLSConfig),
+ WithProxyURL(tt.proxyURL))
+ assert.NilError(t, err)
+ assert.Assert(t, minioClient != nil)
+ ctx := context.Background()
+ tempDir := t.TempDir()
+ path := filepath.Join(tempDir, sourceignore.IgnoreFile)
+ _, err = minioClient.FGetObject(ctx, bucketName, objectName, path)
+ if tt.errSubstring != "" {
+ assert.ErrorContains(t, err, tt.errSubstring)
+ } else {
+ assert.NilError(t, err)
+ }
+ })
+ }
}
func TestFGetObjectNotExists(t *testing.T) {
@@ -349,6 +470,81 @@ func TestValidateSecret(t *testing.T) {
}
}
+func TestValidateSTSConfig(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ bucket *sourcev1.Bucket
+ err string
+ }{
+ {
+ name: "empty STS config",
+ bucket: &sourcev1.Bucket{},
+ },
+ {
+ name: "aws STS provider",
+ bucket: &sourcev1.Bucket{
+ Spec: sourcev1.BucketSpec{
+ Provider: "aws",
+ STS: &sourcev1.BucketSTSSpec{
+ Provider: "aws",
+ Endpoint: "https://something",
+ },
+ },
+ },
+ },
+ {
+ name: "aws bucket provider and unsupported STS provider",
+ bucket: &sourcev1.Bucket{
+ Spec: sourcev1.BucketSpec{
+ Provider: "aws",
+ STS: &sourcev1.BucketSTSSpec{
+ Provider: "ldap",
+ Endpoint: "https://something",
+ },
+ },
+ },
+ err: "STS provider 'ldap' is not supported for 'aws' bucket provider",
+ },
+ {
+ name: "unsupported bucket provider",
+ bucket: &sourcev1.Bucket{
+ Spec: sourcev1.BucketSpec{
+ Provider: "gcp",
+ STS: &sourcev1.BucketSTSSpec{},
+ },
+ },
+ err: "STS configuration is not supported for 'gcp' bucket provider",
+ },
+ {
+ name: "invalid STS endpoint",
+ bucket: &sourcev1.Bucket{
+ Spec: sourcev1.BucketSpec{
+ Provider: "aws",
+ STS: &sourcev1.BucketSTSSpec{
+ Provider: "aws",
+ Endpoint: "something",
+ },
+ },
+ },
+ err: `STS endpoint 'something' does not match pattern "^(http|https)://.*$"`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ err := ValidateSTSConfig(tt.bucket)
+ if tt.err != "" {
+ assert.Error(t, err, tt.err)
+ } else {
+ assert.NilError(t, err)
+ }
+ })
+ }
+}
+
func bucketStub(bucket sourcev1.Bucket, endpoint string) *sourcev1.Bucket {
b := bucket.DeepCopy()
b.Spec.Endpoint = endpoint
diff --git a/tests/listener/listener.go b/tests/listener/listener.go
new file mode 100644
index 000000000..f034b61fb
--- /dev/null
+++ b/tests/listener/listener.go
@@ -0,0 +1,46 @@
+/*
+Copyright 2024 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testlistener
+
+import (
+ "net"
+ "strconv"
+ "strings"
+ "testing"
+
+ "gotest.tools/assert"
+)
+
+// New creates a TCP listener on a random port and returns
+// the listener, the address and the port of this listener.
+// It also registers a cleanup function to close the listener
+// when the test ends.
+func New(t *testing.T) (net.Listener, string, int) {
+ t.Helper()
+
+ lis, err := net.Listen("tcp", ":0")
+ assert.NilError(t, err)
+ t.Cleanup(func() { lis.Close() })
+
+ addr := lis.Addr().String()
+ addrParts := strings.Split(addr, ":")
+ portStr := addrParts[len(addrParts)-1]
+ port, err := strconv.Atoi(portStr)
+ assert.NilError(t, err)
+
+ return lis, addr, port
+}
diff --git a/tests/proxy/proxy.go b/tests/proxy/proxy.go
new file mode 100644
index 000000000..33fadece4
--- /dev/null
+++ b/tests/proxy/proxy.go
@@ -0,0 +1,48 @@
+/*
+Copyright 2024 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testproxy
+
+import (
+ "net/http"
+ "testing"
+
+ "github.com/elazarl/goproxy"
+
+ testlistener "github.com/fluxcd/source-controller/tests/listener"
+)
+
+// New creates a new goproxy server on a random port and returns
+// the address and the port of this server. It also registers a
+// cleanup functions to close the server and the listener when
+// the test ends.
+func New(t *testing.T) (string, int) {
+ t.Helper()
+
+ lis, addr, port := testlistener.New(t)
+
+ handler := goproxy.NewProxyHttpServer()
+ handler.Verbose = true
+
+ server := &http.Server{
+ Addr: addr,
+ Handler: handler,
+ }
+ go server.Serve(lis)
+ t.Cleanup(func() { server.Close() })
+
+ return addr, port
+}
|