From b3bc3b1ca988df1203b34ae31a99471aeaafc711 Mon Sep 17 00:00:00 2001 From: Richard chen <99175581+RichardChen820@users.noreply.github.com> Date: Mon, 4 Mar 2024 10:29:10 +0800 Subject: [PATCH] Support generating cert based TLS type secret (#10) * Support generating cert based TLS type secret * Resolve comments * Resolve comments * Rename to SecretReferenceResolver * Add more test cases --- api/v1/azureappconfigurationprovider_types.go | 6 +- api/v1/zz_generated.deepcopy.go | 54 +- ...fig.io_azureappconfigurationproviders.yaml | 4 +- go.mod | 3 +- go.sum | 8 + .../appconfigurationprovider_controller.go | 276 +++++---- ...ppconfigurationprovider_controller_test.go | 368 ++++++++++- internal/controller/processor.go | 226 ++++--- internal/controller/utils.go | 2 +- .../configuraiton_setting_loader_test.go | 574 +++++++++++++++++- .../loader/configuration_setting_loader.go | 346 ++++++++--- .../loader/keyvault_reference_resolver.go | 18 +- .../keyvault_reference_resolver_test.go | 6 +- .../mock_configuration_settings_retriever.go | 19 +- internal/loader/typed_setting.go | 2 +- 15 files changed, 1557 insertions(+), 355 deletions(-) diff --git a/api/v1/azureappconfigurationprovider_types.go b/api/v1/azureappconfigurationprovider_types.go index 52fae79..523c5d9 100644 --- a/api/v1/azureappconfigurationprovider_types.go +++ b/api/v1/azureappconfigurationprovider_types.go @@ -35,7 +35,7 @@ type AzureAppConfigurationProviderSpec struct { Target ConfigurationGenerationParameters `json:"target"` Auth *AzureAppConfigurationProviderAuth `json:"auth,omitempty"` Configuration AzureAppConfigurationKeyValueOptions `json:"configuration,omitempty"` - Secret *AzureKeyVaultReference `json:"secret,omitempty"` + Secret *SecretReference `json:"secret,omitempty"` FeatureFlag *AzureAppConfigurationFeatureFlagOptions `json:"featureFlag,omitempty"` } @@ -141,8 +141,8 @@ type ManagedIdentityReferenceParameters struct { Key string `json:"key"` } -// AzureKeyVaultReference defines the authentication type used to Azure KeyVault resolve KeyVaultReference -type AzureKeyVaultReference struct { +// SecretReference defines the settings for resolving secret reference type items +type SecretReference struct { Target SecretGenerationParameters `json:"target"` Auth *AzureKeyVaultAuth `json:"auth,omitempty"` Refresh *RefreshSettings `json:"refresh,omitempty"` diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 16f7f52..1169a19 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -179,7 +179,7 @@ func (in *AzureAppConfigurationProviderSpec) DeepCopyInto(out *AzureAppConfigura in.Configuration.DeepCopyInto(&out.Configuration) if in.Secret != nil { in, out := &in.Secret, &out.Secret - *out = new(AzureKeyVaultReference) + *out = new(SecretReference) (*in).DeepCopyInto(*out) } if in.FeatureFlag != nil { @@ -264,32 +264,6 @@ func (in *AzureKeyVaultPerVaultAuth) DeepCopy() *AzureKeyVaultPerVaultAuth { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AzureKeyVaultReference) DeepCopyInto(out *AzureKeyVaultReference) { - *out = *in - out.Target = in.Target - if in.Auth != nil { - in, out := &in.Auth, &out.Auth - *out = new(AzureKeyVaultAuth) - (*in).DeepCopyInto(*out) - } - if in.Refresh != nil { - in, out := &in.Refresh, &out.Refresh - *out = new(RefreshSettings) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureKeyVaultReference. -func (in *AzureKeyVaultReference) DeepCopy() *AzureKeyVaultReference { - if in == nil { - return nil - } - out := new(AzureKeyVaultReference) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConfigMapDataOptions) DeepCopyInto(out *ConfigMapDataOptions) { *out = *in @@ -448,6 +422,32 @@ func (in *SecretGenerationParameters) DeepCopy() *SecretGenerationParameters { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretReference) DeepCopyInto(out *SecretReference) { + *out = *in + out.Target = in.Target + if in.Auth != nil { + in, out := &in.Auth, &out.Auth + *out = new(AzureKeyVaultAuth) + (*in).DeepCopyInto(*out) + } + if in.Refresh != nil { + in, out := &in.Refresh, &out.Refresh + *out = new(RefreshSettings) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretReference. +func (in *SecretReference) DeepCopy() *SecretReference { + if in == nil { + return nil + } + out := new(SecretReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Selector) DeepCopyInto(out *Selector) { *out = *in diff --git a/config/crd/bases/azconfig.io_azureappconfigurationproviders.yaml b/config/crd/bases/azconfig.io_azureappconfigurationproviders.yaml index 8903aa9..a3e2b54 100644 --- a/config/crd/bases/azconfig.io_azureappconfigurationproviders.yaml +++ b/config/crd/bases/azconfig.io_azureappconfigurationproviders.yaml @@ -163,8 +163,8 @@ spec: type: array type: object secret: - description: AzureKeyVaultReference defines the authentication type - used to Azure KeyVault resolve KeyVaultReference + description: SecretReference defines the settings for resolving secret + reference type items properties: auth: properties: diff --git a/go.mod b/go.mod index d37f699..66ec1ad 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0 github.com/golang/mock v1.6.0 github.com/onsi/gomega v1.27.8 + golang.org/x/crypto v0.17.0 golang.org/x/sync v0.3.0 k8s.io/apimachinery v0.26.2 k8s.io/client-go v0.26.2 @@ -67,7 +68,6 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect @@ -90,4 +90,5 @@ require ( sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect + software.sslmate.com/src/go-pkcs12 v0.4.0 ) diff --git a/go.sum b/go.sum index 2caf210..f428c02 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,7 @@ github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= @@ -72,6 +73,7 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -122,8 +124,10 @@ github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -147,6 +151,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= @@ -363,3 +369,5 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kF sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= +software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= diff --git a/internal/controller/appconfigurationprovider_controller.go b/internal/controller/appconfigurationprovider_controller.go index 49d9b1e..42083eb 100644 --- a/internal/controller/appconfigurationprovider_controller.go +++ b/internal/controller/appconfigurationprovider_controller.go @@ -50,25 +50,24 @@ type AzureAppConfigurationProviderReconciler struct { } type ReconciliationState struct { - Generation int64 - ConfigMapResourceVersion *string - SecretResourceVersion *string - SentinelETags map[acpv1.Sentinel]*azcore.ETag - NextSentinelBasedRefreshReconcileTime metav1.Time - NextKeyVaultReferenceRefreshReconcileTime metav1.Time - NextFeatureFlagRefreshReconcileTime metav1.Time - CachedSecretReferences map[string]loader.KeyVaultSecretUriSegment + Generation int64 + ConfigMapResourceVersion *string + SentinelETags map[acpv1.Sentinel]*azcore.ETag + ExistingSecretReferences map[string]*loader.TargetSecretReference + NextSentinelBasedRefreshReconcileTime metav1.Time + NextSecretReferenceRefreshReconcileTime metav1.Time + NextFeatureFlagRefreshReconcileTime metav1.Time } const ( - ProviderName string = "AzureAppConfigurationProvider" - LastReconcileTimeAnnotation string = "azconfig.io/LastReconcileTime" - KeyVaultReferenceContentType string = "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8" - FeatureFlagContentType string = "application/vnd.microsoft.appconfig.ff+json;charset=utf-8" - HeaderRetryAfter string = "Retry-After" - RequeueReconcileAfter time.Duration = time.Second * 30 - RetryAttempt int = 3 - DefaultRefreshInterval time.Duration = time.Second * 30 + ProviderName string = "AzureAppConfigurationProvider" + LastReconcileTimeAnnotation string = "azconfig.io/LastReconcileTime" + SecretReferenceContentType string = "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8" + FeatureFlagContentType string = "application/vnd.microsoft.appconfig.ff+json;charset=utf-8" + HeaderRetryAfter string = "Retry-After" + RequeueReconcileAfter time.Duration = time.Second * 30 + RetryAttempt int = 3 + DefaultRefreshInterval time.Duration = time.Second * 30 ) //Markers for teaching kubebuiler how generate the rabc manifests, see https://book.kubebuilder.io/reference/markers/rbac.html for detail @@ -129,37 +128,74 @@ func (reconciler *AzureAppConfigurationProviderReconciler) Reconcile(ctx context reconciler.logAndSetFailStatus(ctx, err, provider) return reconcile.Result{Requeue: false}, nil } - var existingConfigMap = corev1.ConfigMap{} - err = reconciler.verifyTargetObjectExistence(ctx, provider, &existingConfigMap) + + existingConfigMap := corev1.ConfigMap{} + isExisting := false + _, err = reconciler.verifyTargetObjectExistence(ctx, provider, &existingConfigMap) if err != nil { reconciler.logAndSetFailStatus(ctx, err, provider) return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, nil } - var existingSecret = corev1.Secret{} + + existingSecrets := make(map[string]corev1.Secret) + var existingSecret corev1.Secret if provider.Spec.Secret != nil { - err = reconciler.verifyTargetObjectExistence(ctx, provider, &existingSecret) + existingSecret = corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: provider.Spec.Secret.Target.SecretName, + }, + } + isExisting, err = reconciler.verifyTargetObjectExistence(ctx, provider, &existingSecret) if err != nil { reconciler.logAndSetFailStatus(ctx, err, provider) return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, nil } + if isExisting { + existingSecrets[provider.Spec.Secret.Target.SecretName] = existingSecret + } } - /* Initialize the ReconcileState for the provider*/ - if reconciler.ProvidersReconcileState[req.NamespacedName] == nil { + if reconciler.ProvidersReconcileState[req.NamespacedName] != nil { + for name := range reconciler.ProvidersReconcileState[req.NamespacedName].ExistingSecretReferences { + if _, ok := existingSecrets[name]; !ok { + existingSecret = corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + isExisting, err = reconciler.verifyTargetObjectExistence(ctx, provider, &existingSecret) + if err != nil { + reconciler.logAndSetFailStatus(ctx, err, provider) + return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, nil + } + if isExisting { + existingSecrets[name] = existingSecret + } + } + } + } else { + // Initialize the ReconcileState for the provider reconciler.ProvidersReconcileState[req.NamespacedName] = &ReconciliationState{ Generation: -1, ConfigMapResourceVersion: nil, - SecretResourceVersion: nil, SentinelETags: make(map[acpv1.Sentinel]*azcore.ETag), - CachedSecretReferences: make(map[string]loader.KeyVaultSecretUriSegment), + ExistingSecretReferences: make(map[string]*loader.TargetSecretReference), } } + // Reset the resource version if the configmap or secret was unexpected deleted if existingConfigMap.Name == "" { reconciler.ProvidersReconcileState[req.NamespacedName].ConfigMapResourceVersion = nil } - if provider.Spec.Secret == nil || existingSecret.Name == "" { - reconciler.ProvidersReconcileState[req.NamespacedName].SecretResourceVersion = nil + + if provider.Spec.Secret == nil { + reconciler.ProvidersReconcileState[req.NamespacedName].ExistingSecretReferences = make(map[string]*loader.TargetSecretReference) + } else { + for name := range reconciler.ProvidersReconcileState[req.NamespacedName].ExistingSecretReferences { + if _, ok := existingSecrets[name]; !ok { + reconciler.ProvidersReconcileState[req.NamespacedName].ExistingSecretReferences[name].SecretResourceVersion = "" + } + } } /* Create ConfigurationSettingLoader to get the key-value settings from Azure AppConfiguration. */ @@ -176,24 +212,19 @@ func (reconciler *AzureAppConfigurationProviderReconciler) Reconcile(ctx context retriever = reconciler.Retriever } - /* Check if the reconciler should reconcile unconditionally under some situations, like the provider is updated, the configmap is deleted and so on. */ - shouldReconcile := reconciler.shouldReconcile(provider, &existingConfigMap, &existingSecret) - // Initialize the processor setting in this reconcile processor := &AppConfigurationProviderProcessor{ - Context: ctx, - Provider: provider, - Retriever: &retriever, - CurrentTime: metav1.Now(), - ReconciliationState: reconciler.ProvidersReconcileState, - NamespacedName: req.NamespacedName, - ShouldReconcile: shouldReconcile, - Settings: &loader.TargetKeyValueSettings{}, - RefreshOptions: NewRefreshOptions(), - ResolveSecretReference: nil, - } - - if err := processor.PopulateSettings(&existingConfigMap, &existingSecret); err != nil { + Context: ctx, + Provider: provider, + Retriever: &retriever, + CurrentTime: metav1.Now(), + ReconciliationState: reconciler.ProvidersReconcileState[req.NamespacedName], + Settings: &loader.TargetKeyValueSettings{}, + RefreshOptions: NewRefreshOptions(), + SecretReferenceResolver: nil, + } + + if err := processor.PopulateSettings(&existingConfigMap, existingSecrets); err != nil { return reconciler.requeueWhenGetSettingsFailed(ctx, provider, err) } @@ -204,9 +235,15 @@ func (reconciler *AzureAppConfigurationProviderReconciler) Reconcile(ctx context return result, nil } } + /* Create secret when there are secret settings */ if processor.RefreshOptions.SecretSettingPopulated { - result, err := reconciler.createOrUpdateSecret(ctx, provider, processor.Settings) + result, err := reconciler.createOrUpdateSecrets(ctx, provider, processor.Settings) + if err != nil { + return result, nil + } + + result, err = reconciler.expelRemovedSecrets(ctx, provider, existingSecrets, processor.ReconciliationState.ExistingSecretReferences) if err != nil { return result, nil } @@ -217,29 +254,35 @@ func (reconciler *AzureAppConfigurationProviderReconciler) Reconcile(ctx context return processor.Finish() } -func (reconciler *AzureAppConfigurationProviderReconciler) verifyTargetObjectExistence(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, obj client.Object) error { +func (reconciler *AzureAppConfigurationProviderReconciler) verifyTargetObjectExistence( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + obj client.Object) (bool, error) { // Get and verify the existing configMap or secret, if there's existing configMap/secret which is not owned by current provider, throw error var targetName string if _, ok := obj.(*corev1.ConfigMap); ok { targetName = provider.Spec.Target.ConfigMapName } else if _, ok := obj.(*corev1.Secret); ok { - targetName = provider.Spec.Secret.Target.SecretName + targetName = obj.GetName() } else { // Only verify ConfigMap and Secret object - return nil + return false, nil } err := reconciler.Client.Get(ctx, types.NamespacedName{Namespace: provider.Namespace, Name: targetName}, obj) if err != nil { if apierrors.IsNotFound(err) { - return nil + return false, nil } - return err + return false, err } - return verifyExistingTargetObject(obj, targetName, provider.Name) + return true, verifyExistingTargetObject(obj, targetName, provider.Name) } -func (reconciler *AzureAppConfigurationProviderReconciler) logAndSetFailStatus(ctx context.Context, err error, provider *acpv1.AzureAppConfigurationProvider) { +func (reconciler *AzureAppConfigurationProviderReconciler) logAndSetFailStatus( + ctx context.Context, + err error, + provider *acpv1.AzureAppConfigurationProvider) { var showErrorAsWarning bool = false namespacedName := types.NamespacedName{ Name: provider.Name, @@ -252,7 +295,7 @@ func (reconciler *AzureAppConfigurationProviderReconciler) logAndSetFailStatus(c } else if reconcileState != nil && reconcileState.ConfigMapResourceVersion != nil && (provider.Spec.Secret == nil || - reconcileState.SecretResourceVersion != nil) { + len(reconcileState.ExistingSecretReferences) == 0) { // If the target ConfigMap or Secret does exists, just show error as warning. showErrorAsWarning = true } @@ -266,7 +309,10 @@ func (reconciler *AzureAppConfigurationProviderReconciler) logAndSetFailStatus(c } } -func (reconciler *AzureAppConfigurationProviderReconciler) requeueWhenGetSettingsFailed(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, err error) (ctrl.Result, error) { +func (reconciler *AzureAppConfigurationProviderReconciler) requeueWhenGetSettingsFailed( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + err error) (ctrl.Result, error) { requeueAfter := RequeueReconcileAfter reconciler.logAndSetFailStatus(ctx, err, provider) if errors.Is(err, &loader.ArgumentError{}) { @@ -285,7 +331,10 @@ func (reconciler *AzureAppConfigurationProviderReconciler) requeueWhenGetSetting return reconcile.Result{Requeue: true, RequeueAfter: requeueAfter}, nil } -func (reconciler *AzureAppConfigurationProviderReconciler) createOrUpdateConfigMap(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, settings *loader.TargetKeyValueSettings) (reconcile.Result, error) { +func (reconciler *AzureAppConfigurationProviderReconciler) createOrUpdateConfigMap( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + settings *loader.TargetKeyValueSettings) (reconcile.Result, error) { configMapObj := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: provider.Spec.Target.ConfigMapName, @@ -327,49 +376,85 @@ func (reconciler *AzureAppConfigurationProviderReconciler) createOrUpdateConfigM return reconcile.Result{}, nil } -func (reconciler *AzureAppConfigurationProviderReconciler) createOrUpdateSecret(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, settings *loader.TargetKeyValueSettings) (reconcile.Result, error) { - secretObjName := provider.Spec.Secret.Target.SecretName - secretObj := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretObjName, - Namespace: provider.Namespace, - }, - Type: corev1.SecretTypeOpaque, - } - // Important: set the ownership of secret - if err := controllerutil.SetControllerReference(provider, secretObj, reconciler.Scheme); err != nil { - reconciler.logAndSetFailStatus(ctx, err, provider) - return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, err - } - if provider.Annotations == nil { - provider.Annotations = make(map[string]string) - } - provider.Annotations[LastReconcileTimeAnnotation] = metav1.Now().UTC().String() +func (reconciler *AzureAppConfigurationProviderReconciler) createOrUpdateSecrets( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + settings *loader.TargetKeyValueSettings) (reconcile.Result, error) { if len(settings.SecretSettings) == 0 { klog.V(3).Info("No secret settings are fetched from Azure AppConfiguration") } - operationResult, err := ctrl.CreateOrUpdate(ctx, reconciler.Client, secretObj, func() error { - secretObj.Data = settings.SecretSettings - secretObj.Labels = provider.Labels - secretObj.Annotations = provider.Annotations - return nil - }) - if err != nil { - reconciler.logAndSetFailStatus(ctx, err, provider) - return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, err + if provider.Annotations == nil { + provider.Annotations = make(map[string]string) } + namespacedName := types.NamespacedName{ Name: provider.Name, Namespace: provider.Namespace, } - reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersion = &secretObj.ResourceVersion - klog.V(5).Infof("Secret %q in %q namespace is %s", secretObj.Name, secretObj.Namespace, string(operationResult)) + + for secretName, secret := range settings.SecretSettings { + secretObj := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: provider.Namespace, + }, + Type: secret.Type, + } + // Important: set the ownership of secret + if err := controllerutil.SetControllerReference(provider, secretObj, reconciler.Scheme); err != nil { + reconciler.logAndSetFailStatus(ctx, err, provider) + return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, err + } + + provider.Annotations[LastReconcileTimeAnnotation] = metav1.Now().UTC().String() + operationResult, err := ctrl.CreateOrUpdate(ctx, reconciler.Client, secretObj, func() error { + secretObj.Data = secret.Data + secretObj.Labels = provider.Labels + secretObj.Annotations = provider.Annotations + + return nil + }) + if err != nil { + reconciler.logAndSetFailStatus(ctx, err, provider) + return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, err + } + + reconciler.ProvidersReconcileState[namespacedName].ExistingSecretReferences[secretObj.Name].SecretResourceVersion = secretObj.ResourceVersion + klog.V(5).Infof("Secret %q in %q namespace is %s", secretObj.Name, secretObj.Namespace, string(operationResult)) + } return reconcile.Result{}, nil } -func newProviderStatus(phase acpv1.AppConfigurationSyncPhase, message string, syncTime metav1.Time, refreshStatus acpv1.RefreshStatus) acpv1.AzureAppConfigurationProviderStatus { +func (reconciler *AzureAppConfigurationProviderReconciler) expelRemovedSecrets( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + existingSecrets map[string]corev1.Secret, + secretReferences map[string]*loader.TargetSecretReference) (reconcile.Result, error) { + for name := range existingSecrets { + if _, ok := secretReferences[name]; !ok { + err := reconciler.Client.Delete(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: provider.Namespace, + }, + }) + if err != nil { + reconciler.logAndSetFailStatus(ctx, err, provider) + return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, err + } + } + } + + return reconcile.Result{}, nil +} + +func newProviderStatus( + phase acpv1.AppConfigurationSyncPhase, + message string, + syncTime metav1.Time, + refreshStatus acpv1.RefreshStatus) acpv1.AzureAppConfigurationProviderStatus { return acpv1.AzureAppConfigurationProviderStatus{ Message: message, Phase: phase, @@ -379,31 +464,6 @@ func newProviderStatus(phase acpv1.AppConfigurationSyncPhase, message string, sy } } -func (reconciler *AzureAppConfigurationProviderReconciler) shouldReconcile(provider *acpv1.AzureAppConfigurationProvider, existingConfigMap *corev1.ConfigMap, existingSecret *corev1.Secret) bool { - // Get the name and namespace of the provider - namespacedName := types.NamespacedName{ - Name: provider.Name, - Namespace: provider.Namespace, - } - if provider.Generation != reconciler.ProvidersReconcileState[namespacedName].Generation { - // If the provider is updated, we need to reconcile anyway - return true - } - if reconciler.ProvidersReconcileState[namespacedName].ConfigMapResourceVersion == nil || - *reconciler.ProvidersReconcileState[namespacedName].ConfigMapResourceVersion != existingConfigMap.ResourceVersion { - // If the ConfigMap is removed or updated, we need to reconcile anyway - return true - } - if provider.Spec.Secret != nil && - (reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersion == nil || - *reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersion != existingSecret.ResourceVersion) { - // If the Secret is removed or updated, we need to reconcile anyway - return true - } - - return false -} - // SetupWithManager sets up the controller with the Manager. func (r *AzureAppConfigurationProviderReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/internal/controller/appconfigurationprovider_controller_test.go b/internal/controller/appconfigurationprovider_controller_test.go index 39c2948..d839e0d 100644 --- a/internal/controller/appconfigurationprovider_controller_test.go +++ b/internal/controller/appconfigurationprovider_controller_test.go @@ -146,13 +146,38 @@ var _ = Describe("AppConfiguationProvider controller", func() { It("Should create new secret", func() { By("By getting multiple secret reference settings from AppConfig") - mapResult := make(map[string][]byte) - mapResult["testSecretKey"] = []byte("testValue") - mapResult["testSecretKey2"] = []byte("testValue2") - mapResult["testSecretKey3"] = []byte("testValue3") + secretResult1 := make(map[string][]byte) + secretResult1["tls.crt"] = []byte("fakeCrt") + secretResult1["tls.key"] = []byte("fakeKey") + secretResult2 := make(map[string][]byte) + secretResult2["testSecretKey"] = []byte("testSecretValue") + secretResult2["testSecretKey2"] = []byte("testSecretValue2") + secretResult2["testSecretKey3"] = []byte("testSecretValue3") + + secretName := "secret-to-be-created-3" + secretName2 := "secret-to-be-created-3-1" allSettings := &loader.TargetKeyValueSettings{ - SecretSettings: mapResult, + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult1, + Type: corev1.SecretTypeTLS, + }, + secretName2: { + Data: secretResult2, + Type: corev1.SecretTypeOpaque, + }, + }, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: make(map[string]loader.KeyVaultSecretUriSegment), + }, + secretName2: { + Type: corev1.SecretTypeOpaque, + UriSegments: make(map[string]loader.KeyVaultSecretUriSegment), + }, + }, } mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) @@ -160,7 +185,6 @@ var _ = Describe("AppConfiguationProvider controller", func() { ctx := context.Background() providerName := "test-appconfigurationprovider-3" configMapName := "configmap-to-be-created-3" - secretName := "secret-to-be-created-3" configProvider := &acpv1.AzureAppConfigurationProvider{ TypeMeta: metav1.TypeMeta{ APIVersion: "appconfig.kubernetes.config/v1", @@ -175,7 +199,7 @@ var _ = Describe("AppConfiguationProvider controller", func() { Target: acpv1.ConfigurationGenerationParameters{ ConfigMapName: configMapName, }, - Secret: &acpv1.AzureKeyVaultReference{ + Secret: &acpv1.SecretReference{ Target: acpv1.SecretGenerationParameters{ SecretName: secretName, }, @@ -184,7 +208,9 @@ var _ = Describe("AppConfiguationProvider controller", func() { } Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) secretLookupKey := types.NamespacedName{Name: secretName, Namespace: ProviderNamespace} + secretLookupKey2 := types.NamespacedName{Name: secretName2, Namespace: ProviderNamespace} secret := &corev1.Secret{} + secret2 := &corev1.Secret{} Eventually(func() bool { err := k8sClient.Get(ctx, secretLookupKey, secret) @@ -192,13 +218,26 @@ var _ = Describe("AppConfiguationProvider controller", func() { fmt.Print(err.Error()) } return err == nil - }, time.Second*20, interval).Should(BeTrue()) + }, time.Second*5, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey2, secret2) + if err != nil { + fmt.Print(err.Error()) + } + return err == nil + }, time.Second*5, interval).Should(BeTrue()) Expect(secret.Namespace).Should(Equal(ProviderNamespace)) - Expect(string(secret.Data["testSecretKey"])).Should(Equal("testValue")) - Expect(string(secret.Data["testSecretKey2"])).Should(Equal("testValue2")) - Expect(string(secret.Data["testSecretKey3"])).Should(Equal("testValue3")) - Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + Expect(string(secret.Data["tls.crt"])).Should(Equal("fakeCrt")) + Expect(string(secret.Data["tls.key"])).Should(Equal("fakeKey")) + Expect(secret.Type).Should(Equal(corev1.SecretTypeTLS)) + + Expect(secret2.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret2.Data["testSecretKey"])).Should(Equal("testSecretValue")) + Expect(string(secret2.Data["testSecretKey2"])).Should(Equal("testSecretValue2")) + Expect(string(secret2.Data["testSecretKey3"])).Should(Equal("testSecretValue3")) + Expect(secret2.Type).Should(Equal(corev1.SecretTypeOpaque)) }) It("Should create proper configmap and secret", func() { @@ -213,9 +252,21 @@ var _ = Describe("AppConfiguationProvider controller", func() { secretResult["testSecretKey2"] = []byte("testSecretValue2") secretResult["testSecretKey3"] = []byte("testSecretValue3") + secretName := "secret-to-be-created-5" allSettings := &loader.TargetKeyValueSettings{ - SecretSettings: secretResult, + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, ConfigMapSettings: configMapResult, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + UriSegments: make(map[string]loader.KeyVaultSecretUriSegment), + }, + }, } mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) @@ -223,7 +274,7 @@ var _ = Describe("AppConfiguationProvider controller", func() { ctx := context.Background() providerName := "test-appconfigurationprovider-5" configMapName := "configmap-to-be-created-5" - secretName := "secret-to-be-created-5" + configProvider := &acpv1.AzureAppConfigurationProvider{ TypeMeta: metav1.TypeMeta{ APIVersion: "appconfig.kubernetes.config/v1", @@ -238,7 +289,7 @@ var _ = Describe("AppConfiguationProvider controller", func() { Target: acpv1.ConfigurationGenerationParameters{ ConfigMapName: configMapName, }, - Secret: &acpv1.AzureKeyVaultReference{ + Secret: &acpv1.SecretReference{ Target: acpv1.SecretGenerationParameters{ SecretName: secretName, }, @@ -387,6 +438,293 @@ var _ = Describe("AppConfiguationProvider controller", func() { Expect(configmap.Data["filestyle.json"]).Should(Equal("{\"testKey\":\"testValue\",\"feature_management\":{\"feature_flags\":[{\"id\": \"testFeatureFlag\",\"enabled\": true,\"conditions\": {\"client_filters\": []}}]}}")) Expect(len(configmap.Data)).Should(Equal(1)) }) + + It("Should refresh configMap", func() { + By("By updating the provider and trigger reconciliation") + mapResult := make(map[string]string) + mapResult["testKey"] = "testValue" + mapResult["testKey2"] = "testValue2" + mapResult["testKey3"] = "testValue3" + + allSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + ctx := context.Background() + providerName := "refresh-appconfigurationprovider-1" + configMapName := "configmap-to-be-refresh-1" + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Labels: map[string]string{"foo": "fooValue", "bar": "barValue"}, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Labels["foo"]).Should(Equal("fooValue")) + Expect(configmap.Labels["bar"]).Should(Equal("barValue")) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + + newEndpoint := "https://fake-endpoint-2" + + mapResult2 := make(map[string]string) + mapResult2["testKey"] = "newtestValue" + mapResult2["testKey2"] = "newtestValue2" + mapResult2["testKey3"] = "newtestValue3" + + allSettings2 := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult2, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings2, nil) + + k8sClient.Get(ctx, types.NamespacedName{Name: providerName, Namespace: ProviderNamespace}, configProvider) + configProvider.Spec.Endpoint = &newEndpoint + + Expect(k8sClient.Update(ctx, configProvider)).Should(Succeed()) + + time.Sleep(5 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Data["testKey"]).Should(Equal("newtestValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("newtestValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("newtestValue3")) + + }) + + It("Should refresh configMap", func() { + By("By sentinel value updated in Azure App Configuration") + mapResult := make(map[string]string) + mapResult["testKey"] = "testValue" + mapResult["testKey2"] = "testValue2" + mapResult["testKey3"] = "testValue3" + + allSettings := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult, + } + + mapResult2 := make(map[string]string) + mapResult2["testKey"] = "newtestValue" + mapResult2["testKey2"] = "newtestValue2" + mapResult2["testKey3"] = "newtestValue3" + + allSettings2 := &loader.TargetKeyValueSettings{ + ConfigMapSettings: mapResult2, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + mockConfigurationSettings.EXPECT().CheckAndRefreshSentinels(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil, nil) + mockConfigurationSettings.EXPECT().RefreshKeyValueSettings(gomock.Any(), gomock.Any(), gomock.Any()).Return(allSettings2, nil) + + ctx := context.Background() + providerName := "refresh-appconfigurationprovider-2" + configMapName := "configmap-to-be-refresh-2" + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AzureAppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + Labels: map[string]string{"foo": "fooValue", "bar": "barValue"}, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + Refresh: &acpv1.DynamicConfigurationRefreshParameters{ + Interval: "5s", + Enabled: true, + Monitoring: &acpv1.RefreshMonitoring{ + Sentinels: []acpv1.Sentinel{ + {Key: "testKey", Label: "testLabel"}, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Labels["foo"]).Should(Equal("fooValue")) + Expect(configmap.Labels["bar"]).Should(Equal("barValue")) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + + time.Sleep(6 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Data["testKey"]).Should(Equal("newtestValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("newtestValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("newtestValue3")) + + k8sClient.Delete(ctx, configProvider) + }) + + It("Should refresh secret", func() { + By("By enabling refresh on secret") + configMapResult := make(map[string]string) + configMapResult["testKey"] = "testValue" + configMapResult["testKey2"] = "testValue2" + configMapResult["testKey3"] = "testValue3" + + secretResult := make(map[string][]byte) + secretResult["testSecretKey"] = []byte("testSecretValue") + secretResult["testSecretKey2"] = []byte("testSecretValue2") + secretResult["testSecretKey3"] = []byte("testSecretValue3") + + secretName := "secret-to-be-refreshed-3" + allSettings := &loader.TargetKeyValueSettings{ + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, + ConfigMapSettings: configMapResult, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + UriSegments: make(map[string]loader.KeyVaultSecretUriSegment), + }, + }, + } + + mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) + + ctx := context.Background() + providerName := "refresh-appconfigurationprovider-3" + configMapName := "configmap-to-be-refreshed-3" + + configProvider := &acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "appconfig.kubernetes.config/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: providerName, + Namespace: ProviderNamespace, + }, + Spec: acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: configMapName, + }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: secretName, + }, + Refresh: &acpv1.RefreshSettings{ + Interval: "1m", + Enabled: true, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) + configmapLookupKey := types.NamespacedName{Name: configMapName, Namespace: ProviderNamespace} + configmap := &corev1.ConfigMap{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, configmapLookupKey, configmap) + if err != nil { + fmt.Print(err.Error()) + } + return err == nil + }, timeout, interval).Should(BeTrue()) + + secretLookupKey := types.NamespacedName{Name: secretName, Namespace: ProviderNamespace} + secret := &corev1.Secret{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(configmap.Name).Should(Equal(configMapName)) + Expect(configmap.Namespace).Should(Equal(ProviderNamespace)) + Expect(configmap.Data["testKey"]).Should(Equal("testValue")) + Expect(configmap.Data["testKey2"]).Should(Equal("testValue2")) + Expect(configmap.Data["testKey3"]).Should(Equal("testValue3")) + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret.Data["testSecretKey"])).Should(Equal("testSecretValue")) + Expect(string(secret.Data["testSecretKey2"])).Should(Equal("testSecretValue2")) + Expect(string(secret.Data["testSecretKey3"])).Should(Equal("testSecretValue3")) + Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + + newSecretResult := make(map[string][]byte) + newSecretResult["testSecretKey"] = []byte("newTestSecretValue") + newSecretResult["testSecretKey2"] = []byte("newTestSecretValue2") + newSecretResult["testSecretKey3"] = []byte("newTestSecretValue3") + + newResolvedSecret := map[string]corev1.Secret{ + secretName: { + Data: newSecretResult, + Type: corev1.SecretType("Opaque"), + }, + } + + mockConfigurationSettings.EXPECT().ResolveSecretReferences(gomock.Any(), gomock.Any(), gomock.Any()).Return(newResolvedSecret, nil) + // Refresh interval is 1 minute, wait for 65 seconds to make sure the refresh is triggered + time.Sleep(65 * time.Second) + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey, secret) + return err == nil + }, timeout, interval).Should(BeTrue()) + + Expect(secret.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret.Data["testSecretKey"])).Should(Equal("newTestSecretValue")) + Expect(string(secret.Data["testSecretKey2"])).Should(Equal("newTestSecretValue2")) + Expect(string(secret.Data["testSecretKey3"])).Should(Equal("newTestSecretValue3")) + Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + }) }) Context("Verify exist non escaped value in label", func() { diff --git a/internal/controller/processor.go b/internal/controller/processor.go index f7b3c8b..b5cf537 100644 --- a/internal/controller/processor.go +++ b/internal/controller/processor.go @@ -13,81 +13,80 @@ import ( "golang.org/x/exp/maps" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) type AppConfigurationProviderProcessor struct { - Context context.Context - Retriever *loader.ConfigurationSettingsRetriever - Provider *acpv1.AzureAppConfigurationProvider - Settings *loader.TargetKeyValueSettings - ShouldReconcile bool - ReconciliationState map[types.NamespacedName]*ReconciliationState - CurrentTime metav1.Time - NamespacedName types.NamespacedName - RefreshOptions *RefreshOptions - ResolveSecretReference loader.ResolveSecretReference + Context context.Context + Retriever *loader.ConfigurationSettingsRetriever + Provider *acpv1.AzureAppConfigurationProvider + Settings *loader.TargetKeyValueSettings + ShouldReconcile bool + ReconciliationState *ReconciliationState + CurrentTime metav1.Time + RefreshOptions *RefreshOptions + SecretReferenceResolver loader.SecretReferenceResolver } type RefreshOptions struct { - sentinelBasedRefreshEnabled bool - sentinelChanged bool - keyVaultRefreshEnabled bool - keyVaultRefreshNeeded bool - updatedSentinelETags map[acpv1.Sentinel]*azcore.ETag - featureFlagRefreshEnabled bool - featureFlagRefreshNeeded bool - ConfigMapSettingPopulated bool - SecretSettingPopulated bool + sentinelBasedRefreshEnabled bool + sentinelChanged bool + secretReferenceRefreshEnabled bool + secretReferenceRefreshNeeded bool + featureFlagRefreshEnabled bool + featureFlagRefreshNeeded bool + ConfigMapSettingPopulated bool + SecretSettingPopulated bool + updatedSentinelETags map[acpv1.Sentinel]*azcore.ETag } -func (processor *AppConfigurationProviderProcessor) PopulateSettings(existingConfigMap *corev1.ConfigMap, existingSecret *corev1.Secret) error { - if err := processor.ProcessFullReconciliation(); err != nil { - return err +func (processor *AppConfigurationProviderProcessor) PopulateSettings(existingConfigMap *corev1.ConfigMap, existingSecrets map[string]corev1.Secret) error { + if processor.ShouldReconcile = processor.shouldReconcile(existingConfigMap, existingSecrets); processor.ShouldReconcile { + if err := processor.processFullReconciliation(); err != nil { + return err + } } - if err := processor.ProcessFeatureFlagRefresh(existingConfigMap); err != nil { + if err := processor.processFeatureFlagRefresh(existingConfigMap); err != nil { return err } - if err := processor.ProcessKeyValueRefresh(existingConfigMap); err != nil { + if err := processor.processKeyValueRefresh(existingConfigMap); err != nil { return err } - if err := processor.ProcessKeyVaultReferenceRefresh(existingSecret); err != nil { + if err := processor.processSecretReferenceRefresh(existingSecrets); err != nil { return err } return nil } -func (processor *AppConfigurationProviderProcessor) ProcessFullReconciliation() error { - if processor.ShouldReconcile { - updatedSettings, err := (*processor.Retriever).CreateTargetSettings(processor.Context, processor.ResolveSecretReference) - if err != nil { - return err - } - processor.Settings = updatedSettings - processor.ReconciliationState[processor.NamespacedName].CachedSecretReferences = updatedSettings.KeyVaultReferencesToCache - processor.RefreshOptions.ConfigMapSettingPopulated = true - if processor.Provider.Spec.Secret != nil { - processor.RefreshOptions.SecretSettingPopulated = true - } +func (processor *AppConfigurationProviderProcessor) processFullReconciliation() error { + updatedSettings, err := (*processor.Retriever).CreateTargetSettings(processor.Context, processor.SecretReferenceResolver) + if err != nil { + return err + } + processor.Settings = updatedSettings + processor.ReconciliationState.ExistingSecretReferences = updatedSettings.SecretReferences + processor.RefreshOptions.ConfigMapSettingPopulated = true + if processor.Provider.Spec.Secret != nil { + processor.RefreshOptions.SecretSettingPopulated = true } return nil } -func (processor *AppConfigurationProviderProcessor) ProcessFeatureFlagRefresh(existingConfigMap *corev1.ConfigMap) error { +func (processor *AppConfigurationProviderProcessor) processFeatureFlagRefresh(existingConfigMap *corev1.ConfigMap) error { provider := *processor.Provider - reconcileState := processor.ReconciliationState[processor.NamespacedName] - currentTime := processor.CurrentTime + reconcileState := processor.ReconciliationState var err error // Check if the feature flag dynamic feature if enabled - if provider.Spec.FeatureFlag != nil && provider.Spec.FeatureFlag.Refresh != nil && provider.Spec.FeatureFlag.Refresh.Enabled { + if provider.Spec.FeatureFlag != nil && + provider.Spec.FeatureFlag.Refresh != nil && + provider.Spec.FeatureFlag.Refresh.Enabled { processor.RefreshOptions.featureFlagRefreshEnabled = true } else { reconcileState.NextFeatureFlagRefreshReconcileTime = metav1.Time{} @@ -95,13 +94,13 @@ func (processor *AppConfigurationProviderProcessor) ProcessFeatureFlagRefresh(ex } refreshInterval, _ := time.ParseDuration(provider.Spec.FeatureFlag.Refresh.Interval) - nextFeatureFlagRefreshReconcileTime := metav1.Time{Time: currentTime.Add(refreshInterval)} + nextFeatureFlagRefreshReconcileTime := metav1.Time{Time: processor.CurrentTime.Add(refreshInterval)} if processor.ShouldReconcile { reconcileState.NextFeatureFlagRefreshReconcileTime = nextFeatureFlagRefreshReconcileTime return nil } - if !currentTime.After(reconcileState.NextFeatureFlagRefreshReconcileTime.Time) { + if !processor.CurrentTime.After(reconcileState.NextFeatureFlagRefreshReconcileTime.Time) { return nil } @@ -118,13 +117,13 @@ func (processor *AppConfigurationProviderProcessor) ProcessFeatureFlagRefresh(ex return nil } -func (processor *AppConfigurationProviderProcessor) ProcessKeyValueRefresh(existingConfigMap *corev1.ConfigMap) error { - provider := *processor.Provider - reconcileState := processor.ReconciliationState[processor.NamespacedName] - currentTime := processor.CurrentTime +func (processor *AppConfigurationProviderProcessor) processKeyValueRefresh(existingConfigMap *corev1.ConfigMap) error { + provider := processor.Provider + reconcileState := processor.ReconciliationState var err error // Check if the sentinel based refresh is enabled - if provider.Spec.Configuration.Refresh != nil && provider.Spec.Configuration.Refresh.Enabled { + if provider.Spec.Configuration.Refresh != nil && + provider.Spec.Configuration.Refresh.Enabled { processor.RefreshOptions.sentinelBasedRefreshEnabled = true } else { reconcileState.NextSentinelBasedRefreshReconcileTime = metav1.Time{} @@ -132,13 +131,13 @@ func (processor *AppConfigurationProviderProcessor) ProcessKeyValueRefresh(exist } refreshInterval, _ := time.ParseDuration(provider.Spec.Configuration.Refresh.Interval) - nextSentinelBasedRefreshReconcileTime := metav1.Time{Time: currentTime.Add(refreshInterval)} + nextSentinelBasedRefreshReconcileTime := metav1.Time{Time: processor.CurrentTime.Add(refreshInterval)} if processor.ShouldReconcile { reconcileState.NextSentinelBasedRefreshReconcileTime = nextSentinelBasedRefreshReconcileTime return nil } - if !currentTime.After(reconcileState.NextSentinelBasedRefreshReconcileTime.Time) { + if !processor.CurrentTime.After(reconcileState.NextSentinelBasedRefreshReconcileTime.Time) { return nil } @@ -155,13 +154,13 @@ func (processor *AppConfigurationProviderProcessor) ProcessKeyValueRefresh(exist if processor.Settings.ConfigMapSettings != nil { existingConfigMapSettings = &processor.Settings.ConfigMapSettings } - keyValueRefreshedSettings, err := (*processor.Retriever).RefreshKeyValueSettings(processor.Context, existingConfigMapSettings, processor.ResolveSecretReference) + keyValueRefreshedSettings, err := (*processor.Retriever).RefreshKeyValueSettings(processor.Context, existingConfigMapSettings, processor.SecretReferenceResolver) if err != nil { return err } processor.Settings = keyValueRefreshedSettings - reconcileState.CachedSecretReferences = keyValueRefreshedSettings.KeyVaultReferencesToCache + reconcileState.ExistingSecretReferences = keyValueRefreshedSettings.SecretReferences processor.RefreshOptions.ConfigMapSettingPopulated = true if processor.Provider.Spec.Secret != nil { processor.RefreshOptions.SecretSettingPopulated = true @@ -172,62 +171,119 @@ func (processor *AppConfigurationProviderProcessor) ProcessKeyValueRefresh(exist return nil } -func (processor *AppConfigurationProviderProcessor) ProcessKeyVaultReferenceRefresh(existingSecret *corev1.Secret) error { - provider := *processor.Provider - reconcileState := processor.ReconciliationState[processor.NamespacedName] - currentTime := processor.CurrentTime +func (processor *AppConfigurationProviderProcessor) processSecretReferenceRefresh(existingSecrets map[string]corev1.Secret) error { + provider := processor.Provider + reconcileState := processor.ReconciliationState // Check if the key vault dynamic feature if enabled - if provider.Spec.Secret != nil && provider.Spec.Secret.Refresh != nil && provider.Spec.Secret.Refresh.Enabled { - processor.RefreshOptions.keyVaultRefreshEnabled = true + if provider.Spec.Secret != nil && + provider.Spec.Secret.Refresh != nil && + provider.Spec.Secret.Refresh.Enabled { + processor.RefreshOptions.secretReferenceRefreshEnabled = true } - if !processor.RefreshOptions.keyVaultRefreshEnabled { - reconcileState.NextKeyVaultReferenceRefreshReconcileTime = metav1.Time{} - reconcileState.CachedSecretReferences = make(map[string]loader.KeyVaultSecretUriSegment) + if !processor.RefreshOptions.secretReferenceRefreshEnabled { + reconcileState.NextSecretReferenceRefreshReconcileTime = metav1.Time{} return nil } - if !currentTime.After(reconcileState.NextKeyVaultReferenceRefreshReconcileTime.Time) { + if !processor.CurrentTime.After(reconcileState.NextSecretReferenceRefreshReconcileTime.Time) { return nil } - processor.RefreshOptions.keyVaultRefreshNeeded = true + processor.RefreshOptions.secretReferenceRefreshNeeded = true keyVaultRefreshInterval, _ := time.ParseDuration(provider.Spec.Secret.Refresh.Interval) - nextKeyVaultReferenceRefreshReconcileTime := metav1.Time{Time: currentTime.Add(keyVaultRefreshInterval)} + nextSecretReferenceRefreshReconcileTime := metav1.Time{Time: processor.CurrentTime.Add(keyVaultRefreshInterval)} // When SecretSettingPopulated means ProcessFullReconciliation or ProcessKeyValueRefresh has executed, update next refresh time and return if processor.RefreshOptions.SecretSettingPopulated { - reconcileState.NextKeyVaultReferenceRefreshReconcileTime = nextKeyVaultReferenceRefreshReconcileTime + reconcileState.NextSecretReferenceRefreshReconcileTime = nextSecretReferenceRefreshReconcileTime return nil } - resolvedSecretData, err := (*processor.Retriever).ResolveKeyVaultReferences(processor.Context, reconcileState.CachedSecretReferences, processor.ResolveSecretReference) + // Only resolve the secret references that not specified the secret version + secretReferencesToSolve := make(map[string]*loader.TargetSecretReference) + for secretName, reference := range reconcileState.ExistingSecretReferences { + for key, uriSegment := range reference.UriSegments { + if uriSegment.SecretVersion == "" { + if secretReferencesToSolve[secretName] == nil { + secretReferencesToSolve[secretName] = &loader.TargetSecretReference{ + Type: reference.Type, + UriSegments: make(map[string]loader.KeyVaultSecretUriSegment), + } + } + secretReferencesToSolve[secretName].UriSegments[key] = uriSegment + } + } + } + + resolvedSecretData, err := (*processor.Retriever).ResolveSecretReferences(processor.Context, secretReferencesToSolve, processor.SecretReferenceResolver) if err != nil { return err } - maps.Copy(existingSecret.Data, resolvedSecretData) - processor.Settings.SecretSettings = existingSecret.Data + + for secretName, targetSecret := range resolvedSecretData { + existingSecret, ok := existingSecrets[secretName] + if ok { + maps.Copy(existingSecret.Data, targetSecret.Data) + } + } + processor.Settings.SecretSettings = existingSecrets processor.RefreshOptions.SecretSettingPopulated = true // Update next refresh time only if settings updated successfully - reconcileState.NextKeyVaultReferenceRefreshReconcileTime = nextKeyVaultReferenceRefreshReconcileTime + reconcileState.NextSecretReferenceRefreshReconcileTime = nextSecretReferenceRefreshReconcileTime return nil } +func (processor *AppConfigurationProviderProcessor) shouldReconcile( + existingConfigMap *corev1.ConfigMap, + existingSecrets map[string]corev1.Secret) bool { + + if processor.Provider.Generation != processor.ReconciliationState.Generation { + // If the provider is updated, we need to reconcile anyway + return true + } + + if processor.ReconciliationState.ConfigMapResourceVersion == nil || + *processor.ReconciliationState.ConfigMapResourceVersion != existingConfigMap.ResourceVersion { + // If the ConfigMap is removed or updated, we need to reconcile anyway + return true + } + + if processor.Provider.Spec.Secret == nil { + return false + } + + if len(processor.ReconciliationState.ExistingSecretReferences) == 0 || + len(processor.ReconciliationState.ExistingSecretReferences) != len(existingSecrets) { + return true + } + + for name, secret := range existingSecrets { + if processor.ReconciliationState.ExistingSecretReferences[name].SecretResourceVersion != secret.ResourceVersion { + return true + } + } + + return false +} + func (processor *AppConfigurationProviderProcessor) Finish() (ctrl.Result, error) { - processor.ReconciliationState[processor.NamespacedName].Generation = processor.Provider.Generation - if !processor.RefreshOptions.keyVaultRefreshEnabled && !processor.RefreshOptions.sentinelBasedRefreshEnabled && !processor.RefreshOptions.featureFlagRefreshEnabled { + processor.ReconciliationState.Generation = processor.Provider.Generation + if !processor.RefreshOptions.secretReferenceRefreshEnabled && + !processor.RefreshOptions.sentinelBasedRefreshEnabled && + !processor.RefreshOptions.featureFlagRefreshEnabled { // Do nothing, just complete the reconcile klog.V(1).Infof("Complete reconcile AzureAppConfigurationProvider %q in %q namespace", processor.Provider.Name, processor.Provider.Namespace) return reconcile.Result{}, nil } else { // Update the sentinel ETags and last sentinel refresh time if processor.RefreshOptions.sentinelChanged { - processor.ReconciliationState[processor.NamespacedName].SentinelETags = processor.RefreshOptions.updatedSentinelETags + processor.ReconciliationState.SentinelETags = processor.RefreshOptions.updatedSentinelETags processor.Provider.Status.RefreshStatus.LastSentinelBasedRefreshTime = processor.CurrentTime } // Update provider last key vault refresh time - if processor.RefreshOptions.keyVaultRefreshNeeded { + if processor.RefreshOptions.secretReferenceRefreshNeeded { processor.Provider.Status.RefreshStatus.LastKeyVaultReferenceRefreshTime = processor.CurrentTime } // Update provider last feature flag refresh time @@ -244,22 +300,22 @@ func (processor *AppConfigurationProviderProcessor) Finish() (ctrl.Result, error func NewRefreshOptions() *RefreshOptions { return &RefreshOptions{ - sentinelBasedRefreshEnabled: false, - sentinelChanged: false, - keyVaultRefreshEnabled: false, - keyVaultRefreshNeeded: false, - updatedSentinelETags: make(map[acpv1.Sentinel]*azcore.ETag), - featureFlagRefreshEnabled: false, - featureFlagRefreshNeeded: false, - ConfigMapSettingPopulated: false, - SecretSettingPopulated: false, + sentinelBasedRefreshEnabled: false, + sentinelChanged: false, + secretReferenceRefreshEnabled: false, + secretReferenceRefreshNeeded: false, + featureFlagRefreshEnabled: false, + featureFlagRefreshNeeded: false, + ConfigMapSettingPopulated: false, + SecretSettingPopulated: false, + updatedSentinelETags: make(map[acpv1.Sentinel]*azcore.ETag), } } func (processor *AppConfigurationProviderProcessor) calculateRequeueAfterInterval() time.Duration { - reconcileState := processor.ReconciliationState[processor.NamespacedName] + reconcileState := processor.ReconciliationState nextRefreshTimeList := []metav1.Time{reconcileState.NextSentinelBasedRefreshReconcileTime, - reconcileState.NextKeyVaultReferenceRefreshReconcileTime, reconcileState.NextFeatureFlagRefreshReconcileTime} + reconcileState.NextSecretReferenceRefreshReconcileTime, reconcileState.NextFeatureFlagRefreshReconcileTime} var nextRequeueTime metav1.Time for _, time := range nextRefreshTimeList { diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 7b170ad..16409c5 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -227,7 +227,7 @@ func verifyRefreshInterval(interval string, allowedMinimalRefreshInterval time.D refreshInterval, err := time.ParseDuration(interval) if err == nil { if refreshInterval < allowedMinimalRefreshInterval { - return loader.NewArgumentError(refreshArgument, fmt.Errorf("%s can not be shorter than %s.", refreshArgument, allowedMinimalRefreshInterval.String())) + return loader.NewArgumentError(refreshArgument, fmt.Errorf("%s can not be shorter than %s", refreshArgument, allowedMinimalRefreshInterval.String())) } } else { return loader.NewArgumentError(refreshArgument, err) diff --git a/internal/loader/configuraiton_setting_loader_test.go b/internal/loader/configuraiton_setting_loader_test.go index 8f0d79c..b51feea 100644 --- a/internal/loader/configuraiton_setting_loader_test.go +++ b/internal/loader/configuraiton_setting_loader_test.go @@ -6,14 +6,27 @@ package loader import ( acpv1 "azappconfig/provider/api/v1" "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/pem" "errors" + "fmt" + "math/big" "reflect" "testing" + "time" "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" + "github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets" "github.com/golang/mock/gomock" + "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + pkcs12 "software.sslmate.com/src/go-pkcs12" . "github.com/onsi/gomega" "github.com/stretchr/testify/assert" @@ -70,8 +83,8 @@ func newCommonKeyValueSettings(key string, value string, label string) azappconf } func newKeyVaultSettings(key string, label string) azappconfig.Setting { - vault := "{ \"test\":\"https://fake-vault\"}" - keyVaultContentType := KeyVaultReferenceContentType + vault := "{ \"uri\":\"https://fake-vault/secrets/fakesecret\"}" + keyVaultContentType := SecretReferenceContentType return azappconfig.Setting{ Key: &key, @@ -104,10 +117,10 @@ func (m *MockResolveSecretReference) EXPECT() *MockResolveSecretReferenceMockRec } // Resolve mocks base method. -func (m *MockResolveSecretReference) Resolve(arg0 KeyVaultSecretUriSegment, arg1 context.Context) (*string, error) { +func (m *MockResolveSecretReference) Resolve(arg0 KeyVaultSecretUriSegment, arg1 context.Context) (azsecrets.GetSecretResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Resolve", arg0, arg1) - ret0, _ := ret[0].(*string) + ret0, _ := ret[0].(azsecrets.GetSecretResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -124,6 +137,12 @@ const ( ConfigMapName = "configmap-to-be-created" ) +func TestLoaderAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Test loader APIs") +} + var _ = BeforeEach(func() { mockCtrl = gomock.NewController(GinkgoT()) mockResolveSecretReference = NewMockResolveSecretReference(mockCtrl) @@ -152,6 +171,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Context("When get Key Vault Reference Type Settings", func() { It("Should put into Secret settings collection", func() { By("By resolving the settings from Azure Key Vault") + managedIdentity := uuid.New().String() testSpec := acpv1.AzureAppConfigurationProviderSpec{ Endpoint: &EndpointName, Target: acpv1.ConfigurationGenerationParameters{ @@ -160,6 +180,16 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ TrimKeyPrefixes: []string{"app:"}, }, + Secret: &acpv1.SecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: "targetSecret", + }, + Auth: &acpv1.AzureKeyVaultAuth{ + AzureAppConfigurationProviderAuth: &acpv1.AzureAppConfigurationProviderAuth{ + ManagedIdentityClientId: &managedIdentity, + }, + }, + }, } testProvider := acpv1.AzureAppConfigurationProvider{ TypeMeta: metav1.TypeMeta{ @@ -175,18 +205,21 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) secretValue := "fakeSecretValue" - secretValue2 := "fakeSecretValue2" - mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Times(0).Return(&secretValue, nil) - mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Times(0).Return(&secretValue2, nil) + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) + Expect(err).Should(BeNil()) Expect(len(allSettings.ConfigMapSettings)).Should(Equal(2)) - Expect(len(allSettings.SecretSettings)).Should(Equal(2)) + Expect(len(allSettings.SecretSettings)).Should(Equal(1)) Expect(allSettings.ConfigMapSettings["someKey1"]).Should(Equal("value1")) - Expect(allSettings.ConfigMapSettings["someSubKey1;1"]).Should(Equal("value2")) - Expect(allSettings.SecretSettings["secret:1"]).Should(Equal(secretValue)) - Expect(allSettings.SecretSettings["someSecret"]).Should(Equal(secretValue2)) - Expect(err).Should(BeNil()) + Expect(allSettings.ConfigMapSettings["someSubKey1:1"]).Should(Equal("value4")) + Expect(string(allSettings.SecretSettings["targetSecret"].Data["secret:1"])).Should(Equal(secretValue)) }) It("Should throw exception", func() { @@ -214,13 +247,403 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) - secretValue := "fakeSecretValue" - mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Times(0).Return(&secretValue, nil) - mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Times(0).Return(nil, errors.New("Some error")) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) Expect(allSettings).Should(BeNil()) - Expect(err.Error()).Should(Equal("Some error")) + Expect(err.Error()).Should(Equal("A Key Vault reference is found in App Configuration, but 'spec.secret' was not configured in the Azure App Configuration provider 'testName' in namespace 'testNamespace'")) + }) + + It("Should throw unknown content type error", func() { + By("By getting unknown cert type from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue := "fakeSecretValue" + secretName := "targetSecret" + contentType := "fake-content-type" + kidStr := "fakeKid" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + Kid: &kidStr, + ContentType: &contentType, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + _, err := configurationProvider.ResolveSecretReferences(context.Background(), secretReferencesToResolve, mockResolveSecretReference) + + Expect(err.Error()).Should(Equal("fail to decode the cert 'targetSecret': unknown content type 'fake-content-type'")) + }) + + It("Should throw decode pem block error", func() { + By("By getting unexpected secret value of pem cert from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue := "fakeSecretValue" + secretName := "targetSecret" + contentType := CertTypePem + kidStr := "fakeKid" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + Kid: &kidStr, + ContentType: &contentType, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + _, err := configurationProvider.ResolveSecretReferences(context.Background(), secretReferencesToResolve, mockResolveSecretReference) + + Expect(err.Error()).Should(Equal("fail to decode the cert 'targetSecret': failed to decode pem block")) + }) + + It("Should throw decode pfx error", func() { + By("By getting unexpected secret value of pfx cert from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue := "fakeSecretValue" + secretName := "targetSecret" + contentType := CertTypePfx + kidStr := "fakeKid" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + Kid: &kidStr, + ContentType: &contentType, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + _, err := configurationProvider.ResolveSecretReferences(context.Background(), secretReferencesToResolve, mockResolveSecretReference) + + Expect(err.Error()).Should(Equal("fail to decode the cert 'targetSecret': illegal base64 data at input byte 12")) + }) + + It("Succeeded to get tls type secret", func() { + By("By getting valid pfx cert from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue, _ := createFakePfx() + secretName := "targetSecret" + contentType := CertTypePfx + kidStr := "fakeKid" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + Kid: &kidStr, + ContentType: &contentType, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + secrets, err := configurationProvider.ResolveSecretReferences(context.Background(), secretReferencesToResolve, mockResolveSecretReference) + + Expect(err).Should(BeNil()) + Expect(len(secrets)).Should(Equal(1)) + Expect(string(secrets[secretName].Data["tls.crt"])).Should(ContainSubstring("BEGIN CERTIFICATE")) + Expect(string(secrets[secretName].Data["tls.key"])).Should(ContainSubstring("BEGIN PRIVATE KEY")) + }) + + It("Succeeded to get target tls type secret", func() { + By("By getting valid pem cert from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue, _ := createFakePem() + secretName := "targetSecret" + contentType := CertTypePem + kidStr := "fakeKid" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + Kid: &kidStr, + ContentType: &contentType, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + secrets, err := configurationProvider.ResolveSecretReferences(context.Background(), secretReferencesToResolve, mockResolveSecretReference) + + Expect(err).Should(BeNil()) + Expect(len(secrets)).Should(Equal(1)) + Expect(string(secrets[secretName].Data["tls.crt"])).Should(ContainSubstring("BEGIN CERTIFICATE")) + Expect(string(secrets[secretName].Data["tls.key"])).Should(ContainSubstring("BEGIN RSA PRIVATE KEY")) + }) + + It("Succeeded to get tls type secret", func() { + By("By getting valid non cert based secret from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue, _ := createFakePfx() + secretName := "targetSecret" + ct := CertTypePfx + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + ContentType: &ct, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + secrets, err := configurationProvider.ResolveSecretReferences(context.Background(), secretReferencesToResolve, mockResolveSecretReference) + + Expect(err).Should(BeNil()) + Expect(len(secrets)).Should(Equal(1)) + Expect(string(secrets[secretName].Data["tls.crt"])).Should(ContainSubstring("BEGIN CERTIFICATE")) + Expect(string(secrets[secretName].Data["tls.key"])).Should(ContainSubstring("BEGIN PRIVATE KEY")) + }) + + It("Succeeded to get tls type secret", func() { + By("By getting valid non cert based pem secret from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue, _ := createFakePem() + secretName := "targetSecret" + ct := CertTypePem + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + ContentType: &ct, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + secrets, err := configurationProvider.ResolveSecretReferences(context.Background(), secretReferencesToResolve, mockResolveSecretReference) + + Expect(err).Should(BeNil()) + Expect(len(secrets)).Should(Equal(1)) + Expect(string(secrets[secretName].Data["tls.crt"])).Should(ContainSubstring("BEGIN CERTIFICATE")) + Expect(string(secrets[secretName].Data["tls.key"])).Should(ContainSubstring("BEGIN RSA PRIVATE KEY")) }) }) }) @@ -525,7 +948,7 @@ func TestCreateSecretClients(t *testing.T) { Target: acpv1.ConfigurationGenerationParameters{ ConfigMapName: "configMap-test", }, - Secret: &acpv1.AzureKeyVaultReference{ + Secret: &acpv1.SecretReference{ Target: acpv1.SecretGenerationParameters{ SecretName: "secret-test", }, @@ -557,7 +980,7 @@ func TestCreateSecretClients(t *testing.T) { Target: acpv1.ConfigurationGenerationParameters{ ConfigMapName: "configMap-test", }, - Secret: &acpv1.AzureKeyVaultReference{ + Secret: &acpv1.SecretReference{ Target: acpv1.SecretGenerationParameters{ SecretName: "secretName", }, @@ -593,3 +1016,118 @@ func TestCreateSecretClients(t *testing.T) { assert.NotNil(t, r1) assert.NotNil(t, r2) } + +func createFakeKeyPem(key *rsa.PrivateKey) (string, error) { + keyBytes := x509.MarshalPKCS1PrivateKey(key) + // PEM encoding of private key + keyPEM := string(pem.EncodeToMemory( + &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: keyBytes, + }, + )) + + return keyPEM, nil +} + +func createFakeCertPem(key *rsa.PrivateKey) (string, error) { + notBefore := time.Now() + notAfter := notBefore.Add(365 * 24 * 10 * time.Hour) + + //Create certificate templet + template := x509.Certificate{ + SerialNumber: big.NewInt(0), + Subject: pkix.Name{CommonName: "localhost"}, + SignatureAlgorithm: x509.SHA256WithRSA, + NotBefore: notBefore, + NotAfter: notAfter, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyAgreement | x509.KeyUsageKeyEncipherment | x509.KeyUsageDataEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + //Create certificate using templet + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + return "", err + + } + //pem encoding of certificate + certPem := string(pem.EncodeToMemory( + &pem.Block{ + Type: "CERTIFICATE", + Bytes: derBytes, + }, + )) + + return certPem, nil +} + +func createFakePem() (string, error) { + key, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return "", err + } + + keyPEM, err := createFakeKeyPem(key) + if err != nil { + return "", err + } + + certPem, err := createFakeCertPem(key) + if err != nil { + return "", err + } + + return keyPEM + "\n" + certPem, nil +} + +func createFakePfx() (string, error) { + key, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return "", err + } + + keyPEM, err := createFakeKeyPem(key) + if err != nil { + return "", err + } + + certPem, err := createFakeCertPem(key) + if err != nil { + return "", err + } + + return createPFXFromPEM(keyPEM, certPem) +} + +func createPFXFromPEM(pemPrivateKey, pemCertificate string) (string, error) { + // Decode private key PEM + block, _ := pem.Decode([]byte(pemPrivateKey)) + if block == nil || block.Type != "RSA PRIVATE KEY" { + return "", fmt.Errorf("failed to decode private key PEM") + } + + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return "", err + } + + // Decode certificate PEM + block, _ = pem.Decode([]byte(pemCertificate)) + if block == nil || block.Type != "CERTIFICATE" { + return "", fmt.Errorf("failed to decode certificate PEM") + } + + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return "", err + } + + // Create PKCS12 structure + pfxData, err := pkcs12.Legacy.Encode(privateKey, cert, []*x509.Certificate{}, "") + if err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(pfxData), nil +} diff --git a/internal/loader/configuration_setting_loader.go b/internal/loader/configuration_setting_loader.go index fb1dcdb..5e11693 100644 --- a/internal/loader/configuration_setting_loader.go +++ b/internal/loader/configuration_setting_loader.go @@ -7,7 +7,10 @@ import ( acpv1 "azappconfig/provider/api/v1" "azappconfig/provider/internal/properties" "context" + "crypto/x509" + "encoding/base64" "encoding/json" + "encoding/pem" "errors" "fmt" "net/url" @@ -20,6 +23,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" "github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets" "github.com/google/uuid" + "golang.org/x/crypto/pkcs12" "golang.org/x/exp/maps" "golang.org/x/sync/errgroup" "golang.org/x/sync/syncmap" @@ -39,25 +43,32 @@ type ConfigurationSettingLoader struct { } type TargetKeyValueSettings struct { - ConfigMapSettings map[string]string - SecretSettings map[string][]byte - KeyVaultReferencesToCache map[string]KeyVaultSecretUriSegment + ConfigMapSettings map[string]string + // Multiple secrets could be managed + SecretSettings map[string]corev1.Secret + SecretReferences map[string]*TargetSecretReference +} + +type TargetSecretReference struct { + Type corev1.SecretType + UriSegments map[string]KeyVaultSecretUriSegment + SecretResourceVersion string } type RawSettings struct { - KeyValueSettings map[string]*string - IsJsonContentTypeMap map[string]bool - FeatureFlagSettings map[string]interface{} - SecretSettings map[string][]byte - KeyVaultReferencesToCache map[string]KeyVaultSecretUriSegment + KeyValueSettings map[string]*string + IsJsonContentTypeMap map[string]bool + FeatureFlagSettings map[string]interface{} + SecretSettings map[string]corev1.Secret + SecretReferences map[string]*TargetSecretReference } type ConfigurationSettingsRetriever interface { - CreateTargetSettings(ctx context.Context, resolveSecretReference ResolveSecretReference) (*TargetKeyValueSettings, error) - RefreshKeyValueSettings(ctx context.Context, existingConfigMapSettings *map[string]string, resolveSecretReference ResolveSecretReference) (*TargetKeyValueSettings, error) + CreateTargetSettings(ctx context.Context, resolveSecretReference SecretReferenceResolver) (*TargetKeyValueSettings, error) + RefreshKeyValueSettings(ctx context.Context, existingConfigMapSettings *map[string]string, resolveSecretReference SecretReferenceResolver) (*TargetKeyValueSettings, error) RefreshFeatureFlagSettings(ctx context.Context, existingConfigMapSettings *map[string]string) (*TargetKeyValueSettings, error) CheckAndRefreshSentinels(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, eTags map[acpv1.Sentinel]*azcore.ETag) (bool, map[acpv1.Sentinel]*azcore.ETag, error) - ResolveKeyVaultReferences(ctx context.Context, kvReferencesToResolve map[string]KeyVaultSecretUriSegment, kvResolver ResolveSecretReference) (map[string][]byte, error) + ResolveSecretReferences(ctx context.Context, kvReferencesToResolve map[string]*TargetSecretReference, kvResolver SecretReferenceResolver) (map[string]corev1.Secret, error) } type GetSettingsFunc func(ctx context.Context, filters []acpv1.Selector, client *azappconfig.Client, c chan []azappconfig.Setting, e chan error) @@ -69,7 +80,7 @@ type ServicePrincipleAuthenticationParameters struct { } const ( - KeyVaultReferenceContentType string = "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8" + SecretReferenceContentType string = "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8" FeatureFlagContentType string = "application/vnd.microsoft.appconfig.ff+json;charset=utf-8" AzureClientId string = "azure_client_id" AzureClientSecret string = "azure_client_secret" @@ -78,6 +89,11 @@ const ( FeatureFlagKeyPrefix string = ".appconfig.featureflag/" FeatureFlagSectionName string = "feature_flags" FeatureManagementSectionName string = "feature_management" + PreservedSecretTypeTag string = ".kubernetes.secret.type" + CertTypePem string = "application/x-pem-file" + CertTypePfx string = "application/x-pkcs12" + TlsKey string = "tls.key" + TlsCrt string = "tls.crt" ) var ( @@ -123,7 +139,7 @@ func NewConfigurationSettingLoader(ctx context.Context, provider acpv1.AzureAppC }, nil } -func (csl *ConfigurationSettingLoader) CreateTargetSettings(ctx context.Context, resolveSecretReference ResolveSecretReference) (*TargetKeyValueSettings, error) { +func (csl *ConfigurationSettingLoader) CreateTargetSettings(ctx context.Context, resolveSecretReference SecretReferenceResolver) (*TargetKeyValueSettings, error) { rawSettings, err := csl.CreateKeyValueSettings(ctx, resolveSecretReference) if err != nil { return nil, err @@ -141,13 +157,13 @@ func (csl *ConfigurationSettingLoader) CreateTargetSettings(ctx context.Context, } return &TargetKeyValueSettings{ - ConfigMapSettings: typedSettings, - SecretSettings: rawSettings.SecretSettings, - KeyVaultReferencesToCache: rawSettings.KeyVaultReferencesToCache, + ConfigMapSettings: typedSettings, + SecretSettings: rawSettings.SecretSettings, + SecretReferences: rawSettings.SecretReferences, }, nil } -func (csl *ConfigurationSettingLoader) RefreshKeyValueSettings(ctx context.Context, existingConfigMapSetting *map[string]string, resolveSecretReference ResolveSecretReference) (*TargetKeyValueSettings, error) { +func (csl *ConfigurationSettingLoader) RefreshKeyValueSettings(ctx context.Context, existingConfigMapSetting *map[string]string, resolveSecretReference SecretReferenceResolver) (*TargetKeyValueSettings, error) { rawSettings, err := csl.CreateKeyValueSettings(ctx, resolveSecretReference) if err != nil { return nil, err @@ -166,9 +182,9 @@ func (csl *ConfigurationSettingLoader) RefreshKeyValueSettings(ctx context.Conte } return &TargetKeyValueSettings{ - ConfigMapSettings: typedSettings, - SecretSettings: rawSettings.SecretSettings, - KeyVaultReferencesToCache: rawSettings.KeyVaultReferencesToCache, + ConfigMapSettings: typedSettings, + SecretSettings: rawSettings.SecretSettings, + SecretReferences: rawSettings.SecretReferences, }, nil } @@ -195,20 +211,20 @@ func (csl *ConfigurationSettingLoader) RefreshFeatureFlagSettings(ctx context.Co }, nil } -func (csl *ConfigurationSettingLoader) CreateKeyValueSettings(ctx context.Context, resolveSecretReference ResolveSecretReference) (*RawSettings, error) { +func (csl *ConfigurationSettingLoader) CreateKeyValueSettings(ctx context.Context, secretReferenceResolver SecretReferenceResolver) (*RawSettings, error) { settingsChan := make(chan []azappconfig.Setting) errChan := make(chan error) keyValueFilters := getKeyValueFilters(csl.Spec) go csl.getSettingsFunc(ctx, keyValueFilters, csl.AppConfigClient, settingsChan, errChan) rawSettings := &RawSettings{ - KeyValueSettings: make(map[string]*string), - IsJsonContentTypeMap: make(map[string]bool), - SecretSettings: make(map[string][]byte), - KeyVaultReferencesToCache: make(map[string]KeyVaultSecretUriSegment), + KeyValueSettings: make(map[string]*string), + IsJsonContentTypeMap: make(map[string]bool), + SecretSettings: make(map[string]corev1.Secret), + SecretReferences: make(map[string]*TargetSecretReference), } var settings []azappconfig.Setting - var kvResolver ResolveSecretReference + resolver := secretReferenceResolver for { select { @@ -230,23 +246,30 @@ func (csl *ConfigurationSettingLoader) CreateKeyValueSettings(ctx context.Contex } switch *setting.ContentType { case FeatureFlagContentType: - continue // ignore feature flag at this moment, will support it in later version - case KeyVaultReferenceContentType: + continue // ignore feature flag while getting key value settings + case SecretReferenceContentType: if setting.Value == nil { return nil, fmt.Errorf("The value of Key Vault reference '%s' is null", *setting.Key) } + if csl.Spec.Secret == nil { return nil, fmt.Errorf("A Key Vault reference is found in App Configuration, but 'spec.secret' was not configured in the Azure App Configuration provider '%s' in namespace '%s'", csl.Name, csl.Namespace) } - if kvResolver == nil { - if resolveSecretReference == nil { - if newKvResolver, err := csl.createKeyVaultResolver(ctx); err != nil { - return nil, err - } else { - kvResolver = newKvResolver - } + + var secretType corev1.SecretType = corev1.SecretTypeOpaque + var err error + if secretTypeTag, ok := setting.Tags[PreservedSecretTypeTag]; ok { + secretType, err = parseSecretType(secretTypeTag) + if err != nil { + return nil, err + } + } + + if resolver == nil { + if newResolver, err := csl.createSecretReferenceResolver(ctx); err != nil { + return nil, err } else { - kvResolver = resolveSecretReference + resolver = newResolver } } @@ -256,23 +279,33 @@ func (csl *ConfigurationSettingLoader) CreateKeyValueSettings(ctx context.Contex return nil, err } - // Cache the non-versioned secret reference - if secretUriSegment.SecretVersion == "" { - rawSettings.KeyVaultReferencesToCache[trimmedKey] = *secretUriSegment + secretName := trimmedKey + // If the secret type is not specified, reside it to the Secret with name specified + if secretType == corev1.SecretTypeOpaque { + secretName = csl.Spec.Secret.Target.SecretName } - rawSettings.KeyVaultReferencesToCache[trimmedKey] = *secretUriSegment + if _, ok := rawSettings.SecretReferences[secretName]; !ok { + rawSettings.SecretReferences[secretName] = &TargetSecretReference{ + Type: secretType, + UriSegments: make(map[string]KeyVaultSecretUriSegment), + } + } + rawSettings.SecretReferences[secretName].UriSegments[trimmedKey] = *secretUriSegment default: rawSettings.KeyValueSettings[trimmedKey] = setting.Value rawSettings.IsJsonContentTypeMap[trimmedKey] = isJsonContentType(setting.ContentType) } } - // resolve the keyVault reference settings - if resolvedSecret, err := csl.ResolveKeyVaultReferences(ctx, rawSettings.KeyVaultReferencesToCache, kvResolver); err != nil { + // resolve the secret reference settings + if resolvedSecret, err := csl.ResolveSecretReferences(ctx, rawSettings.SecretReferences, resolver); err != nil { return nil, err } else { - maps.Copy(rawSettings.SecretSettings, resolvedSecret) + err = MergeSecret(rawSettings.SecretSettings, resolvedSecret) + if err != nil { + return nil, err + } } case err := <-errChan: if err != nil { @@ -323,7 +356,10 @@ end: return featureFlagSection, nil } -func (csl *ConfigurationSettingLoader) CheckAndRefreshSentinels(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, eTags map[acpv1.Sentinel]*azcore.ETag) (bool, map[acpv1.Sentinel]*azcore.ETag, error) { +func (csl *ConfigurationSettingLoader) CheckAndRefreshSentinels( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + eTags map[acpv1.Sentinel]*azcore.ETag) (bool, map[acpv1.Sentinel]*azcore.ETag, error) { sentinelChanged := false if provider.Spec.Configuration.Refresh == nil { return sentinelChanged, eTags, NewArgumentError("spec.configuration.refresh", fmt.Errorf("refresh is not specified")) @@ -349,33 +385,76 @@ func (csl *ConfigurationSettingLoader) CheckAndRefreshSentinels(ctx context.Cont return sentinelChanged, refreshedETags, nil } -func (csl *ConfigurationSettingLoader) ResolveKeyVaultReferences(ctx context.Context, keyVaultReferencesToResolve map[string]KeyVaultSecretUriSegment, keyVaultResolver ResolveSecretReference) (map[string][]byte, error) { - if keyVaultResolver == nil { - if kvResolver, err := csl.createKeyVaultResolver(ctx); err != nil { +func (csl *ConfigurationSettingLoader) ResolveSecretReferences( + ctx context.Context, + secretReferencesToResolve map[string]*TargetSecretReference, + resolver SecretReferenceResolver) (map[string]corev1.Secret, error) { + if resolver == nil { + if kvResolver, err := csl.createSecretReferenceResolver(ctx); err != nil { return nil, err } else { - keyVaultResolver = kvResolver + resolver = kvResolver } } - resolvedSecretReferences := make(map[string][]byte) - if len(keyVaultReferencesToResolve) > 0 { + resolvedSecretReferences := make(map[string]corev1.Secret) + for name, targetSecretReference := range secretReferencesToResolve { + resolvedSecretReferences[name] = corev1.Secret{ + Data: make(map[string][]byte), + Type: targetSecretReference.Type, + } + var eg errgroup.Group - lock := &sync.Mutex{} - for key, kvReference := range keyVaultReferencesToResolve { - currentKey := key - currentReference := kvReference + if targetSecretReference.Type == corev1.SecretTypeOpaque { + if len(targetSecretReference.UriSegments) > 0 { + lock := &sync.Mutex{} + for key, kvReference := range targetSecretReference.UriSegments { + currentKey := key + currentReference := kvReference + eg.Go(func() error { + resolvedSecret, err := resolver.Resolve(currentReference, ctx) + if err != nil { + return fmt.Errorf("fail to resolve the Key Vault reference type setting '%s': %s", currentKey, err.Error()) + } + lock.Lock() + defer lock.Unlock() + resolvedSecretReferences[name].Data[currentKey] = []byte(*resolvedSecret.Value) + return nil + }) + } + + if err := eg.Wait(); err != nil { + return nil, err + } + } + } else if targetSecretReference.Type == corev1.SecretTypeTLS { eg.Go(func() error { - resolvedValue, err := keyVaultResolver.Resolve(currentReference, ctx) + resolvedSecret, err := resolver.Resolve(targetSecretReference.UriSegments[name], ctx) if err != nil { - return fmt.Errorf("Fail to resolve the Key Vault reference type setting '%s': %s", currentKey, err.Error()) + return fmt.Errorf("fail to resolve the Key Vault reference type setting '%s': %s", name, err.Error()) } - lock.Lock() - defer lock.Unlock() - resolvedSecretReferences[currentKey] = []byte(*resolvedValue) + + if resolvedSecret.ContentType == nil { + return fmt.Errorf("unspecified content type") + } + + switch *resolvedSecret.ContentType { + case CertTypePfx: + resolvedSecretReferences[name].Data[TlsKey], resolvedSecretReferences[name].Data[TlsCrt], err = decodePkcs12(*resolvedSecret.Value) + case CertTypePem: + resolvedSecretReferences[name].Data[TlsKey], resolvedSecretReferences[name].Data[TlsCrt], err = decodePem(*resolvedSecret.Value) + default: + err = fmt.Errorf("unknown content type '%s'", *resolvedSecret.ContentType) + } + + if err != nil { + return fmt.Errorf("fail to decode the cert '%s': %s", name, err.Error()) + } + return nil }) } + // All other types are not supported if err := eg.Wait(); err != nil { return nil, err @@ -415,7 +494,7 @@ func (csl *ConfigurationSettingLoader) getSentinelSetting(ctx context.Context, p return &sentinelSetting.Setting, nil } -func (csl *ConfigurationSettingLoader) createKeyVaultResolver(ctx context.Context) (ResolveSecretReference, error) { +func (csl *ConfigurationSettingLoader) createSecretReferenceResolver(ctx context.Context) (SecretReferenceResolver, error) { var defaultAuth *acpv1.AzureAppConfigurationProviderAuth = nil if csl.Spec.Secret != nil && csl.Spec.Secret.Auth != nil { defaultAuth = csl.Spec.Secret.Auth.AzureAppConfigurationProviderAuth @@ -428,12 +507,12 @@ func (csl *ConfigurationSettingLoader) createKeyVaultResolver(ctx context.Contex if err != nil { return nil, err } - keyVaultResolver := &KeyVaultReferenceResolver{ + resolver := &KeyVaultConnector{ DefaultTokenCredential: defaultCred, Clients: secretClients, } - return keyVaultResolver, nil + return resolver, nil } func trimPrefix(key string, prefixToTrim []string) string { @@ -448,7 +527,11 @@ func trimPrefix(key string, prefixToTrim []string) string { return key } -func getConfigurationSettings(ctx context.Context, filters []acpv1.Selector, client *azappconfig.Client, c chan []azappconfig.Setting, e chan error) { +func getConfigurationSettings( + ctx context.Context, + filters []acpv1.Selector, + client *azappconfig.Client, + c chan []azappconfig.Setting, e chan error) { nullString := "\x00" for _, filter := range filters { @@ -475,7 +558,10 @@ func getConfigurationSettings(ctx context.Context, filters []acpv1.Selector, cli c <- make([]azappconfig.Setting, 0) } -func createTokenCredential(ctx context.Context, acpAuth *acpv1.AzureAppConfigurationProviderAuth, namespace string) (azcore.TokenCredential, error) { +func createTokenCredential( + ctx context.Context, + acpAuth *acpv1.AzureAppConfigurationProviderAuth, + namespace string) (azcore.TokenCredential, error) { // If User explicitly specify the authentication method if acpAuth != nil { if acpAuth.WorkloadIdentity != nil { @@ -506,7 +592,10 @@ func createTokenCredential(ctx context.Context, acpAuth *acpv1.AzureAppConfigura return nil, nil } -func getWorkloadIdentityClientId(ctx context.Context, workloadIdentityAuth *acpv1.WorkloadIdentityParameters, namespace string) (string, error) { +func getWorkloadIdentityClientId( + ctx context.Context, + workloadIdentityAuth *acpv1.WorkloadIdentityParameters, + namespace string) (string, error) { if workloadIdentityAuth.ManagedIdentityClientIdReference == nil { return *workloadIdentityAuth.ManagedIdentityClientId, nil } else { @@ -528,8 +617,10 @@ func getWorkloadIdentityClientId(ctx context.Context, workloadIdentityAuth *acpv } } -func getConnectionStringParameter(ctx context.Context, namespacedSecretName types.NamespacedName) (string, error) { - secret, err := getSecret(ctx, namespacedSecretName) +func getConnectionStringParameter( + ctx context.Context, + namespacedSecretName types.NamespacedName) (string, error) { + secret, err := GetSecret(ctx, namespacedSecretName) if err != nil { return "", err } @@ -537,8 +628,10 @@ func getConnectionStringParameter(ctx context.Context, namespacedSecretName type return string(secret.Data[AzureAppConfigurationConnectionString]), nil } -func getServicePrincipleAuthenticationParameters(ctx context.Context, namespacedSecretName types.NamespacedName) (*ServicePrincipleAuthenticationParameters, error) { - secret, err := getSecret(ctx, namespacedSecretName) +func getServicePrincipleAuthenticationParameters( + ctx context.Context, + namespacedSecretName types.NamespacedName) (*ServicePrincipleAuthenticationParameters, error) { + secret, err := GetSecret(ctx, namespacedSecretName) if err != nil { return nil, err } @@ -550,7 +643,8 @@ func getServicePrincipleAuthenticationParameters(ctx context.Context, namespaced }, nil } -func getConfigMap(ctx context.Context, namespacedConfigMapName types.NamespacedName) (*corev1.ConfigMap, error) { +func getConfigMap(ctx context.Context, + namespacedConfigMapName types.NamespacedName) (*corev1.ConfigMap, error) { cfg, err := config.GetConfig() if err != nil { return nil, err @@ -569,7 +663,8 @@ func getConfigMap(ctx context.Context, namespacedConfigMapName types.NamespacedN return configMapObject, nil } -func getSecret(ctx context.Context, namespacedSecretName types.NamespacedName) (*corev1.Secret, error) { +func GetSecret(ctx context.Context, + namespacedSecretName types.NamespacedName) (*corev1.Secret, error) { cfg, err := config.GetConfig() if err != nil { return nil, err @@ -655,7 +750,9 @@ func reverse(arr []acpv1.Selector) { } } -func createSecretClients(ctx context.Context, acp acpv1.AzureAppConfigurationProvider) (*syncmap.Map, error) { +func createSecretClients( + ctx context.Context, + acp acpv1.AzureAppConfigurationProvider) (*syncmap.Map, error) { secretClients := &syncmap.Map{} if acp.Spec.Secret == nil || acp.Spec.Secret.Auth == nil { return secretClients, nil @@ -679,3 +776,108 @@ func createSecretClients(ctx context.Context, acp acpv1.AzureAppConfigurationPro return secretClients, nil } + +func parseSecretType(secretType string) (corev1.SecretType, error) { + secretTypeMap := map[string]corev1.SecretType{ + "opaque": corev1.SecretTypeOpaque, + "kubernetes.io/tls": corev1.SecretTypeTLS, + } + + if parsedType, ok := secretTypeMap[secretType]; ok { + if parsedType != corev1.SecretTypeTLS { + return "", fmt.Errorf("secret type %q is not supported", secretType) + } else { + return parsedType, nil + } + } else { + return "", fmt.Errorf("secret type %q is not supported", secretType) + } +} + +func decodePkcs12(value string) (key []byte, crt []byte, err error) { + pfxRaw, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return nil, nil, err + } + // using ToPEM to extract more than one certificate and key in pfxData + pemBlock, err := pkcs12.ToPEM(pfxRaw, "") + if err != nil { + return nil, nil, err + } + + return parsePemBlock(pemBlock) +} + +func decodePem(value string) (key []byte, crt []byte, err error) { + pemBlocks := []*pem.Block{} + for pemBlock, rest := pem.Decode([]byte(value)); pemBlock != nil; pemBlock, rest = pem.Decode(rest) { + pemBlocks = append(pemBlocks, pemBlock) + } + if len(pemBlocks) == 0 { + return nil, nil, fmt.Errorf("failed to decode pem block") + } + + return parsePemBlock(pemBlocks) +} + +func parsePemBlock(pemBlock []*pem.Block) ([]byte, []byte, error) { + // PEM block encoded form contains the headers + // -----BEGIN Type----- + // Headers + // base64-encoded Bytes + // -----END Type----- + // Setting headers to nil to ensure no headers included in the encoded block + var pemKeyData, pemCertData []byte + for _, block := range pemBlock { + + block.Headers = make(map[string]string) + if block.Type == "CERTIFICATE" { + pemCertData = append(pemCertData, pem.EncodeToMemory(block)...) + } else { + key, err := parsePrivateKey(block.Bytes) + if err != nil { + return nil, nil, err + } + // pkcs1 RSA private key PEM file is specific for RSA keys. RSA is not used exclusively inside X509 + // and SSL/TLS, a more generic key format is available in the form of PKCS#8 that identifies the type + // of private key and contains the relevant data. + // Converting to pkcs8 private key as ToPEM uses pkcs1 + // The driver determines the key type from the pkcs8 form of the key and marshals appropriately + block.Bytes, err = x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, nil, err + } + pemKeyData = append(pemKeyData, pem.EncodeToMemory(block)...) + } + } + + return pemKeyData, pemCertData, nil +} + +func parsePrivateKey(block []byte) (interface{}, error) { + if key, err := x509.ParsePKCS1PrivateKey(block); err == nil { + return key, nil + } + if key, err := x509.ParsePKCS8PrivateKey(block); err == nil { + return key, nil + } + if key, err := x509.ParseECPrivateKey(block); err == nil { + return key, nil + } + return nil, fmt.Errorf("failed to parse key for type pkcs1, pkcs8 or ec") +} + +func MergeSecret(secret map[string]corev1.Secret, newSecret map[string]corev1.Secret) error { + for k, v := range newSecret { + if _, ok := secret[k]; !ok { + secret[k] = v + } else if secret[k].Type != v.Type { + return fmt.Errorf("secret type mismatch for key %q", k) + + } else { + maps.Copy(secret[k].Data, v.Data) + } + } + + return nil +} diff --git a/internal/loader/keyvault_reference_resolver.go b/internal/loader/keyvault_reference_resolver.go index 4813555..8ad9ddf 100644 --- a/internal/loader/keyvault_reference_resolver.go +++ b/internal/loader/keyvault_reference_resolver.go @@ -15,7 +15,7 @@ import ( "golang.org/x/sync/syncmap" ) -type KeyVaultReferenceResolver struct { +type KeyVaultConnector struct { DefaultTokenCredential azcore.TokenCredential Clients *syncmap.Map //map[string]*azsecrets.Client } @@ -30,27 +30,25 @@ type KeyVaultSecretUriSegment struct { SecretVersion string } -type ResolveSecretReference interface { - Resolve(secretUriSegment KeyVaultSecretUriSegment, ctx context.Context) (*string, error) +type SecretReferenceResolver interface { + Resolve(secretUriSegment KeyVaultSecretUriSegment, ctx context.Context) (azsecrets.GetSecretResponse, error) } -func (resolver *KeyVaultReferenceResolver) Resolve(secretUriSegment KeyVaultSecretUriSegment, ctx context.Context) (*string, error) { +func (resolver *KeyVaultConnector) Resolve( + secretUriSegment KeyVaultSecretUriSegment, + ctx context.Context) (azsecrets.GetSecretResponse, error) { var secretClient any var ok bool if secretClient, ok = resolver.Clients.Load(secretUriSegment.HostName); !ok { newSecretClient, err := azsecrets.NewClient("https://"+secretUriSegment.HostName, resolver.DefaultTokenCredential, nil) if err != nil { - return nil, err + return azsecrets.GetSecretResponse{}, err } secretClient = newSecretClient resolver.Clients.Store(secretUriSegment.HostName, newSecretClient) } - secretValue, err := secretClient.(*azsecrets.Client).GetSecret(ctx, secretUriSegment.SecretName, secretUriSegment.SecretVersion, nil) - if err != nil { - return nil, err - } - return secretValue.Value, nil + return secretClient.(*azsecrets.Client).GetSecret(ctx, secretUriSegment.SecretName, secretUriSegment.SecretVersion, nil) } func parse(settingValue string) (*KeyVaultSecretUriSegment, error) { diff --git a/internal/loader/keyvault_reference_resolver_test.go b/internal/loader/keyvault_reference_resolver_test.go index a80e596..a7c937c 100644 --- a/internal/loader/keyvault_reference_resolver_test.go +++ b/internal/loader/keyvault_reference_resolver_test.go @@ -96,13 +96,13 @@ func TestResolveSecretReferenceSetting(t *testing.T) { valueToTest := "{ \"uri\":\"https://FAKE-VAULT/secrets/fake-secret/testversion/notvalid\"}" defaultCredential := mockTokeCredential{} clients := &syncmap.Map{} - resolver := &KeyVaultReferenceResolver{ + resolver := &KeyVaultConnector{ DefaultTokenCredential: defaultCredential, Clients: clients, } secretUriSegment, _ := parse(valueToTest) - resolvedValue, err := resolver.Resolve(*secretUriSegment, context.Background()) + resolvedSecret, err := resolver.Resolve(*secretUriSegment, context.Background()) assert.Contains(t, err.Error(), "fake-vault") length := 0 @@ -111,7 +111,7 @@ func TestResolveSecretReferenceSetting(t *testing.T) { return true }) assert.Equal(t, 1, length) - assert.Nil(t, resolvedValue) + assert.Nil(t, resolvedSecret.Value) } type mockTokeCredential struct { diff --git a/internal/loader/mocks/mock_configuration_settings_retriever.go b/internal/loader/mocks/mock_configuration_settings_retriever.go index edbf282..18052e4 100644 --- a/internal/loader/mocks/mock_configuration_settings_retriever.go +++ b/internal/loader/mocks/mock_configuration_settings_retriever.go @@ -12,6 +12,7 @@ import ( azcore "github.com/Azure/azure-sdk-for-go/sdk/azcore" gomock "github.com/golang/mock/gomock" + v10 "k8s.io/api/core/v1" ) // MockConfigurationSettingsRetriever is a mock of ConfigurationSettingsRetriever interface. @@ -54,7 +55,7 @@ func (mr *MockConfigurationSettingsRetrieverMockRecorder) CheckAndRefreshSentine } // CreateTargetSettings mocks base method. -func (m *MockConfigurationSettingsRetriever) CreateTargetSettings(arg0 context.Context, arg1 loader.ResolveSecretReference) (*loader.TargetKeyValueSettings, error) { +func (m *MockConfigurationSettingsRetriever) CreateTargetSettings(arg0 context.Context, arg1 loader.SecretReferenceResolver) (*loader.TargetKeyValueSettings, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateTargetSettings", arg0, arg1) ret0, _ := ret[0].(*loader.TargetKeyValueSettings) @@ -84,7 +85,7 @@ func (mr *MockConfigurationSettingsRetrieverMockRecorder) RefreshFeatureFlagSett } // RefreshKeyValueSettings mocks base method. -func (m *MockConfigurationSettingsRetriever) RefreshKeyValueSettings(arg0 context.Context, arg1 *map[string]string, arg2 loader.ResolveSecretReference) (*loader.TargetKeyValueSettings, error) { +func (m *MockConfigurationSettingsRetriever) RefreshKeyValueSettings(arg0 context.Context, arg1 *map[string]string, arg2 loader.SecretReferenceResolver) (*loader.TargetKeyValueSettings, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "RefreshKeyValueSettings", arg0, arg1, arg2) ret0, _ := ret[0].(*loader.TargetKeyValueSettings) @@ -98,17 +99,17 @@ func (mr *MockConfigurationSettingsRetrieverMockRecorder) RefreshKeyValueSetting return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshKeyValueSettings", reflect.TypeOf((*MockConfigurationSettingsRetriever)(nil).RefreshKeyValueSettings), arg0, arg1, arg2) } -// ResolveKeyVaultReferences mocks base method. -func (m *MockConfigurationSettingsRetriever) ResolveKeyVaultReferences(arg0 context.Context, arg1 map[string]loader.KeyVaultSecretUriSegment, arg2 loader.ResolveSecretReference) (map[string][]byte, error) { +// ResolveSecretReferences mocks base method. +func (m *MockConfigurationSettingsRetriever) ResolveSecretReferences(arg0 context.Context, arg1 map[string]*loader.TargetSecretReference, arg2 loader.SecretReferenceResolver) (map[string]v10.Secret, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ResolveKeyVaultReferences", arg0, arg1, arg2) - ret0, _ := ret[0].(map[string][]byte) + ret := m.ctrl.Call(m, "ResolveSecretReferences", arg0, arg1, arg2) + ret0, _ := ret[0].(map[string]v10.Secret) ret1, _ := ret[1].(error) return ret0, ret1 } -// ResolveKeyVaultReferences indicates an expected call of ResolveKeyVaultReferences. -func (mr *MockConfigurationSettingsRetrieverMockRecorder) ResolveKeyVaultReferences(arg0, arg1, arg2 interface{}) *gomock.Call { +// ResolveSecretReferences indicates an expected call of ResolveSecretReferences. +func (mr *MockConfigurationSettingsRetrieverMockRecorder) ResolveSecretReferences(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveKeyVaultReferences", reflect.TypeOf((*MockConfigurationSettingsRetriever)(nil).ResolveKeyVaultReferences), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveSecretReferences", reflect.TypeOf((*MockConfigurationSettingsRetriever)(nil).ResolveSecretReferences), arg0, arg1, arg2) } diff --git a/internal/loader/typed_setting.go b/internal/loader/typed_setting.go index eab8ac9..c3d1b15 100644 --- a/internal/loader/typed_setting.go +++ b/internal/loader/typed_setting.go @@ -82,7 +82,7 @@ func createTypedSettings(rawSettings *RawSettings, dataOptions *acpv1.ConfigMapD func marshalProperties(settings map[string]string) string { stringBuilder := strings.Builder{} separator := "\n" - if settings != nil && len(settings) > 0 { + if settings != nil { i := 0 for k, v := range settings { stringBuilder.WriteString(fmt.Sprintf("%s=%s", k, v))