diff --git a/api/v1/azureappconfigurationprovider_types.go b/api/v1/azureappconfigurationprovider_types.go index eda9e80..26e63fd 100644 --- a/api/v1/azureappconfigurationprovider_types.go +++ b/api/v1/azureappconfigurationprovider_types.go @@ -35,7 +35,7 @@ type AzureAppConfigurationProviderSpec struct { Target ConfigurationGenerationParameters `json:"target"` Auth *AzureAppConfigurationProviderAuth `json:"auth,omitempty"` Configuration AzureAppConfigurationKeyValueOptions `json:"configuration,omitempty"` - Secret *AzureKeyVaultReference `json:"secret,omitempty"` + Secret *AzureSecretReference `json:"secret,omitempty"` FeatureFlag *AzureAppConfigurationFeatureFlagOptions `json:"featureFlag,omitempty"` } @@ -141,8 +141,8 @@ type ManagedIdentityReferenceParameters struct { Key string `json:"key"` } -// AzureKeyVaultReference defines the authentication type used to Azure KeyVault resolve KeyVaultReference -type AzureKeyVaultReference struct { +// AzureSecretReference defines the authentication type used to Azure KeyVault resolve KeyVaultReference +type AzureSecretReference struct { Target SecretGenerationParameters `json:"target"` Auth *AzureKeyVaultAuth `json:"auth,omitempty"` Refresh *RefreshSettings `json:"refresh,omitempty"` diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index d0ad60e..8149386 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -179,7 +179,7 @@ func (in *AzureAppConfigurationProviderSpec) DeepCopyInto(out *AzureAppConfigura in.Configuration.DeepCopyInto(&out.Configuration) if in.Secret != nil { in, out := &in.Secret, &out.Secret - *out = new(AzureKeyVaultReference) + *out = new(AzureSecretReference) (*in).DeepCopyInto(*out) } if in.FeatureFlag != nil { @@ -265,7 +265,7 @@ func (in *AzureKeyVaultPerVaultAuth) DeepCopy() *AzureKeyVaultPerVaultAuth { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AzureKeyVaultReference) DeepCopyInto(out *AzureKeyVaultReference) { +func (in *AzureSecretReference) DeepCopyInto(out *AzureSecretReference) { *out = *in out.Target = in.Target if in.Auth != nil { @@ -280,12 +280,12 @@ func (in *AzureKeyVaultReference) DeepCopyInto(out *AzureKeyVaultReference) { } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureKeyVaultReference. -func (in *AzureKeyVaultReference) DeepCopy() *AzureKeyVaultReference { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureSecretReference. +func (in *AzureSecretReference) DeepCopy() *AzureSecretReference { if in == nil { return nil } - out := new(AzureKeyVaultReference) + out := new(AzureSecretReference) in.DeepCopyInto(out) return out } diff --git a/config/crd/bases/azconfig.io_azureappconfigurationproviders.yaml b/config/crd/bases/azconfig.io_azureappconfigurationproviders.yaml index b16aec0..5fec3d3 100644 --- a/config/crd/bases/azconfig.io_azureappconfigurationproviders.yaml +++ b/config/crd/bases/azconfig.io_azureappconfigurationproviders.yaml @@ -162,7 +162,7 @@ spec: type: array type: object secret: - description: AzureKeyVaultReference defines the authentication type + description: AzureSecretReference defines the authentication type used to Azure KeyVault resolve KeyVaultReference properties: auth: diff --git a/go.mod b/go.mod index 8be4ce2..d6823ee 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.11.0 github.com/golang/mock v1.6.0 github.com/onsi/gomega v1.27.8 + golang.org/x/crypto v0.17.0 golang.org/x/sync v0.3.0 k8s.io/apimachinery v0.26.2 k8s.io/client-go v0.26.2 @@ -67,7 +68,6 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.5.0 // indirect @@ -90,4 +90,5 @@ require ( sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect + software.sslmate.com/src/go-pkcs12 v0.4.0 ) diff --git a/go.sum b/go.sum index 3b37f8d..4bc84b8 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,7 @@ github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= @@ -72,6 +73,7 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -122,8 +124,10 @@ github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -147,6 +151,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= @@ -363,3 +369,5 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kF sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= +software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= diff --git a/internal/controller/appconfigurationprovider_controller.go b/internal/controller/appconfigurationprovider_controller.go index 49d9b1e..5083a1e 100644 --- a/internal/controller/appconfigurationprovider_controller.go +++ b/internal/controller/appconfigurationprovider_controller.go @@ -50,25 +50,25 @@ type AzureAppConfigurationProviderReconciler struct { } type ReconciliationState struct { - Generation int64 - ConfigMapResourceVersion *string - SecretResourceVersion *string - SentinelETags map[acpv1.Sentinel]*azcore.ETag - NextSentinelBasedRefreshReconcileTime metav1.Time - NextKeyVaultReferenceRefreshReconcileTime metav1.Time - NextFeatureFlagRefreshReconcileTime metav1.Time - CachedSecretReferences map[string]loader.KeyVaultSecretUriSegment + Generation int64 + ConfigMapResourceVersion *string + SecretResourceVersions map[string]*string + SentinelETags map[acpv1.Sentinel]*azcore.ETag + ExistingSecretReferences map[string]*loader.TargetSecretReference + NextSentinelBasedRefreshReconcileTime metav1.Time + NextSecretReferenceRefreshReconcileTime metav1.Time + NextFeatureFlagRefreshReconcileTime metav1.Time } const ( - ProviderName string = "AzureAppConfigurationProvider" - LastReconcileTimeAnnotation string = "azconfig.io/LastReconcileTime" - KeyVaultReferenceContentType string = "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8" - FeatureFlagContentType string = "application/vnd.microsoft.appconfig.ff+json;charset=utf-8" - HeaderRetryAfter string = "Retry-After" - RequeueReconcileAfter time.Duration = time.Second * 30 - RetryAttempt int = 3 - DefaultRefreshInterval time.Duration = time.Second * 30 + ProviderName string = "AzureAppConfigurationProvider" + LastReconcileTimeAnnotation string = "azconfig.io/LastReconcileTime" + SecretReferenceContentType string = "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8" + FeatureFlagContentType string = "application/vnd.microsoft.appconfig.ff+json;charset=utf-8" + HeaderRetryAfter string = "Retry-After" + RequeueReconcileAfter time.Duration = time.Second * 30 + RetryAttempt int = 3 + DefaultRefreshInterval time.Duration = time.Second * 30 ) //Markers for teaching kubebuiler how generate the rabc manifests, see https://book.kubebuilder.io/reference/markers/rbac.html for detail @@ -129,37 +129,55 @@ func (reconciler *AzureAppConfigurationProviderReconciler) Reconcile(ctx context reconciler.logAndSetFailStatus(ctx, err, provider) return reconcile.Result{Requeue: false}, nil } - var existingConfigMap = corev1.ConfigMap{} + + existingConfigMap := corev1.ConfigMap{} err = reconciler.verifyTargetObjectExistence(ctx, provider, &existingConfigMap) if err != nil { reconciler.logAndSetFailStatus(ctx, err, provider) return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, nil } - var existingSecret = corev1.Secret{} + + existingSecrets := make(map[string]corev1.Secret) + existingSecret := corev1.Secret{} if provider.Spec.Secret != nil { err = reconciler.verifyTargetObjectExistence(ctx, provider, &existingSecret) if err != nil { reconciler.logAndSetFailStatus(ctx, err, provider) return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, nil } - } - - /* Initialize the ReconcileState for the provider*/ - if reconciler.ProvidersReconcileState[req.NamespacedName] == nil { + existingSecrets[provider.Spec.Secret.Target.SecretName] = existingSecret + } + + if reconciler.ProvidersReconcileState[req.NamespacedName] != nil { + for name := range reconciler.ProvidersReconcileState[req.NamespacedName].ExistingSecretReferences { + if _, ok := existingSecrets[name]; !ok { + existingSecret = corev1.Secret{} + err = reconciler.verifyTargetObjectExistence(ctx, provider, &existingSecret) + if err != nil { + reconciler.logAndSetFailStatus(ctx, err, provider) + return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, nil + } + existingSecrets[name] = existingSecret + } + } + } else { + // Initialize the ReconcileState for the provider reconciler.ProvidersReconcileState[req.NamespacedName] = &ReconciliationState{ Generation: -1, ConfigMapResourceVersion: nil, - SecretResourceVersion: nil, + SecretResourceVersions: nil, SentinelETags: make(map[acpv1.Sentinel]*azcore.ETag), - CachedSecretReferences: make(map[string]loader.KeyVaultSecretUriSegment), + ExistingSecretReferences: make(map[string]*loader.TargetSecretReference), } } + // Reset the resource version if the configmap or secret was unexpected deleted if existingConfigMap.Name == "" { reconciler.ProvidersReconcileState[req.NamespacedName].ConfigMapResourceVersion = nil } + if provider.Spec.Secret == nil || existingSecret.Name == "" { - reconciler.ProvidersReconcileState[req.NamespacedName].SecretResourceVersion = nil + reconciler.ProvidersReconcileState[req.NamespacedName].SecretResourceVersions = nil } /* Create ConfigurationSettingLoader to get the key-value settings from Azure AppConfiguration. */ @@ -177,7 +195,7 @@ func (reconciler *AzureAppConfigurationProviderReconciler) Reconcile(ctx context } /* Check if the reconciler should reconcile unconditionally under some situations, like the provider is updated, the configmap is deleted and so on. */ - shouldReconcile := reconciler.shouldReconcile(provider, &existingConfigMap, &existingSecret) + shouldReconcile := reconciler.shouldReconcile(provider, &existingConfigMap, existingSecrets) // Initialize the processor setting in this reconcile processor := &AppConfigurationProviderProcessor{ @@ -185,15 +203,14 @@ func (reconciler *AzureAppConfigurationProviderReconciler) Reconcile(ctx context Provider: provider, Retriever: &retriever, CurrentTime: metav1.Now(), - ReconciliationState: reconciler.ProvidersReconcileState, - NamespacedName: req.NamespacedName, + ReconciliationState: reconciler.ProvidersReconcileState[req.NamespacedName], ShouldReconcile: shouldReconcile, Settings: &loader.TargetKeyValueSettings{}, RefreshOptions: NewRefreshOptions(), ResolveSecretReference: nil, } - if err := processor.PopulateSettings(&existingConfigMap, &existingSecret); err != nil { + if err := processor.PopulateSettings(&existingConfigMap, existingSecrets); err != nil { return reconciler.requeueWhenGetSettingsFailed(ctx, provider, err) } @@ -206,7 +223,12 @@ func (reconciler *AzureAppConfigurationProviderReconciler) Reconcile(ctx context } /* Create secret when there are secret settings */ if processor.RefreshOptions.SecretSettingPopulated { - result, err := reconciler.createOrUpdateSecret(ctx, provider, processor.Settings) + result, err := reconciler.createOrUpdateSecrets(ctx, provider, processor.Settings) + if err != nil { + return result, nil + } + + result, err = reconciler.expelRemovedSecrets(ctx, provider, existingSecrets, processor.ReconciliationState.ExistingSecretReferences) if err != nil { return result, nil } @@ -217,7 +239,10 @@ func (reconciler *AzureAppConfigurationProviderReconciler) Reconcile(ctx context return processor.Finish() } -func (reconciler *AzureAppConfigurationProviderReconciler) verifyTargetObjectExistence(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, obj client.Object) error { +func (reconciler *AzureAppConfigurationProviderReconciler) verifyTargetObjectExistence( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + obj client.Object) error { // Get and verify the existing configMap or secret, if there's existing configMap/secret which is not owned by current provider, throw error var targetName string if _, ok := obj.(*corev1.ConfigMap); ok { @@ -239,7 +264,10 @@ func (reconciler *AzureAppConfigurationProviderReconciler) verifyTargetObjectExi return verifyExistingTargetObject(obj, targetName, provider.Name) } -func (reconciler *AzureAppConfigurationProviderReconciler) logAndSetFailStatus(ctx context.Context, err error, provider *acpv1.AzureAppConfigurationProvider) { +func (reconciler *AzureAppConfigurationProviderReconciler) logAndSetFailStatus( + ctx context.Context, + err error, + provider *acpv1.AzureAppConfigurationProvider) { var showErrorAsWarning bool = false namespacedName := types.NamespacedName{ Name: provider.Name, @@ -252,7 +280,7 @@ func (reconciler *AzureAppConfigurationProviderReconciler) logAndSetFailStatus(c } else if reconcileState != nil && reconcileState.ConfigMapResourceVersion != nil && (provider.Spec.Secret == nil || - reconcileState.SecretResourceVersion != nil) { + reconcileState.SecretResourceVersions != nil) { // If the target ConfigMap or Secret does exists, just show error as warning. showErrorAsWarning = true } @@ -266,7 +294,10 @@ func (reconciler *AzureAppConfigurationProviderReconciler) logAndSetFailStatus(c } } -func (reconciler *AzureAppConfigurationProviderReconciler) requeueWhenGetSettingsFailed(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, err error) (ctrl.Result, error) { +func (reconciler *AzureAppConfigurationProviderReconciler) requeueWhenGetSettingsFailed( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + err error) (ctrl.Result, error) { requeueAfter := RequeueReconcileAfter reconciler.logAndSetFailStatus(ctx, err, provider) if errors.Is(err, &loader.ArgumentError{}) { @@ -285,7 +316,10 @@ func (reconciler *AzureAppConfigurationProviderReconciler) requeueWhenGetSetting return reconcile.Result{Requeue: true, RequeueAfter: requeueAfter}, nil } -func (reconciler *AzureAppConfigurationProviderReconciler) createOrUpdateConfigMap(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, settings *loader.TargetKeyValueSettings) (reconcile.Result, error) { +func (reconciler *AzureAppConfigurationProviderReconciler) createOrUpdateConfigMap( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + settings *loader.TargetKeyValueSettings) (reconcile.Result, error) { configMapObj := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: provider.Spec.Target.ConfigMapName, @@ -327,49 +361,88 @@ func (reconciler *AzureAppConfigurationProviderReconciler) createOrUpdateConfigM return reconcile.Result{}, nil } -func (reconciler *AzureAppConfigurationProviderReconciler) createOrUpdateSecret(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, settings *loader.TargetKeyValueSettings) (reconcile.Result, error) { - secretObjName := provider.Spec.Secret.Target.SecretName - secretObj := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretObjName, - Namespace: provider.Namespace, - }, - Type: corev1.SecretTypeOpaque, - } - // Important: set the ownership of secret - if err := controllerutil.SetControllerReference(provider, secretObj, reconciler.Scheme); err != nil { - reconciler.logAndSetFailStatus(ctx, err, provider) - return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, err +func (reconciler *AzureAppConfigurationProviderReconciler) createOrUpdateSecrets( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + settings *loader.TargetKeyValueSettings) (reconcile.Result, error) { + if len(settings.SecretSettings) == 0 { + klog.V(3).Info("No secret settings are fetched from Azure AppConfiguration") } + if provider.Annotations == nil { provider.Annotations = make(map[string]string) } - provider.Annotations[LastReconcileTimeAnnotation] = metav1.Now().UTC().String() - if len(settings.SecretSettings) == 0 { - klog.V(3).Info("No secret settings are fetched from Azure AppConfiguration") - } - operationResult, err := ctrl.CreateOrUpdate(ctx, reconciler.Client, secretObj, func() error { - secretObj.Data = settings.SecretSettings - secretObj.Labels = provider.Labels - secretObj.Annotations = provider.Annotations - return nil - }) - if err != nil { - reconciler.logAndSetFailStatus(ctx, err, provider) - return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, err + for secretName, secret := range settings.SecretSettings { + secretObj := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: provider.Namespace, + }, + Type: secret.Type, + } + // Important: set the ownership of secret + if err := controllerutil.SetControllerReference(provider, secretObj, reconciler.Scheme); err != nil { + reconciler.logAndSetFailStatus(ctx, err, provider) + return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, err + } + + provider.Annotations[LastReconcileTimeAnnotation] = metav1.Now().UTC().String() + operationResult, err := ctrl.CreateOrUpdate(ctx, reconciler.Client, secretObj, func() error { + secretObj.Data = secret.Data + secretObj.Labels = provider.Labels + secretObj.Annotations = provider.Annotations + + return nil + }) + if err != nil { + reconciler.logAndSetFailStatus(ctx, err, provider) + return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, err + } + + namespacedName := types.NamespacedName{ + Name: provider.Name, + Namespace: provider.Namespace, + } + + if reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersions == nil { + reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersions = make(map[string]*string) + } + reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersions[secretObj.Name] = &secretObj.ResourceVersion + klog.V(5).Infof("Secret %q in %q namespace is %s", secretObj.Name, secretObj.Namespace, string(operationResult)) } - namespacedName := types.NamespacedName{ - Name: provider.Name, - Namespace: provider.Namespace, + + return reconcile.Result{}, nil +} + +func (reconciler *AzureAppConfigurationProviderReconciler) expelRemovedSecrets( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + existingSecrets map[string]corev1.Secret, + secretReferences map[string]*loader.TargetSecretReference) (reconcile.Result, error) { + for name := range existingSecrets { + if _, ok := secretReferences[name]; !ok { + err := reconciler.Client.Delete(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: provider.Namespace, + }, + }) + if err != nil { + reconciler.logAndSetFailStatus(ctx, err, provider) + return reconcile.Result{Requeue: true, RequeueAfter: RequeueReconcileAfter}, err + } + } } - reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersion = &secretObj.ResourceVersion - klog.V(5).Infof("Secret %q in %q namespace is %s", secretObj.Name, secretObj.Namespace, string(operationResult)) return reconcile.Result{}, nil } -func newProviderStatus(phase acpv1.AppConfigurationSyncPhase, message string, syncTime metav1.Time, refreshStatus acpv1.RefreshStatus) acpv1.AzureAppConfigurationProviderStatus { +func newProviderStatus( + phase acpv1.AppConfigurationSyncPhase, + message string, + syncTime metav1.Time, + refreshStatus acpv1.RefreshStatus) acpv1.AzureAppConfigurationProviderStatus { return acpv1.AzureAppConfigurationProviderStatus{ Message: message, Phase: phase, @@ -379,28 +452,42 @@ func newProviderStatus(phase acpv1.AppConfigurationSyncPhase, message string, sy } } -func (reconciler *AzureAppConfigurationProviderReconciler) shouldReconcile(provider *acpv1.AzureAppConfigurationProvider, existingConfigMap *corev1.ConfigMap, existingSecret *corev1.Secret) bool { +func (reconciler *AzureAppConfigurationProviderReconciler) shouldReconcile( + provider *acpv1.AzureAppConfigurationProvider, + existingConfigMap *corev1.ConfigMap, + existingSecrets map[string]corev1.Secret) bool { // Get the name and namespace of the provider namespacedName := types.NamespacedName{ Name: provider.Name, Namespace: provider.Namespace, } + if provider.Generation != reconciler.ProvidersReconcileState[namespacedName].Generation { // If the provider is updated, we need to reconcile anyway return true } + if reconciler.ProvidersReconcileState[namespacedName].ConfigMapResourceVersion == nil || *reconciler.ProvidersReconcileState[namespacedName].ConfigMapResourceVersion != existingConfigMap.ResourceVersion { // If the ConfigMap is removed or updated, we need to reconcile anyway return true } - if provider.Spec.Secret != nil && - (reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersion == nil || - *reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersion != existingSecret.ResourceVersion) { - // If the Secret is removed or updated, we need to reconcile anyway + + if provider.Spec.Secret == nil { + return false + } + + if reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersions == nil { return true } + for name, secret := range existingSecrets { + if reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersions[name] == nil || + *reconciler.ProvidersReconcileState[namespacedName].SecretResourceVersions[name] != secret.ResourceVersion { + return true + } + } + return false } diff --git a/internal/controller/appconfigurationprovider_controller_test.go b/internal/controller/appconfigurationprovider_controller_test.go index 3b623a6..8ddd700 100644 --- a/internal/controller/appconfigurationprovider_controller_test.go +++ b/internal/controller/appconfigurationprovider_controller_test.go @@ -146,13 +146,38 @@ var _ = Describe("AppConfiguationProvider controller", func() { It("Should create new secret", func() { By("By getting multiple secret reference settings from AppConfig") - mapResult := make(map[string][]byte) - mapResult["testSecretKey"] = []byte("testValue") - mapResult["testSecretKey2"] = []byte("testValue2") - mapResult["testSecretKey3"] = []byte("testValue3") + secretResult1 := make(map[string][]byte) + secretResult1["tls.crt"] = []byte("fakeCrt") + secretResult1["tls.key"] = []byte("fakeKey") + secretResult2 := make(map[string][]byte) + secretResult2["testSecretKey"] = []byte("testSecretValue") + secretResult2["testSecretKey2"] = []byte("testSecretValue2") + secretResult2["testSecretKey3"] = []byte("testSecretValue3") + + secretName := "secret-to-be-created-3" + secretName2 := "secret-to-be-created-3-1" allSettings := &loader.TargetKeyValueSettings{ - SecretSettings: mapResult, + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult1, + Type: corev1.SecretTypeTLS, + }, + secretName2: { + Data: secretResult2, + Type: corev1.SecretTypeOpaque, + }, + }, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: make(map[string]loader.KeyVaultSecretUriSegment), + }, + secretName2: { + Type: corev1.SecretTypeOpaque, + UriSegments: make(map[string]loader.KeyVaultSecretUriSegment), + }, + }, } mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) @@ -160,7 +185,6 @@ var _ = Describe("AppConfiguationProvider controller", func() { ctx := context.Background() providerName := "test-appconfigurationprovider-3" configMapName := "configmap-to-be-created-3" - secretName := "secret-to-be-created-3" configProvider := &acpv1.AzureAppConfigurationProvider{ TypeMeta: metav1.TypeMeta{ APIVersion: "appconfig.kubernetes.config/v1", @@ -175,7 +199,7 @@ var _ = Describe("AppConfiguationProvider controller", func() { Target: acpv1.ConfigurationGenerationParameters{ ConfigMapName: configMapName, }, - Secret: &acpv1.AzureKeyVaultReference{ + Secret: &acpv1.AzureSecretReference{ Target: acpv1.SecretGenerationParameters{ SecretName: secretName, }, @@ -184,7 +208,9 @@ var _ = Describe("AppConfiguationProvider controller", func() { } Expect(k8sClient.Create(ctx, configProvider)).Should(Succeed()) secretLookupKey := types.NamespacedName{Name: secretName, Namespace: ProviderNamespace} + secretLookupKey2 := types.NamespacedName{Name: secretName2, Namespace: ProviderNamespace} secret := &corev1.Secret{} + secret2 := &corev1.Secret{} Eventually(func() bool { err := k8sClient.Get(ctx, secretLookupKey, secret) @@ -192,13 +218,26 @@ var _ = Describe("AppConfiguationProvider controller", func() { fmt.Print(err.Error()) } return err == nil - }, time.Second*20, interval).Should(BeTrue()) + }, time.Second*5, interval).Should(BeTrue()) + + Eventually(func() bool { + err := k8sClient.Get(ctx, secretLookupKey2, secret2) + if err != nil { + fmt.Print(err.Error()) + } + return err == nil + }, time.Second*5, interval).Should(BeTrue()) Expect(secret.Namespace).Should(Equal(ProviderNamespace)) - Expect(string(secret.Data["testSecretKey"])).Should(Equal("testValue")) - Expect(string(secret.Data["testSecretKey2"])).Should(Equal("testValue2")) - Expect(string(secret.Data["testSecretKey3"])).Should(Equal("testValue3")) - Expect(secret.Type).Should(Equal(corev1.SecretType("Opaque"))) + Expect(string(secret.Data["tls.crt"])).Should(Equal("fakeCrt")) + Expect(string(secret.Data["tls.key"])).Should(Equal("fakeKey")) + Expect(secret.Type).Should(Equal(corev1.SecretTypeTLS)) + + Expect(secret2.Namespace).Should(Equal(ProviderNamespace)) + Expect(string(secret2.Data["testSecretKey"])).Should(Equal("testSecretValue")) + Expect(string(secret2.Data["testSecretKey2"])).Should(Equal("testSecretValue2")) + Expect(string(secret2.Data["testSecretKey3"])).Should(Equal("testSecretValue3")) + Expect(secret2.Type).Should(Equal(corev1.SecretTypeOpaque)) }) It("Should create proper configmap and secret", func() { @@ -213,9 +252,21 @@ var _ = Describe("AppConfiguationProvider controller", func() { secretResult["testSecretKey2"] = []byte("testSecretValue2") secretResult["testSecretKey3"] = []byte("testSecretValue3") + secretName := "secret-to-be-created-5" allSettings := &loader.TargetKeyValueSettings{ - SecretSettings: secretResult, + SecretSettings: map[string]corev1.Secret{ + secretName: { + Data: secretResult, + Type: corev1.SecretType("Opaque"), + }, + }, ConfigMapSettings: configMapResult, + SecretReferences: map[string]*loader.TargetSecretReference{ + secretName: { + Type: corev1.SecretType("Opaque"), + UriSegments: make(map[string]loader.KeyVaultSecretUriSegment), + }, + }, } mockConfigurationSettings.EXPECT().CreateTargetSettings(gomock.Any(), gomock.Any()).Return(allSettings, nil) @@ -223,7 +274,7 @@ var _ = Describe("AppConfiguationProvider controller", func() { ctx := context.Background() providerName := "test-appconfigurationprovider-5" configMapName := "configmap-to-be-created-5" - secretName := "secret-to-be-created-5" + configProvider := &acpv1.AzureAppConfigurationProvider{ TypeMeta: metav1.TypeMeta{ APIVersion: "appconfig.kubernetes.config/v1", @@ -238,7 +289,7 @@ var _ = Describe("AppConfiguationProvider controller", func() { Target: acpv1.ConfigurationGenerationParameters{ ConfigMapName: configMapName, }, - Secret: &acpv1.AzureKeyVaultReference{ + Secret: &acpv1.AzureSecretReference{ Target: acpv1.SecretGenerationParameters{ SecretName: secretName, }, diff --git a/internal/controller/processor.go b/internal/controller/processor.go index f7b3c8b..1ce58fd 100644 --- a/internal/controller/processor.go +++ b/internal/controller/processor.go @@ -13,7 +13,6 @@ import ( "golang.org/x/exp/maps" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -25,26 +24,25 @@ type AppConfigurationProviderProcessor struct { Provider *acpv1.AzureAppConfigurationProvider Settings *loader.TargetKeyValueSettings ShouldReconcile bool - ReconciliationState map[types.NamespacedName]*ReconciliationState + ReconciliationState *ReconciliationState CurrentTime metav1.Time - NamespacedName types.NamespacedName RefreshOptions *RefreshOptions - ResolveSecretReference loader.ResolveSecretReference + ResolveSecretReference loader.SecretReferenceResolver } type RefreshOptions struct { - sentinelBasedRefreshEnabled bool - sentinelChanged bool - keyVaultRefreshEnabled bool - keyVaultRefreshNeeded bool - updatedSentinelETags map[acpv1.Sentinel]*azcore.ETag - featureFlagRefreshEnabled bool - featureFlagRefreshNeeded bool - ConfigMapSettingPopulated bool - SecretSettingPopulated bool + sentinelBasedRefreshEnabled bool + sentinelChanged bool + secretReferenceRefreshEnabled bool + secretReferenceRefreshNeeded bool + featureFlagRefreshEnabled bool + featureFlagRefreshNeeded bool + ConfigMapSettingPopulated bool + SecretSettingPopulated bool + updatedSentinelETags map[acpv1.Sentinel]*azcore.ETag } -func (processor *AppConfigurationProviderProcessor) PopulateSettings(existingConfigMap *corev1.ConfigMap, existingSecret *corev1.Secret) error { +func (processor *AppConfigurationProviderProcessor) PopulateSettings(existingConfigMap *corev1.ConfigMap, existingSecrets map[string]corev1.Secret) error { if err := processor.ProcessFullReconciliation(); err != nil { return err } @@ -57,7 +55,7 @@ func (processor *AppConfigurationProviderProcessor) PopulateSettings(existingCon return err } - if err := processor.ProcessKeyVaultReferenceRefresh(existingSecret); err != nil { + if err := processor.ProcessSecretReferenceRefresh(existingSecrets); err != nil { return err } @@ -71,7 +69,7 @@ func (processor *AppConfigurationProviderProcessor) ProcessFullReconciliation() return err } processor.Settings = updatedSettings - processor.ReconciliationState[processor.NamespacedName].CachedSecretReferences = updatedSettings.KeyVaultReferencesToCache + processor.ReconciliationState.ExistingSecretReferences = updatedSettings.SecretReferences processor.RefreshOptions.ConfigMapSettingPopulated = true if processor.Provider.Spec.Secret != nil { processor.RefreshOptions.SecretSettingPopulated = true @@ -83,11 +81,12 @@ func (processor *AppConfigurationProviderProcessor) ProcessFullReconciliation() func (processor *AppConfigurationProviderProcessor) ProcessFeatureFlagRefresh(existingConfigMap *corev1.ConfigMap) error { provider := *processor.Provider - reconcileState := processor.ReconciliationState[processor.NamespacedName] - currentTime := processor.CurrentTime + reconcileState := processor.ReconciliationState var err error // Check if the feature flag dynamic feature if enabled - if provider.Spec.FeatureFlag != nil && provider.Spec.FeatureFlag.Refresh != nil && provider.Spec.FeatureFlag.Refresh.Enabled { + if provider.Spec.FeatureFlag != nil && + provider.Spec.FeatureFlag.Refresh != nil && + provider.Spec.FeatureFlag.Refresh.Enabled { processor.RefreshOptions.featureFlagRefreshEnabled = true } else { reconcileState.NextFeatureFlagRefreshReconcileTime = metav1.Time{} @@ -95,13 +94,13 @@ func (processor *AppConfigurationProviderProcessor) ProcessFeatureFlagRefresh(ex } refreshInterval, _ := time.ParseDuration(provider.Spec.FeatureFlag.Refresh.Interval) - nextFeatureFlagRefreshReconcileTime := metav1.Time{Time: currentTime.Add(refreshInterval)} + nextFeatureFlagRefreshReconcileTime := metav1.Time{Time: processor.CurrentTime.Add(refreshInterval)} if processor.ShouldReconcile { reconcileState.NextFeatureFlagRefreshReconcileTime = nextFeatureFlagRefreshReconcileTime return nil } - if !currentTime.After(reconcileState.NextFeatureFlagRefreshReconcileTime.Time) { + if !processor.CurrentTime.After(reconcileState.NextFeatureFlagRefreshReconcileTime.Time) { return nil } @@ -119,12 +118,12 @@ func (processor *AppConfigurationProviderProcessor) ProcessFeatureFlagRefresh(ex } func (processor *AppConfigurationProviderProcessor) ProcessKeyValueRefresh(existingConfigMap *corev1.ConfigMap) error { - provider := *processor.Provider - reconcileState := processor.ReconciliationState[processor.NamespacedName] - currentTime := processor.CurrentTime + provider := processor.Provider + reconcileState := processor.ReconciliationState var err error // Check if the sentinel based refresh is enabled - if provider.Spec.Configuration.Refresh != nil && provider.Spec.Configuration.Refresh.Enabled { + if provider.Spec.Configuration.Refresh != nil && + provider.Spec.Configuration.Refresh.Enabled { processor.RefreshOptions.sentinelBasedRefreshEnabled = true } else { reconcileState.NextSentinelBasedRefreshReconcileTime = metav1.Time{} @@ -132,13 +131,13 @@ func (processor *AppConfigurationProviderProcessor) ProcessKeyValueRefresh(exist } refreshInterval, _ := time.ParseDuration(provider.Spec.Configuration.Refresh.Interval) - nextSentinelBasedRefreshReconcileTime := metav1.Time{Time: currentTime.Add(refreshInterval)} + nextSentinelBasedRefreshReconcileTime := metav1.Time{Time: processor.CurrentTime.Add(refreshInterval)} if processor.ShouldReconcile { reconcileState.NextSentinelBasedRefreshReconcileTime = nextSentinelBasedRefreshReconcileTime return nil } - if !currentTime.After(reconcileState.NextSentinelBasedRefreshReconcileTime.Time) { + if !processor.CurrentTime.After(reconcileState.NextSentinelBasedRefreshReconcileTime.Time) { return nil } @@ -161,7 +160,7 @@ func (processor *AppConfigurationProviderProcessor) ProcessKeyValueRefresh(exist } processor.Settings = keyValueRefreshedSettings - reconcileState.CachedSecretReferences = keyValueRefreshedSettings.KeyVaultReferencesToCache + reconcileState.ExistingSecretReferences = keyValueRefreshedSettings.SecretReferences processor.RefreshOptions.ConfigMapSettingPopulated = true if processor.Provider.Spec.Secret != nil { processor.RefreshOptions.SecretSettingPopulated = true @@ -172,62 +171,71 @@ func (processor *AppConfigurationProviderProcessor) ProcessKeyValueRefresh(exist return nil } -func (processor *AppConfigurationProviderProcessor) ProcessKeyVaultReferenceRefresh(existingSecret *corev1.Secret) error { - provider := *processor.Provider - reconcileState := processor.ReconciliationState[processor.NamespacedName] - currentTime := processor.CurrentTime +func (processor *AppConfigurationProviderProcessor) ProcessSecretReferenceRefresh(existingSecrets map[string]corev1.Secret) error { + provider := processor.Provider + reconcileState := processor.ReconciliationState // Check if the key vault dynamic feature if enabled - if provider.Spec.Secret != nil && provider.Spec.Secret.Refresh != nil && provider.Spec.Secret.Refresh.Enabled { - processor.RefreshOptions.keyVaultRefreshEnabled = true + if provider.Spec.Secret != nil && + provider.Spec.Secret.Refresh != nil && + provider.Spec.Secret.Refresh.Enabled { + processor.RefreshOptions.secretReferenceRefreshEnabled = true } - if !processor.RefreshOptions.keyVaultRefreshEnabled { - reconcileState.NextKeyVaultReferenceRefreshReconcileTime = metav1.Time{} - reconcileState.CachedSecretReferences = make(map[string]loader.KeyVaultSecretUriSegment) + if !processor.RefreshOptions.secretReferenceRefreshEnabled { + reconcileState.NextSecretReferenceRefreshReconcileTime = metav1.Time{} return nil } - if !currentTime.After(reconcileState.NextKeyVaultReferenceRefreshReconcileTime.Time) { + if !processor.CurrentTime.After(reconcileState.NextSecretReferenceRefreshReconcileTime.Time) { return nil } - processor.RefreshOptions.keyVaultRefreshNeeded = true + processor.RefreshOptions.secretReferenceRefreshNeeded = true keyVaultRefreshInterval, _ := time.ParseDuration(provider.Spec.Secret.Refresh.Interval) - nextKeyVaultReferenceRefreshReconcileTime := metav1.Time{Time: currentTime.Add(keyVaultRefreshInterval)} + nextSecretReferenceRefreshReconcileTime := metav1.Time{Time: processor.CurrentTime.Add(keyVaultRefreshInterval)} // When SecretSettingPopulated means ProcessFullReconciliation or ProcessKeyValueRefresh has executed, update next refresh time and return if processor.RefreshOptions.SecretSettingPopulated { - reconcileState.NextKeyVaultReferenceRefreshReconcileTime = nextKeyVaultReferenceRefreshReconcileTime + reconcileState.NextSecretReferenceRefreshReconcileTime = nextSecretReferenceRefreshReconcileTime return nil } - resolvedSecretData, err := (*processor.Retriever).ResolveKeyVaultReferences(processor.Context, reconcileState.CachedSecretReferences, processor.ResolveSecretReference) + resolvedSecretData, err := (*processor.Retriever).ResolveSecretReferences(processor.Context, true, reconcileState.ExistingSecretReferences, processor.ResolveSecretReference) if err != nil { return err } - maps.Copy(existingSecret.Data, resolvedSecretData) - processor.Settings.SecretSettings = existingSecret.Data + + for secretName, targetSecret := range resolvedSecretData { + existingSecret, ok := existingSecrets[secretName] + if ok { + maps.Copy(existingSecret.Data, targetSecret.Data) + } + + } + processor.Settings.SecretSettings = existingSecrets processor.RefreshOptions.SecretSettingPopulated = true // Update next refresh time only if settings updated successfully - reconcileState.NextKeyVaultReferenceRefreshReconcileTime = nextKeyVaultReferenceRefreshReconcileTime + reconcileState.NextSecretReferenceRefreshReconcileTime = nextSecretReferenceRefreshReconcileTime return nil } func (processor *AppConfigurationProviderProcessor) Finish() (ctrl.Result, error) { - processor.ReconciliationState[processor.NamespacedName].Generation = processor.Provider.Generation - if !processor.RefreshOptions.keyVaultRefreshEnabled && !processor.RefreshOptions.sentinelBasedRefreshEnabled && !processor.RefreshOptions.featureFlagRefreshEnabled { + processor.ReconciliationState.Generation = processor.Provider.Generation + if !processor.RefreshOptions.secretReferenceRefreshEnabled && + !processor.RefreshOptions.sentinelBasedRefreshEnabled && + !processor.RefreshOptions.featureFlagRefreshEnabled { // Do nothing, just complete the reconcile klog.V(1).Infof("Complete reconcile AzureAppConfigurationProvider %q in %q namespace", processor.Provider.Name, processor.Provider.Namespace) return reconcile.Result{}, nil } else { // Update the sentinel ETags and last sentinel refresh time if processor.RefreshOptions.sentinelChanged { - processor.ReconciliationState[processor.NamespacedName].SentinelETags = processor.RefreshOptions.updatedSentinelETags + processor.ReconciliationState.SentinelETags = processor.RefreshOptions.updatedSentinelETags processor.Provider.Status.RefreshStatus.LastSentinelBasedRefreshTime = processor.CurrentTime } // Update provider last key vault refresh time - if processor.RefreshOptions.keyVaultRefreshNeeded { + if processor.RefreshOptions.secretReferenceRefreshNeeded { processor.Provider.Status.RefreshStatus.LastKeyVaultReferenceRefreshTime = processor.CurrentTime } // Update provider last feature flag refresh time @@ -244,22 +252,22 @@ func (processor *AppConfigurationProviderProcessor) Finish() (ctrl.Result, error func NewRefreshOptions() *RefreshOptions { return &RefreshOptions{ - sentinelBasedRefreshEnabled: false, - sentinelChanged: false, - keyVaultRefreshEnabled: false, - keyVaultRefreshNeeded: false, - updatedSentinelETags: make(map[acpv1.Sentinel]*azcore.ETag), - featureFlagRefreshEnabled: false, - featureFlagRefreshNeeded: false, - ConfigMapSettingPopulated: false, - SecretSettingPopulated: false, + sentinelBasedRefreshEnabled: false, + sentinelChanged: false, + secretReferenceRefreshEnabled: false, + secretReferenceRefreshNeeded: false, + featureFlagRefreshEnabled: false, + featureFlagRefreshNeeded: false, + ConfigMapSettingPopulated: false, + SecretSettingPopulated: false, + updatedSentinelETags: make(map[acpv1.Sentinel]*azcore.ETag), } } func (processor *AppConfigurationProviderProcessor) calculateRequeueAfterInterval() time.Duration { - reconcileState := processor.ReconciliationState[processor.NamespacedName] + reconcileState := processor.ReconciliationState nextRefreshTimeList := []metav1.Time{reconcileState.NextSentinelBasedRefreshReconcileTime, - reconcileState.NextKeyVaultReferenceRefreshReconcileTime, reconcileState.NextFeatureFlagRefreshReconcileTime} + reconcileState.NextSecretReferenceRefreshReconcileTime, reconcileState.NextFeatureFlagRefreshReconcileTime} var nextRequeueTime metav1.Time for _, time := range nextRefreshTimeList { diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 2745370..e0a894c 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -227,7 +227,7 @@ func verifyRefreshInterval(interval string, allowedMinimalRefreshInterval time.D refreshInterval, err := time.ParseDuration(interval) if err == nil { if refreshInterval < allowedMinimalRefreshInterval { - return loader.NewArgumentError(refreshArgument, fmt.Errorf("%s can not be shorter than %s.", refreshArgument, allowedMinimalRefreshInterval.String())) + return loader.NewArgumentError(refreshArgument, fmt.Errorf("%s can not be shorter than %s", refreshArgument, allowedMinimalRefreshInterval.String())) } } else { return loader.NewArgumentError(refreshArgument, err) diff --git a/internal/loader/configuraiton_setting_loader_test.go b/internal/loader/configuraiton_setting_loader_test.go index 8f0d79c..4a95db1 100644 --- a/internal/loader/configuraiton_setting_loader_test.go +++ b/internal/loader/configuraiton_setting_loader_test.go @@ -6,14 +6,27 @@ package loader import ( acpv1 "azappconfig/provider/api/v1" "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/pem" "errors" + "fmt" + "math/big" "reflect" "testing" + "time" "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" + "github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets" "github.com/golang/mock/gomock" + "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + pkcs12 "software.sslmate.com/src/go-pkcs12" . "github.com/onsi/gomega" "github.com/stretchr/testify/assert" @@ -70,8 +83,8 @@ func newCommonKeyValueSettings(key string, value string, label string) azappconf } func newKeyVaultSettings(key string, label string) azappconfig.Setting { - vault := "{ \"test\":\"https://fake-vault\"}" - keyVaultContentType := KeyVaultReferenceContentType + vault := "{ \"uri\":\"https://fake-vault/secrets/fakesecret\"}" + keyVaultContentType := SecretReferenceContentType return azappconfig.Setting{ Key: &key, @@ -104,10 +117,10 @@ func (m *MockResolveSecretReference) EXPECT() *MockResolveSecretReferenceMockRec } // Resolve mocks base method. -func (m *MockResolveSecretReference) Resolve(arg0 KeyVaultSecretUriSegment, arg1 context.Context) (*string, error) { +func (m *MockResolveSecretReference) Resolve(arg0 KeyVaultSecretUriSegment, arg1 context.Context) (azsecrets.GetSecretResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Resolve", arg0, arg1) - ret0, _ := ret[0].(*string) + ret0, _ := ret[0].(azsecrets.GetSecretResponse) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -124,6 +137,12 @@ const ( ConfigMapName = "configmap-to-be-created" ) +func TestLoaderAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Test loader APIs") +} + var _ = BeforeEach(func() { mockCtrl = gomock.NewController(GinkgoT()) mockResolveSecretReference = NewMockResolveSecretReference(mockCtrl) @@ -152,6 +171,7 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Context("When get Key Vault Reference Type Settings", func() { It("Should put into Secret settings collection", func() { By("By resolving the settings from Azure Key Vault") + managedIdentity := uuid.New().String() testSpec := acpv1.AzureAppConfigurationProviderSpec{ Endpoint: &EndpointName, Target: acpv1.ConfigurationGenerationParameters{ @@ -160,6 +180,16 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ TrimKeyPrefixes: []string{"app:"}, }, + Secret: &acpv1.AzureSecretReference{ + Target: acpv1.SecretGenerationParameters{ + SecretName: "targetSecret", + }, + Auth: &acpv1.AzureKeyVaultAuth{ + AzureAppConfigurationProviderAuth: &acpv1.AzureAppConfigurationProviderAuth{ + ManagedIdentityClientId: &managedIdentity, + }, + }, + }, } testProvider := acpv1.AzureAppConfigurationProvider{ TypeMeta: metav1.TypeMeta{ @@ -175,18 +205,21 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) secretValue := "fakeSecretValue" - secretValue2 := "fakeSecretValue2" - mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Times(0).Return(&secretValue, nil) - mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Times(0).Return(&secretValue2, nil) + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) + Expect(err).Should(BeNil()) Expect(len(allSettings.ConfigMapSettings)).Should(Equal(2)) - Expect(len(allSettings.SecretSettings)).Should(Equal(2)) + Expect(len(allSettings.SecretSettings)).Should(Equal(1)) Expect(allSettings.ConfigMapSettings["someKey1"]).Should(Equal("value1")) - Expect(allSettings.ConfigMapSettings["someSubKey1;1"]).Should(Equal("value2")) - Expect(allSettings.SecretSettings["secret:1"]).Should(Equal(secretValue)) - Expect(allSettings.SecretSettings["someSecret"]).Should(Equal(secretValue2)) - Expect(err).Should(BeNil()) + Expect(allSettings.ConfigMapSettings["someSubKey1:1"]).Should(Equal("value4")) + Expect(string(allSettings.SecretSettings["targetSecret"].Data["secret:1"])).Should(Equal(secretValue)) }) It("Should throw exception", func() { @@ -214,13 +247,399 @@ var _ = Describe("AppConfiguationProvider Get All Settings", func() { configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) - secretValue := "fakeSecretValue" - mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Times(0).Return(&secretValue, nil) - mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Times(0).Return(nil, errors.New("Some error")) allSettings, err := configurationProvider.CreateTargetSettings(context.Background(), mockResolveSecretReference) Expect(allSettings).Should(BeNil()) - Expect(err.Error()).Should(Equal("Some error")) + Expect(err.Error()).Should(Equal("A Key Vault reference is found in App Configuration, but 'spec.secret' was not configured in the Azure App Configuration provider 'testName' in namespace 'testNamespace'")) + }) + + It("Should throw unknown content type error", func() { + By("By getting unknown cert type from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue := "fakeSecretValue" + secretName := "targetSecret" + contentType := "fake-content-type" + kidStr := "fakeKid" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + Kid: &kidStr, + ContentType: &contentType, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + _, err := configurationProvider.ResolveSecretReferences(context.Background(), false, secretReferencesToResolve, mockResolveSecretReference) + + Expect(err.Error()).Should(Equal("fail to decode the cert 'targetSecret': failed to get certificate, unknown content type 'fake-content-type'")) + }) + + It("Should throw decode pem block error", func() { + By("By getting unexpected secret value of pem cert from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue := "fakeSecretValue" + secretName := "targetSecret" + contentType := CertTypePem + kidStr := "fakeKid" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + Kid: &kidStr, + ContentType: &contentType, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + _, err := configurationProvider.ResolveSecretReferences(context.Background(), false, secretReferencesToResolve, mockResolveSecretReference) + + Expect(err.Error()).Should(Equal("fail to decode the cert 'targetSecret': failed to decode pem block")) + }) + + It("Should throw decode pfx error", func() { + By("By getting unexpected secret value of pfx cert from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue := "fakeSecretValue" + secretName := "targetSecret" + contentType := CertTypePfx + kidStr := "fakeKid" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + Kid: &kidStr, + ContentType: &contentType, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + _, err := configurationProvider.ResolveSecretReferences(context.Background(), false, secretReferencesToResolve, mockResolveSecretReference) + + Expect(err.Error()).Should(Equal("fail to decode the cert 'targetSecret': illegal base64 data at input byte 12")) + }) + + It("Succeeded to get tls type secret", func() { + By("By getting valid pfx cert from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue, _ := createFakePfx() + secretName := "targetSecret" + contentType := CertTypePfx + kidStr := "fakeKid" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + Kid: &kidStr, + ContentType: &contentType, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + secrets, err := configurationProvider.ResolveSecretReferences(context.Background(), false, secretReferencesToResolve, mockResolveSecretReference) + + Expect(err).Should(BeNil()) + Expect(len(secrets)).Should(Equal(1)) + Expect(string(secrets[secretName].Data["tls.crt"])).Should(ContainSubstring("BEGIN CERTIFICATE")) + Expect(string(secrets[secretName].Data["tls.key"])).Should(ContainSubstring("BEGIN PRIVATE KEY")) + }) + + It("Succeeded to get target tls type secret", func() { + By("By getting valid pem cert from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue, _ := createFakePem() + secretName := "targetSecret" + contentType := CertTypePem + kidStr := "fakeKid" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + Kid: &kidStr, + ContentType: &contentType, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + secrets, err := configurationProvider.ResolveSecretReferences(context.Background(), false, secretReferencesToResolve, mockResolveSecretReference) + + Expect(err).Should(BeNil()) + Expect(len(secrets)).Should(Equal(1)) + Expect(string(secrets[secretName].Data["tls.crt"])).Should(ContainSubstring("BEGIN CERTIFICATE")) + Expect(string(secrets[secretName].Data["tls.key"])).Should(ContainSubstring("BEGIN RSA PRIVATE KEY")) + }) + + It("Succeeded to get tls type secret", func() { + By("By getting valid non cert based secret from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue, _ := createFakePfx() + secretName := "targetSecret" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + secrets, err := configurationProvider.ResolveSecretReferences(context.Background(), false, secretReferencesToResolve, mockResolveSecretReference) + + Expect(err).Should(BeNil()) + Expect(len(secrets)).Should(Equal(1)) + Expect(string(secrets[secretName].Data["tls.crt"])).Should(ContainSubstring("BEGIN CERTIFICATE")) + Expect(string(secrets[secretName].Data["tls.key"])).Should(ContainSubstring("BEGIN PRIVATE KEY")) + }) + + It("Succeeded to get tls type secret", func() { + By("By getting valid non cert based pem secret from Azure Key Vault") + testSpec := acpv1.AzureAppConfigurationProviderSpec{ + Endpoint: &EndpointName, + Target: acpv1.ConfigurationGenerationParameters{ + ConfigMapName: ConfigMapName, + }, + Configuration: acpv1.AzureAppConfigurationKeyValueOptions{ + TrimKeyPrefixes: []string{"app:"}, + }, + } + testProvider := acpv1.AzureAppConfigurationProvider{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "azconfig.io/v1", + Kind: "AppConfigurationProvider", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testName", + Namespace: "testNamespace", + }, + Spec: testSpec, + } + + configurationProvider, _ := NewConfigurationSettingLoader(context.Background(), testProvider, mockGetConfigurationSettingsWithKV) + secretValue, _ := createFakePem() + secretName := "targetSecret" + secret1 := azsecrets.GetSecretResponse{ + SecretBundle: azsecrets.SecretBundle{ + Value: &secretValue, + }, + } + + secretReferencesToResolve := map[string]*TargetSecretReference{ + secretName: { + Type: corev1.SecretTypeTLS, + UriSegments: map[string]KeyVaultSecretUriSegment{ + secretName: { + HostName: "fake-vault", + SecretName: "fake-secret", + SecretVersion: "fake-version", + }, + }, + }, + } + + mockResolveSecretReference.EXPECT().Resolve(gomock.Any(), gomock.Any()).Return(secret1, nil) + secrets, err := configurationProvider.ResolveSecretReferences(context.Background(), false, secretReferencesToResolve, mockResolveSecretReference) + + Expect(err).Should(BeNil()) + Expect(len(secrets)).Should(Equal(1)) + Expect(string(secrets[secretName].Data["tls.crt"])).Should(ContainSubstring("BEGIN CERTIFICATE")) + Expect(string(secrets[secretName].Data["tls.key"])).Should(ContainSubstring("BEGIN RSA PRIVATE KEY")) }) }) }) @@ -525,7 +944,7 @@ func TestCreateSecretClients(t *testing.T) { Target: acpv1.ConfigurationGenerationParameters{ ConfigMapName: "configMap-test", }, - Secret: &acpv1.AzureKeyVaultReference{ + Secret: &acpv1.AzureSecretReference{ Target: acpv1.SecretGenerationParameters{ SecretName: "secret-test", }, @@ -557,7 +976,7 @@ func TestCreateSecretClients(t *testing.T) { Target: acpv1.ConfigurationGenerationParameters{ ConfigMapName: "configMap-test", }, - Secret: &acpv1.AzureKeyVaultReference{ + Secret: &acpv1.AzureSecretReference{ Target: acpv1.SecretGenerationParameters{ SecretName: "secretName", }, @@ -593,3 +1012,118 @@ func TestCreateSecretClients(t *testing.T) { assert.NotNil(t, r1) assert.NotNil(t, r2) } + +func createFakeKeyPem(key *rsa.PrivateKey) (string, error) { + keyBytes := x509.MarshalPKCS1PrivateKey(key) + // PEM encoding of private key + keyPEM := string(pem.EncodeToMemory( + &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: keyBytes, + }, + )) + + return keyPEM, nil +} + +func createFakeCertPem(key *rsa.PrivateKey) (string, error) { + notBefore := time.Now() + notAfter := notBefore.Add(365 * 24 * 10 * time.Hour) + + //Create certificate templet + template := x509.Certificate{ + SerialNumber: big.NewInt(0), + Subject: pkix.Name{CommonName: "localhost"}, + SignatureAlgorithm: x509.SHA256WithRSA, + NotBefore: notBefore, + NotAfter: notAfter, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyAgreement | x509.KeyUsageKeyEncipherment | x509.KeyUsageDataEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + //Create certificate using templet + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + return "", err + + } + //pem encoding of certificate + certPem := string(pem.EncodeToMemory( + &pem.Block{ + Type: "CERTIFICATE", + Bytes: derBytes, + }, + )) + + return certPem, nil +} + +func createFakePem() (string, error) { + key, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return "", err + } + + keyPEM, err := createFakeKeyPem(key) + if err != nil { + return "", err + } + + certPem, err := createFakeCertPem(key) + if err != nil { + return "", err + } + + return keyPEM + "\n" + certPem, nil +} + +func createFakePfx() (string, error) { + key, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return "", err + } + + keyPEM, err := createFakeKeyPem(key) + if err != nil { + return "", err + } + + certPem, err := createFakeCertPem(key) + if err != nil { + return "", err + } + + return createPFXFromPEM(keyPEM, certPem) +} + +func createPFXFromPEM(pemPrivateKey, pemCertificate string) (string, error) { + // Decode private key PEM + block, _ := pem.Decode([]byte(pemPrivateKey)) + if block == nil || block.Type != "RSA PRIVATE KEY" { + return "", fmt.Errorf("failed to decode private key PEM") + } + + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return "", err + } + + // Decode certificate PEM + block, _ = pem.Decode([]byte(pemCertificate)) + if block == nil || block.Type != "CERTIFICATE" { + return "", fmt.Errorf("failed to decode certificate PEM") + } + + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return "", err + } + + // Create PKCS12 structure + pfxData, err := pkcs12.Legacy.Encode(privateKey, cert, []*x509.Certificate{}, "") + if err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(pfxData), nil +} diff --git a/internal/loader/configuration_setting_loader.go b/internal/loader/configuration_setting_loader.go index c356fc0..97d1842 100644 --- a/internal/loader/configuration_setting_loader.go +++ b/internal/loader/configuration_setting_loader.go @@ -7,7 +7,10 @@ import ( acpv1 "azappconfig/provider/api/v1" "azappconfig/provider/internal/properties" "context" + "crypto/x509" + "encoding/base64" "encoding/json" + "encoding/pem" "errors" "fmt" "net/url" @@ -20,6 +23,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/data/azappconfig" "github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets" "github.com/google/uuid" + "golang.org/x/crypto/pkcs12" "golang.org/x/exp/maps" "golang.org/x/sync/errgroup" "golang.org/x/sync/syncmap" @@ -39,25 +43,31 @@ type ConfigurationSettingLoader struct { } type TargetKeyValueSettings struct { - ConfigMapSettings map[string]string - SecretSettings map[string][]byte - KeyVaultReferencesToCache map[string]KeyVaultSecretUriSegment + ConfigMapSettings map[string]string + // Multiple secrets could be managed + SecretSettings map[string]corev1.Secret + SecretReferences map[string]*TargetSecretReference +} + +type TargetSecretReference struct { + Type corev1.SecretType + UriSegments map[string]KeyVaultSecretUriSegment } type RawSettings struct { - KeyValueSettings map[string]*string - IsJsonContentTypeMap map[string]bool - FeatureFlagSettings map[string]interface{} - SecretSettings map[string][]byte - KeyVaultReferencesToCache map[string]KeyVaultSecretUriSegment + KeyValueSettings map[string]*string + IsJsonContentTypeMap map[string]bool + FeatureFlagSettings map[string]interface{} + SecretSettings map[string]corev1.Secret + SecretReferences map[string]*TargetSecretReference } type ConfigurationSettingsRetriever interface { - CreateTargetSettings(ctx context.Context, resolveSecretReference ResolveSecretReference) (*TargetKeyValueSettings, error) - RefreshKeyValueSettings(ctx context.Context, existingConfigMapSettings *map[string]string, resolveSecretReference ResolveSecretReference) (*TargetKeyValueSettings, error) + CreateTargetSettings(ctx context.Context, resolveSecretReference SecretReferenceResolver) (*TargetKeyValueSettings, error) + RefreshKeyValueSettings(ctx context.Context, existingConfigMapSettings *map[string]string, resolveSecretReference SecretReferenceResolver) (*TargetKeyValueSettings, error) RefreshFeatureFlagSettings(ctx context.Context, existingConfigMapSettings *map[string]string) (*TargetKeyValueSettings, error) CheckAndRefreshSentinels(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, eTags map[acpv1.Sentinel]*azcore.ETag) (bool, map[acpv1.Sentinel]*azcore.ETag, error) - ResolveKeyVaultReferences(ctx context.Context, kvReferencesToResolve map[string]KeyVaultSecretUriSegment, kvResolver ResolveSecretReference) (map[string][]byte, error) + ResolveSecretReferences(ctx context.Context, versionedOnly bool, kvReferencesToResolve map[string]*TargetSecretReference, kvResolver SecretReferenceResolver) (map[string]corev1.Secret, error) } type GetSettingsFunc func(ctx context.Context, filters []acpv1.Selector, client *azappconfig.Client, c chan []azappconfig.Setting, e chan error) @@ -69,7 +79,7 @@ type ServicePrincipleAuthenticationParameters struct { } const ( - KeyVaultReferenceContentType string = "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8" + SecretReferenceContentType string = "application/vnd.microsoft.appconfig.keyvaultref+json;charset=utf-8" FeatureFlagContentType string = "application/vnd.microsoft.appconfig.ff+json;charset=utf-8" AzureClientId string = "azure_client_id" AzureClientSecret string = "azure_client_secret" @@ -78,6 +88,11 @@ const ( FeatureFlagKeyPrefix string = ".appconfig.featureflag/" FeatureFlagSectionName string = "FeatureFlags" FeatureManagementSectionName string = "FeatureManagement" + PreservedSecretTypeTag string = ".kubernetes.secret.type" + CertTypePem string = "application/x-pem-file" + CertTypePfx string = "application/x-pkcs12" + TlsKey string = "tls.key" + TlsCrt string = "tls.crt" ) var ( @@ -123,7 +138,7 @@ func NewConfigurationSettingLoader(ctx context.Context, provider acpv1.AzureAppC }, nil } -func (csl *ConfigurationSettingLoader) CreateTargetSettings(ctx context.Context, resolveSecretReference ResolveSecretReference) (*TargetKeyValueSettings, error) { +func (csl *ConfigurationSettingLoader) CreateTargetSettings(ctx context.Context, resolveSecretReference SecretReferenceResolver) (*TargetKeyValueSettings, error) { rawSettings, err := csl.CreateKeyValueSettings(ctx, resolveSecretReference) if err != nil { return nil, err @@ -141,13 +156,13 @@ func (csl *ConfigurationSettingLoader) CreateTargetSettings(ctx context.Context, } return &TargetKeyValueSettings{ - ConfigMapSettings: typedSettings, - SecretSettings: rawSettings.SecretSettings, - KeyVaultReferencesToCache: rawSettings.KeyVaultReferencesToCache, + ConfigMapSettings: typedSettings, + SecretSettings: rawSettings.SecretSettings, + SecretReferences: rawSettings.SecretReferences, }, nil } -func (csl *ConfigurationSettingLoader) RefreshKeyValueSettings(ctx context.Context, existingConfigMapSetting *map[string]string, resolveSecretReference ResolveSecretReference) (*TargetKeyValueSettings, error) { +func (csl *ConfigurationSettingLoader) RefreshKeyValueSettings(ctx context.Context, existingConfigMapSetting *map[string]string, resolveSecretReference SecretReferenceResolver) (*TargetKeyValueSettings, error) { rawSettings, err := csl.CreateKeyValueSettings(ctx, resolveSecretReference) if err != nil { return nil, err @@ -166,9 +181,9 @@ func (csl *ConfigurationSettingLoader) RefreshKeyValueSettings(ctx context.Conte } return &TargetKeyValueSettings{ - ConfigMapSettings: typedSettings, - SecretSettings: rawSettings.SecretSettings, - KeyVaultReferencesToCache: rawSettings.KeyVaultReferencesToCache, + ConfigMapSettings: typedSettings, + SecretSettings: rawSettings.SecretSettings, + SecretReferences: rawSettings.SecretReferences, }, nil } @@ -195,20 +210,20 @@ func (csl *ConfigurationSettingLoader) RefreshFeatureFlagSettings(ctx context.Co }, nil } -func (csl *ConfigurationSettingLoader) CreateKeyValueSettings(ctx context.Context, resolveSecretReference ResolveSecretReference) (*RawSettings, error) { +func (csl *ConfigurationSettingLoader) CreateKeyValueSettings(ctx context.Context, resolveSecretReference SecretReferenceResolver) (*RawSettings, error) { settingsChan := make(chan []azappconfig.Setting) errChan := make(chan error) keyValueFilters := getKeyValueFilters(csl.Spec) go csl.getSettingsFunc(ctx, keyValueFilters, csl.AppConfigClient, settingsChan, errChan) rawSettings := &RawSettings{ - KeyValueSettings: make(map[string]*string), - IsJsonContentTypeMap: make(map[string]bool), - SecretSettings: make(map[string][]byte), - KeyVaultReferencesToCache: make(map[string]KeyVaultSecretUriSegment), + KeyValueSettings: make(map[string]*string), + IsJsonContentTypeMap: make(map[string]bool), + SecretSettings: make(map[string]corev1.Secret), + SecretReferences: make(map[string]*TargetSecretReference), } var settings []azappconfig.Setting - var kvResolver ResolveSecretReference + var kvResolver SecretReferenceResolver for { select { @@ -230,17 +245,26 @@ func (csl *ConfigurationSettingLoader) CreateKeyValueSettings(ctx context.Contex } switch *setting.ContentType { case FeatureFlagContentType: - continue // ignore feature flag at this moment, will support it in later version - case KeyVaultReferenceContentType: + continue // ignore feature flag while getting key value settings + case SecretReferenceContentType: if setting.Value == nil { return nil, fmt.Errorf("The value of Key Vault reference '%s' is null", *setting.Key) } - if csl.Spec.Secret == nil { + + var secretType corev1.SecretType = corev1.SecretTypeOpaque + var err error + if secretTypeTag, ok := setting.Tags[PreservedSecretTypeTag]; ok { + secretType, err = parseSecretType(secretTypeTag) + if err != nil { + return nil, err + } + } else if csl.Spec.Secret == nil { return nil, fmt.Errorf("A Key Vault reference is found in App Configuration, but 'spec.secret' was not configured in the Azure App Configuration provider '%s' in namespace '%s'", csl.Name, csl.Namespace) } + if kvResolver == nil { if resolveSecretReference == nil { - if newKvResolver, err := csl.createKeyVaultResolver(ctx); err != nil { + if newKvResolver, err := csl.createSecretReferenceResolver(ctx); err != nil { return nil, err } else { kvResolver = newKvResolver @@ -256,23 +280,33 @@ func (csl *ConfigurationSettingLoader) CreateKeyValueSettings(ctx context.Contex return nil, err } - // Cache the non-versioned secret reference - if secretUriSegment.SecretVersion == "" { - rawSettings.KeyVaultReferencesToCache[trimmedKey] = *secretUriSegment + secretName := trimmedKey + // If the secret type is not specified, reside it to the Secret with name specified + if secretType == corev1.SecretTypeOpaque { + secretName = csl.Spec.Secret.Target.SecretName } - rawSettings.KeyVaultReferencesToCache[trimmedKey] = *secretUriSegment + if _, ok := rawSettings.SecretReferences[secretName]; !ok { + rawSettings.SecretReferences[secretName] = &TargetSecretReference{ + Type: secretType, + UriSegments: make(map[string]KeyVaultSecretUriSegment), + } + } + rawSettings.SecretReferences[secretName].UriSegments[trimmedKey] = *secretUriSegment default: rawSettings.KeyValueSettings[trimmedKey] = setting.Value rawSettings.IsJsonContentTypeMap[trimmedKey] = isJsonContentType(setting.ContentType) } } - // resolve the keyVault reference settings - if resolvedSecret, err := csl.ResolveKeyVaultReferences(ctx, rawSettings.KeyVaultReferencesToCache, kvResolver); err != nil { + // resolve the secret reference settings + if resolvedSecret, err := csl.ResolveSecretReferences(ctx, false, rawSettings.SecretReferences, kvResolver); err != nil { return nil, err } else { - maps.Copy(rawSettings.SecretSettings, resolvedSecret) + err = MergeSecret(rawSettings.SecretSettings, resolvedSecret) + if err != nil { + return nil, err + } } case err := <-errChan: if err != nil { @@ -323,7 +357,10 @@ end: return featureFlagSection, nil } -func (csl *ConfigurationSettingLoader) CheckAndRefreshSentinels(ctx context.Context, provider *acpv1.AzureAppConfigurationProvider, eTags map[acpv1.Sentinel]*azcore.ETag) (bool, map[acpv1.Sentinel]*azcore.ETag, error) { +func (csl *ConfigurationSettingLoader) CheckAndRefreshSentinels( + ctx context.Context, + provider *acpv1.AzureAppConfigurationProvider, + eTags map[acpv1.Sentinel]*azcore.ETag) (bool, map[acpv1.Sentinel]*azcore.ETag, error) { sentinelChanged := false if provider.Spec.Configuration.Refresh == nil { return sentinelChanged, eTags, NewArgumentError("spec.configuration.refresh", fmt.Errorf("refresh is not specified")) @@ -349,33 +386,87 @@ func (csl *ConfigurationSettingLoader) CheckAndRefreshSentinels(ctx context.Cont return sentinelChanged, refreshedETags, nil } -func (csl *ConfigurationSettingLoader) ResolveKeyVaultReferences(ctx context.Context, keyVaultReferencesToResolve map[string]KeyVaultSecretUriSegment, keyVaultResolver ResolveSecretReference) (map[string][]byte, error) { - if keyVaultResolver == nil { - if kvResolver, err := csl.createKeyVaultResolver(ctx); err != nil { +func (csl *ConfigurationSettingLoader) ResolveSecretReferences( + ctx context.Context, + versionedOnly bool, + secretReferencesToResolve map[string]*TargetSecretReference, + resolver SecretReferenceResolver) (map[string]corev1.Secret, error) { + if resolver == nil { + if kvResolver, err := csl.createSecretReferenceResolver(ctx); err != nil { return nil, err } else { - keyVaultResolver = kvResolver + resolver = kvResolver } } - resolvedSecretReferences := make(map[string][]byte) - if len(keyVaultReferencesToResolve) > 0 { + resolvedSecretReferences := make(map[string]corev1.Secret) + for name, targetSecretReference := range secretReferencesToResolve { + resolvedSecretReferences[name] = corev1.Secret{ + Data: make(map[string][]byte), + Type: targetSecretReference.Type, + } + var eg errgroup.Group - lock := &sync.Mutex{} - for key, kvReference := range keyVaultReferencesToResolve { - currentKey := key - currentReference := kvReference + if targetSecretReference.Type == corev1.SecretTypeOpaque { + if len(targetSecretReference.UriSegments) > 0 { + lock := &sync.Mutex{} + for key, kvReference := range targetSecretReference.UriSegments { + if versionedOnly && kvReference.SecretVersion == "" { + continue + } + + currentKey := key + currentReference := kvReference + eg.Go(func() error { + resolvedSecret, err := resolver.Resolve(currentReference, ctx) + if err != nil { + return fmt.Errorf("fail to resolve the Key Vault reference type setting '%s': %s", currentKey, err.Error()) + } + lock.Lock() + defer lock.Unlock() + resolvedSecretReferences[name].Data[currentKey] = []byte(*resolvedSecret.Value) + return nil + }) + } + + if err := eg.Wait(); err != nil { + return nil, err + } + } + } else if targetSecretReference.Type == corev1.SecretTypeTLS { + if versionedOnly && targetSecretReference.UriSegments[name].SecretVersion == "" { + continue + } + eg.Go(func() error { - resolvedValue, err := keyVaultResolver.Resolve(currentReference, ctx) + resolvedSecret, err := resolver.Resolve(targetSecretReference.UriSegments[name], ctx) if err != nil { - return fmt.Errorf("Fail to resolve the Key Vault reference type setting '%s': %s", currentKey, err.Error()) + return fmt.Errorf("fail to resolve the Key Vault reference type setting '%s': %s", name, err.Error()) + } + + if resolvedSecret.Kid != nil && len(*resolvedSecret.Kid) > 0 { + switch *resolvedSecret.ContentType { + case CertTypePfx: + resolvedSecretReferences[name].Data[TlsKey], resolvedSecretReferences[name].Data[TlsCrt], err = decodePkcs12(*resolvedSecret.Value) + case CertTypePem: + resolvedSecretReferences[name].Data[TlsKey], resolvedSecretReferences[name].Data[TlsCrt], err = decodePem(*resolvedSecret.Value) + default: + err = fmt.Errorf("failed to get certificate, unknown content type '%s'", *resolvedSecret.ContentType) + } + } else { + resolvedSecretReferences[name].Data[TlsKey], resolvedSecretReferences[name].Data[TlsCrt], err = decodePkcs12(*resolvedSecret.Value) + if err != nil { + resolvedSecretReferences[name].Data[TlsKey], resolvedSecretReferences[name].Data[TlsCrt], err = decodePem(*resolvedSecret.Value) + } + } + + if err != nil { + return fmt.Errorf("fail to decode the cert '%s': %s", name, err.Error()) } - lock.Lock() - defer lock.Unlock() - resolvedSecretReferences[currentKey] = []byte(*resolvedValue) return nil }) } + // All other types are not supported if err := eg.Wait(); err != nil { return nil, err @@ -415,7 +506,7 @@ func (csl *ConfigurationSettingLoader) getSentinelSetting(ctx context.Context, p return &sentinelSetting.Setting, nil } -func (csl *ConfigurationSettingLoader) createKeyVaultResolver(ctx context.Context) (ResolveSecretReference, error) { +func (csl *ConfigurationSettingLoader) createSecretReferenceResolver(ctx context.Context) (SecretReferenceResolver, error) { var defaultAuth *acpv1.AzureAppConfigurationProviderAuth = nil if csl.Spec.Secret != nil && csl.Spec.Secret.Auth != nil { defaultAuth = csl.Spec.Secret.Auth.AzureAppConfigurationProviderAuth @@ -428,12 +519,12 @@ func (csl *ConfigurationSettingLoader) createKeyVaultResolver(ctx context.Contex if err != nil { return nil, err } - keyVaultResolver := &KeyVaultReferenceResolver{ + resolver := &KeyVaultConnector{ DefaultTokenCredential: defaultCred, Clients: secretClients, } - return keyVaultResolver, nil + return resolver, nil } func trimPrefix(key string, prefixToTrim []string) string { @@ -448,7 +539,11 @@ func trimPrefix(key string, prefixToTrim []string) string { return key } -func getConfigurationSettings(ctx context.Context, filters []acpv1.Selector, client *azappconfig.Client, c chan []azappconfig.Setting, e chan error) { +func getConfigurationSettings( + ctx context.Context, + filters []acpv1.Selector, + client *azappconfig.Client, + c chan []azappconfig.Setting, e chan error) { nullString := "\x00" for _, filter := range filters { @@ -475,7 +570,10 @@ func getConfigurationSettings(ctx context.Context, filters []acpv1.Selector, cli c <- make([]azappconfig.Setting, 0) } -func createTokenCredential(ctx context.Context, acpAuth *acpv1.AzureAppConfigurationProviderAuth, namespace string) (azcore.TokenCredential, error) { +func createTokenCredential( + ctx context.Context, + acpAuth *acpv1.AzureAppConfigurationProviderAuth, + namespace string) (azcore.TokenCredential, error) { // If User explicitly specify the authentication method if acpAuth != nil { if acpAuth.WorkloadIdentity != nil { @@ -506,7 +604,10 @@ func createTokenCredential(ctx context.Context, acpAuth *acpv1.AzureAppConfigura return nil, nil } -func getWorkloadIdentityClientId(ctx context.Context, workloadIdentityAuth *acpv1.WorkloadIdentityParameters, namespace string) (string, error) { +func getWorkloadIdentityClientId( + ctx context.Context, + workloadIdentityAuth *acpv1.WorkloadIdentityParameters, + namespace string) (string, error) { if workloadIdentityAuth.ManagedIdentityClientIdReference == nil { return *workloadIdentityAuth.ManagedIdentityClientId, nil } else { @@ -528,8 +629,10 @@ func getWorkloadIdentityClientId(ctx context.Context, workloadIdentityAuth *acpv } } -func getConnectionStringParameter(ctx context.Context, namespacedSecretName types.NamespacedName) (string, error) { - secret, err := getSecret(ctx, namespacedSecretName) +func getConnectionStringParameter( + ctx context.Context, + namespacedSecretName types.NamespacedName) (string, error) { + secret, err := GetSecret(ctx, namespacedSecretName) if err != nil { return "", err } @@ -537,8 +640,10 @@ func getConnectionStringParameter(ctx context.Context, namespacedSecretName type return string(secret.Data[AzureAppConfigurationConnectionString]), nil } -func getServicePrincipleAuthenticationParameters(ctx context.Context, namespacedSecretName types.NamespacedName) (*ServicePrincipleAuthenticationParameters, error) { - secret, err := getSecret(ctx, namespacedSecretName) +func getServicePrincipleAuthenticationParameters( + ctx context.Context, + namespacedSecretName types.NamespacedName) (*ServicePrincipleAuthenticationParameters, error) { + secret, err := GetSecret(ctx, namespacedSecretName) if err != nil { return nil, err } @@ -550,7 +655,8 @@ func getServicePrincipleAuthenticationParameters(ctx context.Context, namespaced }, nil } -func getConfigMap(ctx context.Context, namespacedConfigMapName types.NamespacedName) (*corev1.ConfigMap, error) { +func getConfigMap(ctx context.Context, + namespacedConfigMapName types.NamespacedName) (*corev1.ConfigMap, error) { cfg, err := config.GetConfig() if err != nil { return nil, err @@ -569,7 +675,8 @@ func getConfigMap(ctx context.Context, namespacedConfigMapName types.NamespacedN return configMapObject, nil } -func getSecret(ctx context.Context, namespacedSecretName types.NamespacedName) (*corev1.Secret, error) { +func GetSecret(ctx context.Context, + namespacedSecretName types.NamespacedName) (*corev1.Secret, error) { cfg, err := config.GetConfig() if err != nil { return nil, err @@ -655,7 +762,9 @@ func reverse(arr []acpv1.Selector) { } } -func createSecretClients(ctx context.Context, acp acpv1.AzureAppConfigurationProvider) (*syncmap.Map, error) { +func createSecretClients( + ctx context.Context, + acp acpv1.AzureAppConfigurationProvider) (*syncmap.Map, error) { secretClients := &syncmap.Map{} if acp.Spec.Secret == nil || acp.Spec.Secret.Auth == nil { return secretClients, nil @@ -679,3 +788,108 @@ func createSecretClients(ctx context.Context, acp acpv1.AzureAppConfigurationPro return secretClients, nil } + +func parseSecretType(secretType string) (corev1.SecretType, error) { + secretTypeMap := map[string]corev1.SecretType{ + "opaque": corev1.SecretTypeOpaque, + "kubernetes.io/tls": corev1.SecretTypeTLS, + } + + if parsedType, ok := secretTypeMap[secretType]; ok { + if parsedType != corev1.SecretTypeTLS { + return "", fmt.Errorf("secret type %q is not supported", secretType) + } else { + return parsedType, nil + } + } else { + return "", fmt.Errorf("secret type %q is not supported", secretType) + } +} + +func decodePkcs12(value string) (key []byte, crt []byte, err error) { + pfxRaw, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return nil, nil, err + } + // using ToPEM to extract more than one certificate and key in pfxData + pemBlock, err := pkcs12.ToPEM(pfxRaw, "") + if err != nil { + return nil, nil, err + } + + return parsePemBlock(pemBlock) +} + +func decodePem(value string) (key []byte, crt []byte, err error) { + pemBlocks := []*pem.Block{} + for pemBlock, rest := pem.Decode([]byte(value)); pemBlock != nil; pemBlock, rest = pem.Decode(rest) { + pemBlocks = append(pemBlocks, pemBlock) + } + if len(pemBlocks) == 0 { + return nil, nil, fmt.Errorf("failed to decode pem block") + } + + return parsePemBlock(pemBlocks) +} + +func parsePemBlock(pemBlock []*pem.Block) ([]byte, []byte, error) { + // PEM block encoded form contains the headers + // -----BEGIN Type----- + // Headers + // base64-encoded Bytes + // -----END Type----- + // Setting headers to nil to ensure no headers included in the encoded block + var pemKeyData, pemCertData []byte + for _, block := range pemBlock { + + block.Headers = make(map[string]string) + if block.Type == "CERTIFICATE" { + pemCertData = append(pemCertData, pem.EncodeToMemory(block)...) + } else { + key, err := parsePrivateKey(block.Bytes) + if err != nil { + return nil, nil, err + } + // pkcs1 RSA private key PEM file is specific for RSA keys. RSA is not used exclusively inside X509 + // and SSL/TLS, a more generic key format is available in the form of PKCS#8 that identifies the type + // of private key and contains the relevant data. + // Converting to pkcs8 private key as ToPEM uses pkcs1 + // The driver determines the key type from the pkcs8 form of the key and marshals appropriately + block.Bytes, err = x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, nil, err + } + pemKeyData = append(pemKeyData, pem.EncodeToMemory(block)...) + } + } + + return pemKeyData, pemCertData, nil +} + +func parsePrivateKey(block []byte) (interface{}, error) { + if key, err := x509.ParsePKCS1PrivateKey(block); err == nil { + return key, nil + } + if key, err := x509.ParsePKCS8PrivateKey(block); err == nil { + return key, nil + } + if key, err := x509.ParseECPrivateKey(block); err == nil { + return key, nil + } + return nil, fmt.Errorf("failed to parse key for type pkcs1, pkcs8 or ec") +} + +func MergeSecret(secret map[string]corev1.Secret, newSecret map[string]corev1.Secret) error { + for k, v := range newSecret { + if _, ok := secret[k]; !ok { + secret[k] = v + } else if secret[k].Type != v.Type { + return fmt.Errorf("secret type mismatch for key %q", k) + + } else { + maps.Copy(secret[k].Data, v.Data) + } + } + + return nil +} diff --git a/internal/loader/keyvault_reference_resolver.go b/internal/loader/keyvault_reference_resolver.go index 4813555..8ad9ddf 100644 --- a/internal/loader/keyvault_reference_resolver.go +++ b/internal/loader/keyvault_reference_resolver.go @@ -15,7 +15,7 @@ import ( "golang.org/x/sync/syncmap" ) -type KeyVaultReferenceResolver struct { +type KeyVaultConnector struct { DefaultTokenCredential azcore.TokenCredential Clients *syncmap.Map //map[string]*azsecrets.Client } @@ -30,27 +30,25 @@ type KeyVaultSecretUriSegment struct { SecretVersion string } -type ResolveSecretReference interface { - Resolve(secretUriSegment KeyVaultSecretUriSegment, ctx context.Context) (*string, error) +type SecretReferenceResolver interface { + Resolve(secretUriSegment KeyVaultSecretUriSegment, ctx context.Context) (azsecrets.GetSecretResponse, error) } -func (resolver *KeyVaultReferenceResolver) Resolve(secretUriSegment KeyVaultSecretUriSegment, ctx context.Context) (*string, error) { +func (resolver *KeyVaultConnector) Resolve( + secretUriSegment KeyVaultSecretUriSegment, + ctx context.Context) (azsecrets.GetSecretResponse, error) { var secretClient any var ok bool if secretClient, ok = resolver.Clients.Load(secretUriSegment.HostName); !ok { newSecretClient, err := azsecrets.NewClient("https://"+secretUriSegment.HostName, resolver.DefaultTokenCredential, nil) if err != nil { - return nil, err + return azsecrets.GetSecretResponse{}, err } secretClient = newSecretClient resolver.Clients.Store(secretUriSegment.HostName, newSecretClient) } - secretValue, err := secretClient.(*azsecrets.Client).GetSecret(ctx, secretUriSegment.SecretName, secretUriSegment.SecretVersion, nil) - if err != nil { - return nil, err - } - return secretValue.Value, nil + return secretClient.(*azsecrets.Client).GetSecret(ctx, secretUriSegment.SecretName, secretUriSegment.SecretVersion, nil) } func parse(settingValue string) (*KeyVaultSecretUriSegment, error) { diff --git a/internal/loader/keyvault_reference_resolver_test.go b/internal/loader/keyvault_reference_resolver_test.go index a80e596..a7c937c 100644 --- a/internal/loader/keyvault_reference_resolver_test.go +++ b/internal/loader/keyvault_reference_resolver_test.go @@ -96,13 +96,13 @@ func TestResolveSecretReferenceSetting(t *testing.T) { valueToTest := "{ \"uri\":\"https://FAKE-VAULT/secrets/fake-secret/testversion/notvalid\"}" defaultCredential := mockTokeCredential{} clients := &syncmap.Map{} - resolver := &KeyVaultReferenceResolver{ + resolver := &KeyVaultConnector{ DefaultTokenCredential: defaultCredential, Clients: clients, } secretUriSegment, _ := parse(valueToTest) - resolvedValue, err := resolver.Resolve(*secretUriSegment, context.Background()) + resolvedSecret, err := resolver.Resolve(*secretUriSegment, context.Background()) assert.Contains(t, err.Error(), "fake-vault") length := 0 @@ -111,7 +111,7 @@ func TestResolveSecretReferenceSetting(t *testing.T) { return true }) assert.Equal(t, 1, length) - assert.Nil(t, resolvedValue) + assert.Nil(t, resolvedSecret.Value) } type mockTokeCredential struct { diff --git a/internal/loader/mocks/mock_configuration_settings_retriever.go b/internal/loader/mocks/mock_configuration_settings_retriever.go index edbf282..8d8eb40 100644 --- a/internal/loader/mocks/mock_configuration_settings_retriever.go +++ b/internal/loader/mocks/mock_configuration_settings_retriever.go @@ -12,6 +12,7 @@ import ( azcore "github.com/Azure/azure-sdk-for-go/sdk/azcore" gomock "github.com/golang/mock/gomock" + v10 "k8s.io/api/core/v1" ) // MockConfigurationSettingsRetriever is a mock of ConfigurationSettingsRetriever interface. @@ -54,7 +55,7 @@ func (mr *MockConfigurationSettingsRetrieverMockRecorder) CheckAndRefreshSentine } // CreateTargetSettings mocks base method. -func (m *MockConfigurationSettingsRetriever) CreateTargetSettings(arg0 context.Context, arg1 loader.ResolveSecretReference) (*loader.TargetKeyValueSettings, error) { +func (m *MockConfigurationSettingsRetriever) CreateTargetSettings(arg0 context.Context, arg1 loader.SecretReferenceResolver) (*loader.TargetKeyValueSettings, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateTargetSettings", arg0, arg1) ret0, _ := ret[0].(*loader.TargetKeyValueSettings) @@ -84,7 +85,7 @@ func (mr *MockConfigurationSettingsRetrieverMockRecorder) RefreshFeatureFlagSett } // RefreshKeyValueSettings mocks base method. -func (m *MockConfigurationSettingsRetriever) RefreshKeyValueSettings(arg0 context.Context, arg1 *map[string]string, arg2 loader.ResolveSecretReference) (*loader.TargetKeyValueSettings, error) { +func (m *MockConfigurationSettingsRetriever) RefreshKeyValueSettings(arg0 context.Context, arg1 *map[string]string, arg2 loader.SecretReferenceResolver) (*loader.TargetKeyValueSettings, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "RefreshKeyValueSettings", arg0, arg1, arg2) ret0, _ := ret[0].(*loader.TargetKeyValueSettings) @@ -98,17 +99,17 @@ func (mr *MockConfigurationSettingsRetrieverMockRecorder) RefreshKeyValueSetting return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshKeyValueSettings", reflect.TypeOf((*MockConfigurationSettingsRetriever)(nil).RefreshKeyValueSettings), arg0, arg1, arg2) } -// ResolveKeyVaultReferences mocks base method. -func (m *MockConfigurationSettingsRetriever) ResolveKeyVaultReferences(arg0 context.Context, arg1 map[string]loader.KeyVaultSecretUriSegment, arg2 loader.ResolveSecretReference) (map[string][]byte, error) { +// ResolveSecretReferences mocks base method. +func (m *MockConfigurationSettingsRetriever) ResolveSecretReferences(arg0 context.Context, arg1 bool, arg2 map[string]*loader.TargetSecretReference, arg3 loader.SecretReferenceResolver) (map[string]v10.Secret, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ResolveKeyVaultReferences", arg0, arg1, arg2) - ret0, _ := ret[0].(map[string][]byte) + ret := m.ctrl.Call(m, "ResolveSecretReferences", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(map[string]v10.Secret) ret1, _ := ret[1].(error) return ret0, ret1 } -// ResolveKeyVaultReferences indicates an expected call of ResolveKeyVaultReferences. -func (mr *MockConfigurationSettingsRetrieverMockRecorder) ResolveKeyVaultReferences(arg0, arg1, arg2 interface{}) *gomock.Call { +// ResolveSecretReferences indicates an expected call of ResolveSecretReferences. +func (mr *MockConfigurationSettingsRetrieverMockRecorder) ResolveSecretReferences(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveKeyVaultReferences", reflect.TypeOf((*MockConfigurationSettingsRetriever)(nil).ResolveKeyVaultReferences), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveSecretReferences", reflect.TypeOf((*MockConfigurationSettingsRetriever)(nil).ResolveSecretReferences), arg0, arg1, arg2, arg3) } diff --git a/internal/loader/typed_setting.go b/internal/loader/typed_setting.go index 4b12047..0b70374 100644 --- a/internal/loader/typed_setting.go +++ b/internal/loader/typed_setting.go @@ -82,7 +82,7 @@ func createTypedSettings(rawSettings *RawSettings, dataOptions *acpv1.ConfigMapD func marshalProperties(settings map[string]string) string { stringBuilder := strings.Builder{} separator := "\n" - if settings != nil && len(settings) > 0 { + if settings != nil { i := 0 for k, v := range settings { stringBuilder.WriteString(fmt.Sprintf("%s=%s", k, v))