From 5845650ffcf730f95ef40b1db74109cb9113218b Mon Sep 17 00:00:00 2001 From: TheAlain <43777839+asaintsever@users.noreply.github.com> Date: Wed, 19 May 2021 16:04:18 +0200 Subject: [PATCH] Add support for admission.k8s.io/v1 AdmissionReview and admissionregistration.k8s.io/v1 MutatingWebhookConfiguration (in addition to v1beta1) (#49) --- CHANGELOG.md | 10 +- Dockerfile | 2 +- Dockerfile.local | 2 +- VERSION_CHART | 2 +- VERSION_RELEASE | 2 +- VERSION_VSI | 2 +- deploy/helm/templates/_helpers.tpl | 11 + .../mutatingwebhookconfiguration.yaml | 6 +- pkg/k8s/k8s.go | 39 ++- pkg/webhook/convert.go | 89 ++++++ pkg/webhook/mutate.go | 109 +++++++ pkg/webhook/mutate_test.go | 240 +++++++++++++++ pkg/webhook/utils.go | 18 +- pkg/webhook/webhook-server.go | 182 +++++------ pkg/webhook/webhook-server_test.go | 288 ++++++------------ 15 files changed, 681 insertions(+), 321 deletions(-) create mode 100644 pkg/webhook/convert.go create mode 100644 pkg/webhook/mutate.go create mode 100644 pkg/webhook/mutate_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f5a7978..e3de434 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,20 @@ # Changelog for Vault Sidecar Injector -## Release v7.1.2 - 2021-XX-XX +## Release v7.2.0 - 2021-05-XX + +This release comes with support for `admission.k8s.io/v1` AdmissionReview and `admissionregistration.k8s.io/v1` MutatingWebhookConfiguration on Kubernetes 1.16+. As a result, Vault Sidecar Injector now handles both v1 and v1beta1 versions of those resources. + +*Note that `admission.k8s.io/v1beta1` AdmissionReview and `admissionregistration.k8s.io/v1beta1` MutatingWebhookConfiguration should not be supported (nor available) anymore on Kubernetes 1.22+* **Changed** - [VSI #48](https://github.com/Talend/vault-sidecar-injector/pull/48) - Minor chart updates (adjust CPU & memory for injected containers, add checks during chart install) - [VSI #51](https://github.com/Talend/vault-sidecar-injector/pull/51) - Update base image to CentOS 7.9.2009 +**Added** + +- [VSI #49](https://github.com/Talend/vault-sidecar-injector/pull/49) - Add support for `admission.k8s.io/v1` AdmissionReview and `admissionregistration.k8s.io/v1` MutatingWebhookConfiguration (in addition to v1beta1) + ## Release v7.1.1 - 2021-04-02 **Fixed** diff --git a/Dockerfile b/Dockerfile index 5098a3e..b4aba78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ENV TALEND_USER=talend ENV TALEND_USERGROUP=$TALEND_USER ENV TALEND_UID=61000 -# Update CentOS (note that --security flag does not work on CentOS: https://www.caseylabs.com/centos-automatic-security-updates-do-not-work/) +# Update CentOS (note that --security flag does not work on CentOS: https://forums.centos.org/viewtopic.php?t=59369) RUN set -x \ && yum -y update \ && yum clean all \ diff --git a/Dockerfile.local b/Dockerfile.local index 691c32a..f1f2f46 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -8,7 +8,7 @@ ENV TALEND_USER=talend ENV TALEND_USERGROUP=$TALEND_USER ENV TALEND_UID=61000 -# Update CentOS (note that --security flag does not work on CentOS: https://www.caseylabs.com/centos-automatic-security-updates-do-not-work/) +# Update CentOS (note that --security flag does not work on CentOS: https://forums.centos.org/viewtopic.php?t=59369) RUN set -x \ && yum -y update \ && yum clean all \ diff --git a/VERSION_CHART b/VERSION_CHART index 078bf8b..8191138 100644 --- a/VERSION_CHART +++ b/VERSION_CHART @@ -1 +1 @@ -4.2.2 \ No newline at end of file +4.3.0 \ No newline at end of file diff --git a/VERSION_RELEASE b/VERSION_RELEASE index 0e7b60d..4b49d9b 100644 --- a/VERSION_RELEASE +++ b/VERSION_RELEASE @@ -1 +1 @@ -7.1.2 \ No newline at end of file +7.2.0 \ No newline at end of file diff --git a/VERSION_VSI b/VERSION_VSI index ef09838..4b49d9b 100644 --- a/VERSION_VSI +++ b/VERSION_VSI @@ -1 +1 @@ -7.1.1 \ No newline at end of file +7.2.0 \ No newline at end of file diff --git a/deploy/helm/templates/_helpers.tpl b/deploy/helm/templates/_helpers.tpl index b4671f5..d53d7de 100644 --- a/deploy/helm/templates/_helpers.tpl +++ b/deploy/helm/templates/_helpers.tpl @@ -108,3 +108,14 @@ Add Vault flag to skip verification of TLS certificates -tls-skip-verify {{- end -}} {{- end -}} + +{{/* +Return the appropriate apiVersion for MutatingWebhookConfiguration +*/}} +{{- define "mutatingwebhookconfiguration.apiversion" -}} +{{- if semverCompare ">=1.16" .Capabilities.KubeVersion.Version -}} +"admissionregistration.k8s.io/v1" +{{- else -}} +"admissionregistration.k8s.io/v1beta1" +{{- end -}} +{{- end -}} diff --git a/deploy/helm/templates/mutatingwebhookconfiguration.yaml b/deploy/helm/templates/mutatingwebhookconfiguration.yaml index 76ece01..b4e966f 100644 --- a/deploy/helm/templates/mutatingwebhookconfiguration.yaml +++ b/deploy/helm/templates/mutatingwebhookconfiguration.yaml @@ -1,4 +1,4 @@ -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: {{ include "mutatingwebhookconfiguration.apiversion" . }} kind: MutatingWebhookConfiguration metadata: name: {{ include "talend-vault-sidecar-injector.fullname" . }} @@ -16,5 +16,9 @@ webhooks: apiGroups: [""] apiVersions: ["v1"] resources: ["pods"] +{{- if semverCompare ">=1.16" .Capabilities.KubeVersion.Version }} + admissionReviewVersions: ["v1", "v1beta1"] + sideEffects: None +{{- end }} failurePolicy: {{ include "talend-vault-sidecar-injector.failurePolicy" .Values }} {{ include "talend-vault-sidecar-injector.namespaceSelector" . | indent 4 }} \ No newline at end of file diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go index 5060347..5b46307 100644 --- a/pkg/k8s/k8s.go +++ b/pkg/k8s/k8s.go @@ -1,4 +1,4 @@ -// Copyright © 2019-2020 Talend - www.talend.com +// Copyright © 2019-2021 Talend - www.talend.com // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -71,6 +71,12 @@ func (k8sctl *K8SClient) CreateCertSecret(ca, cert, key []byte) error { // Other way to get current namespace: //ns, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + // If secret already exists: log a warning before deleting it + if _, err := k8sctl.CoreV1().Secrets(strings.TrimSpace(string(ns))).Get(k8sctl.WebhookSecretName, metav1.GetOptions{}); err == nil { + klog.Warning("Webhook secret already exists: will be deleted then created again from new generated certificate") + k8sctl.DeleteCertSecret() + } + // Create Secret in same namespace as webhook _, err := k8sctl.CoreV1().Secrets(strings.TrimSpace(string(ns))).Create(secret) if err != nil { @@ -111,17 +117,28 @@ func (k8sctl *K8SClient) PatchWebhookConfiguration(cacertfile string) error { return err } + webhookPatch := []byte(fmt.Sprintf( + `[{ + "op": "add", + "path": "/webhooks/0/clientConfig/caBundle", + "value": %q + }]`, base64.StdEncoding.EncodeToString(caPEM))) + // Patch MutatingWebhookConfiguration resource with CA (should be base64-encoded PEM-encoded) - _, err = k8sctl.AdmissionregistrationV1beta1().MutatingWebhookConfigurations().Patch( - k8sctl.WebhookCfgName, types.JSONPatchType, []byte(fmt.Sprintf( - `[{ - "op": "add", - "path": "/webhooks/0/clientConfig/caBundle", - "value": %q - }]`, base64.StdEncoding.EncodeToString(caPEM)))) - if err != nil { - klog.Errorf("Error patching MutatingWebhookConfiguration's caBundle: %s", err) - return err + if _, err = k8sctl.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(k8sctl.WebhookCfgName, metav1.GetOptions{}); err == nil { + // v1 support + klog.Infof("Patching MutatingWebhookConfiguration v1 resource %v", k8sctl.WebhookCfgName) + if _, err = k8sctl.AdmissionregistrationV1().MutatingWebhookConfigurations().Patch(k8sctl.WebhookCfgName, types.JSONPatchType, webhookPatch); err != nil { + klog.Errorf("Error patching MutatingWebhookConfiguration's caBundle: %s", err) + return err + } + } else { + // v1beta1 support + klog.Infof("Patching MutatingWebhookConfiguration v1beta1 resource %v", k8sctl.WebhookCfgName) + if _, err = k8sctl.AdmissionregistrationV1beta1().MutatingWebhookConfigurations().Patch(k8sctl.WebhookCfgName, types.JSONPatchType, webhookPatch); err != nil { + klog.Errorf("Error patching MutatingWebhookConfiguration's caBundle: %s", err) + return err + } } return nil diff --git a/pkg/webhook/convert.go b/pkg/webhook/convert.go new file mode 100644 index 0000000..43e1660 --- /dev/null +++ b/pkg/webhook/convert.go @@ -0,0 +1,89 @@ +// Copyright © 2019-2021 Talend - www.talend.com +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package webhook + +import ( + "unsafe" + + admv1 "k8s.io/api/admission/v1" + admv1beta1 "k8s.io/api/admission/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// Note: +// ===== +// These conversions come from https://github.com/jetstack/cert-manager/blob/ab0cd57dc58fd73a76fd96bd9d1402bd5ae96582/pkg/webhook/server/util/convert.go +// (which are adapted from https://github.com/kubernetes/kubernetes/blob/03d322035d2f199f2163658d94a153ed2b9de667/pkg/apis/admission/v1beta1/zz_generated.conversion.go) + +func Convert_v1beta1_AdmissionReview_To_admission_AdmissionReview(in *admv1beta1.AdmissionReview, out *admv1.AdmissionReview) { + if in.Request != nil { + if out.Request == nil { + out.Request = &admv1.AdmissionRequest{} + } + in, out := &in.Request, &out.Request + *out = new(admv1.AdmissionRequest) + Convert_v1beta1_AdmissionRequest_To_admission_AdmissionRequest(*in, *out) + } else { + out.Request = nil + } + out.Response = (*admv1.AdmissionResponse)(unsafe.Pointer(in.Response)) +} + +func Convert_v1beta1_AdmissionRequest_To_admission_AdmissionRequest(in *admv1beta1.AdmissionRequest, out *admv1.AdmissionRequest) { + out.UID = types.UID(in.UID) + out.Kind = in.Kind + out.Resource = in.Resource + out.SubResource = in.SubResource + out.RequestKind = (*metav1.GroupVersionKind)(unsafe.Pointer(in.RequestKind)) + out.RequestResource = (*metav1.GroupVersionResource)(unsafe.Pointer(in.RequestResource)) + out.RequestSubResource = in.RequestSubResource + out.Name = in.Name + out.Namespace = in.Namespace + out.Operation = admv1.Operation(in.Operation) + out.Object = in.Object + out.OldObject = in.OldObject + out.Options = in.Options +} + +func Convert_admission_AdmissionReview_To_v1beta1_AdmissionReview(in *admv1.AdmissionReview, out *admv1beta1.AdmissionReview) { + if in.Request != nil { + if out.Request == nil { + out.Request = &admv1beta1.AdmissionRequest{} + } + in, out := &in.Request, &out.Request + *out = new(admv1beta1.AdmissionRequest) + Convert_admission_AdmissionRequest_To_v1beta1_AdmissionRequest(*in, *out) + } else { + out.Request = nil + } + out.Response = (*admv1beta1.AdmissionResponse)(unsafe.Pointer(in.Response)) +} + +func Convert_admission_AdmissionRequest_To_v1beta1_AdmissionRequest(in *admv1.AdmissionRequest, out *admv1beta1.AdmissionRequest) { + out.UID = types.UID(in.UID) + out.Kind = in.Kind + out.Resource = in.Resource + out.SubResource = in.SubResource + out.RequestKind = (*metav1.GroupVersionKind)(unsafe.Pointer(in.RequestKind)) + out.RequestResource = (*metav1.GroupVersionResource)(unsafe.Pointer(in.RequestResource)) + out.RequestSubResource = in.RequestSubResource + out.Name = in.Name + out.Namespace = in.Namespace + out.Operation = admv1beta1.Operation(in.Operation) + out.Object = in.Object + out.OldObject = in.OldObject + out.Options = in.Options +} diff --git a/pkg/webhook/mutate.go b/pkg/webhook/mutate.go new file mode 100644 index 0000000..92783ad --- /dev/null +++ b/pkg/webhook/mutate.go @@ -0,0 +1,109 @@ +// Copyright © 2019-2021 Talend - www.talend.com +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package webhook + +import ( + "encoding/json" + ctx "talend/vault-sidecar-injector/pkg/context" + + admv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" +) + +func (vaultInjector *VaultInjector) mutate(ar *admv1.AdmissionReview) *admv1.AdmissionResponse { + var pod corev1.Pod + var podName, podNamespace string + + req := ar.Request + + if err := json.Unmarshal(req.Object.Raw, &pod); err != nil { + klog.Errorf("Could not unmarshal raw object: %v", err) + return &admv1.AdmissionResponse{ + UID: req.UID, + Result: &metav1.Status{ + Message: err.Error(), + }, + } + } + + if klog.V(5) { // enabled by providing '-v=5' at least + klog.Infof("Pod=%+v", pod) + } + + if pod.Name == "" { + podName = pod.GenerateName + } else { + podName = pod.Name + } + + if pod.Namespace == "" { + podNamespace = metav1.NamespaceDefault + } else { + podNamespace = pod.Namespace + } + + klog.Infof("AdmissionReview '%v' for '%+v', Namespace=%v Name='%v (%s/%s)' UID=%v patchOperation=%v", + ar.GroupVersionKind(), req.Kind, req.Namespace, req.Name, podNamespace, podName, req.UID, req.Operation) + + // Determine whether to perform mutation + if !mutationRequired(ignoredNamespaces, vaultInjector.VaultInjectorAnnotationsFQ, &pod.ObjectMeta) { + klog.Infof("Skipping mutation for %s/%s due to policy check", podNamespace, podName) + return &admv1.AdmissionResponse{ + UID: req.UID, + Allowed: true, + } + } + + annotations := map[string]string{vaultInjector.VaultInjectorAnnotationsFQ[ctx.VaultInjectorAnnotationStatusKey]: ctx.VaultInjectorStatusInjected} + patchBytes, err := vaultInjector.createPatch(&pod, annotations) + if err != nil { + return &admv1.AdmissionResponse{ + UID: req.UID, + Allowed: false, + Result: &metav1.Status{ + Message: err.Error(), + }, + } + } + + klog.Infof("AdmissionResponse: patch=%v\n", string(patchBytes)) + return &admv1.AdmissionResponse{ + UID: req.UID, + Allowed: true, + Patch: patchBytes, + PatchType: func() *admv1.PatchType { + pt := admv1.PatchTypeJSONPatch + return &pt + }(), + } +} + +// Create mutation patch for resources +func (vaultInjector *VaultInjector) createPatch(pod *corev1.Pod, annotations map[string]string) ([]byte, error) { + + patchPodSpec, err := vaultInjector.updatePodSpec(pod) + if err != nil { + return nil, err + } + + var patch []ctx.PatchOperation + + patch = append(patch, patchPodSpec...) + patch = append(patch, updateAnnotation(pod.Annotations, annotations)...) + + return json.Marshal(patch) +} diff --git a/pkg/webhook/mutate_test.go b/pkg/webhook/mutate_test.go new file mode 100644 index 0000000..4674e7e --- /dev/null +++ b/pkg/webhook/mutate_test.go @@ -0,0 +1,240 @@ +// Copyright © 2019-2021 Talend - www.talend.com +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package webhook + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + cfg "talend/vault-sidecar-injector/pkg/config" + ctx "talend/vault-sidecar-injector/pkg/context" + "testing" + + "k8s.io/apimachinery/pkg/util/uuid" + + admv1 "k8s.io/api/admission/v1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog" + + "github.com/ghodss/yaml" + "github.com/stretchr/testify/assert" +) + +var clientDeserializer = scheme.Codecs.UniversalDeserializer() + +type testResource struct { + manifest string + name string + namespace string + podTemplateSpec *corev1.PodTemplateSpec +} + +type assertFunc func(*admv1.AdmissionResponse) + +func TestMutateOK(t *testing.T) { + err := mutateWorkloads("../../test/workloads/ok/*.yaml", + func(resp *admv1.AdmissionResponse) { + assert.Condition(t, func() bool { + // Handle injection cases *and* also pod submitted without `inject: "true"` annotation + if (resp.Allowed && resp.Patch != nil && resp.Result == nil) || (resp.Allowed && resp.Patch == nil && resp.Result == nil) { + return true + } + + return false + }, "Inconsistent AdmissionResponse") + + if resp.Patch != nil { + var patch []ctx.PatchOperation + if err := yaml.Unmarshal(resp.Patch, &patch); err != nil { + t.Errorf("JSON Patch unmarshal error \"%s\"", err) + } + + klog.Infof("JSON Patch=%+v", patch) + } + }) + + if err != nil { + t.Fatalf("%s", err) + } +} + +func TestMutateKO(t *testing.T) { + err := mutateWorkloads("../../test/workloads/ko/*.yaml", + func(resp *admv1.AdmissionResponse) { + assert.Condition(t, func() bool { + // Handle error cases + if !resp.Allowed && resp.Patch == nil && resp.Result != nil { + return true + } + + return false + }, "Inconsistent AdmissionResponse") + + klog.Infof("Result=%+v", resp.Result) + }) + + if err != nil { + t.Fatalf("%s", err) + } +} + +func mutateWorkloads(manifestsPattern string, test assertFunc) error { + verbose, _ := strconv.ParseBool(os.Getenv("VERBOSE")) + if verbose { + // Set Klog verbosity level to have detailed logs from our webhook (where we use level 5+ to log such info) + klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(klogFlags) + klogFlags.Set("v", "5") + } + + // Create webhook instance + vaultInjector, err := createTestVaultInjector() + if err != nil { + return fmt.Errorf("Loading error: %s", err) + } + + // Get all test workloads + workloads, err := filepath.Glob(manifestsPattern) + if err != nil { + return fmt.Errorf("Fail listing files: %s", err) + } + + // Loop on all test workloads: mutate and display JSON Patch structure + for _, workloadManifest := range workloads { + klog.Info("================================================================================================") + klog.Infof("Loading workload %s", workloadManifest) + + ar, err := (&testResource{manifest: workloadManifest}).load() + if err != nil { + return fmt.Errorf("Error creating AR: %s", err) + } + + // Mutate pod and test result + test(vaultInjector.mutate(ar)) + } + + return nil +} + +func createTestVaultInjector() (*VaultInjector, error) { + vsiCfg, err := cfg.Load( + cfg.WhSvrParameters{ + Port: 0, MetricsPort: 0, + CACertFile: "", CertFile: "", KeyFile: "", + WebhookCfgName: "", + AnnotationKeyPrefix: "sidecar.vault.talend.org", AppLabelKey: "com.talend.application", AppServiceLabelKey: "com.talend.service", + InjectionCfgFile: "../../test/config/injectionconfig.yaml", + ProxyCfgFile: "../../test/config/proxyconfig.hcl", + TemplateBlockFile: "../../test/config/tmplblock.hcl", + TemplateDefaultFile: "../../test/config/tmpldefault.tmpl", + PodLifecycleHooksFile: "../../test/config/podlifecyclehooks.yaml", + }, + ) + if err != nil { + return nil, err + } + + // Create webhook instance + return New(vsiCfg, nil), nil +} + +func (tr *testResource) load() (*admv1.AdmissionReview, error) { + data, err := ioutil.ReadFile(tr.manifest) + if err != nil { + return nil, err + } + + obj, _, err := clientDeserializer.Decode(data, nil, nil) + if err != nil { + return nil, err + } + + switch resource := obj.(type) { + // Beware: despite content being the same, golang does not support 'fallthrough' keyword in type switch (see https://stackoverflow.com/questions/11531264/why-isnt-fallthrough-allowed-in-a-type-switch) + case *appsv1.Deployment: // here 'resource' type is now *appsv1.Deployment + tr.name = resource.Name + tr.namespace = resource.Namespace + tr.podTemplateSpec = &resource.Spec.Template + case *batchv1.Job: // here 'resource' type is now *batchv1.Job + tr.name = resource.Name + tr.namespace = resource.Namespace + tr.podTemplateSpec = &resource.Spec.Template + default: + return nil, errors.New("Worload not supported") + } + + tr.addSATokenVolume() + return tr.createAdmissionReview() +} + +func (tr *testResource) addSATokenVolume() { + // We expect to find serviceaccount token volume. It is dynamically added to the pod by the Service Account Admission Controller. + // Add it manually here to pass internal check. + saTokenVolumeMount := corev1.VolumeMount{ + Name: "default-token-1234", + ReadOnly: true, + MountPath: k8sDefaultSATokenVolMountPath, + } + + saTokenVolume := corev1.Volume{ + Name: "default-token-1234", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "default-token", + }, + }, + } + + if len(tr.podTemplateSpec.Spec.Containers) > 0 { + tr.podTemplateSpec.Spec.Containers[0].VolumeMounts = append(tr.podTemplateSpec.Spec.Containers[0].VolumeMounts, saTokenVolumeMount) + } + + tr.podTemplateSpec.Spec.Volumes = append(tr.podTemplateSpec.Spec.Volumes, saTokenVolume) +} + +func (tr *testResource) createAdmissionReview() (*admv1.AdmissionReview, error) { + rawPod, err := json.Marshal(tr.podTemplateSpec) + if err != nil { + return nil, err + } + + ar := admv1.AdmissionReview{} + ar.SetGroupVersionKind(admv1.SchemeGroupVersion.WithKind("AdmissionReview")) + ar.Request = &admv1.AdmissionRequest{ + UID: uuid.NewUUID(), + Kind: metav1.GroupVersionKind{ + Version: "v1", + Kind: "Pod", + }, + Name: tr.name, + Namespace: tr.namespace, + Operation: admv1.Create, + Object: runtime.RawExtension{ + Raw: rawPod, + }, + } + + return &ar, nil +} diff --git a/pkg/webhook/utils.go b/pkg/webhook/utils.go index ecc5cec..2fcd342 100644 --- a/pkg/webhook/utils.go +++ b/pkg/webhook/utils.go @@ -1,4 +1,4 @@ -// Copyright © 2019-2020 Talend - www.talend.com +// Copyright © 2019-2021 Talend - www.talend.com // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,8 +20,10 @@ import ( ctx "talend/vault-sidecar-injector/pkg/context" - "k8s.io/api/admission/v1beta1" - admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" + admv1 "k8s.io/api/admission/v1" + admv1beta1 "k8s.io/api/admission/v1beta1" + admregv1 "k8s.io/api/admissionregistration/v1beta1" + admregv1beta1 "k8s.io/api/admissionregistration/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" @@ -29,8 +31,14 @@ import ( func init() { must(corev1.AddToScheme(runtimeScheme)) - must(v1beta1.AddToScheme(runtimeScheme)) - must(admissionregistrationv1beta1.AddToScheme(runtimeScheme)) + + // admission v1 + must(admv1.AddToScheme(runtimeScheme)) + must(admregv1.AddToScheme(runtimeScheme)) + + // admission v1beta1 + must(admv1beta1.AddToScheme(runtimeScheme)) + must(admregv1beta1.AddToScheme(runtimeScheme)) } func must(err error) { diff --git a/pkg/webhook/webhook-server.go b/pkg/webhook/webhook-server.go index ee2364a..79d4139 100644 --- a/pkg/webhook/webhook-server.go +++ b/pkg/webhook/webhook-server.go @@ -1,4 +1,4 @@ -// Copyright © 2019-2020 Talend - www.talend.com +// Copyright © 2019-2021 Talend - www.talend.com // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,11 +20,10 @@ import ( "io/ioutil" "net/http" cfg "talend/vault-sidecar-injector/pkg/config" - ctx "talend/vault-sidecar-injector/pkg/context" m "talend/vault-sidecar-injector/pkg/mode" - "k8s.io/api/admission/v1beta1" - corev1 "k8s.io/api/core/v1" + admv1 "k8s.io/api/admission/v1" + admv1beta1 "k8s.io/api/admission/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -65,142 +64,105 @@ func New(config *cfg.VSIConfig, server *http.Server) *VaultInjector { } } -// Create mutation patch for resources -func (vaultInjector *VaultInjector) createPatch(pod *corev1.Pod, annotations map[string]string) ([]byte, error) { - - patchPodSpec, err := vaultInjector.updatePodSpec(pod) - if err != nil { - return nil, err - } - - var patch []ctx.PatchOperation - - patch = append(patch, patchPodSpec...) - patch = append(patch, updateAnnotation(pod.Annotations, annotations)...) - - return json.Marshal(patch) -} - -// Main mutation process -func (vaultInjector *VaultInjector) mutate(ar *v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { - req := ar.Request - var pod corev1.Pod - var podName string - var podNamespace string - - if klog.V(5) { // enabled by providing '-v=5' at least - klog.Infof("Request=%+v", req) - } - - if err := json.Unmarshal(req.Object.Raw, &pod); err != nil { - klog.Errorf("Could not unmarshal raw object: %v", err) - return &v1beta1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, - } - } +// Serve method for webhook server +func (vaultInjector *VaultInjector) Serve(w http.ResponseWriter, r *http.Request) { + var body []byte if klog.V(5) { // enabled by providing '-v=5' at least - klog.Infof("Pod=%+v", pod) - } - - if pod.Name == "" { - podName = pod.GenerateName - } else { - podName = pod.Name - } - - if pod.Namespace == "" { - podNamespace = metav1.NamespaceDefault - } else { - podNamespace = pod.Namespace - } - - klog.Infof("AdmissionReview for GroupVersionKind=%+v, Namespace=%v Name=%v (%v) UID=%v patchOperation=%v UserInfo=%+v", - req.Kind, req.Namespace, req.Name, podName, req.UID, req.Operation, req.UserInfo) - - // Determine whether to perform mutation - if !mutationRequired(ignoredNamespaces, vaultInjector.VaultInjectorAnnotationsFQ, &pod.ObjectMeta) { - klog.Infof("Skipping mutation for %s/%s due to policy check", podNamespace, podName) - return &v1beta1.AdmissionResponse{ - Allowed: true, - } + klog.Infof("HTTP Request=%+v", r) } - annotations := map[string]string{vaultInjector.VaultInjectorAnnotationsFQ[ctx.VaultInjectorAnnotationStatusKey]: ctx.VaultInjectorStatusInjected} - patchBytes, err := vaultInjector.createPatch(&pod, annotations) - if err != nil { - return &v1beta1.AdmissionResponse{ - Allowed: false, - Result: &metav1.Status{ - Message: err.Error(), - }, - } - } - - klog.Infof("AdmissionResponse: patch=%v\n", string(patchBytes)) - return &v1beta1.AdmissionResponse{ - Allowed: true, - Patch: patchBytes, - PatchType: func() *v1beta1.PatchType { - pt := v1beta1.PatchTypeJSONPatch - return &pt - }(), - } -} - -// Serve method for webhook server -func (vaultInjector *VaultInjector) Serve(w http.ResponseWriter, r *http.Request) { - var body []byte if r.Body != nil { if data, err := ioutil.ReadAll(r.Body); err == nil { body = data } } + if len(body) == 0 { - klog.Error("empty body") - http.Error(w, "empty body", http.StatusBadRequest) + klog.Error("Empty body") + http.Error(w, "Empty body", http.StatusBadRequest) return } - // Verify the content type is accurate contentType := r.Header.Get("Content-Type") if contentType != "application/json" { klog.Errorf("Content-Type=%s, expect application/json", contentType) - http.Error(w, "invalid Content-Type, expect `application/json`", http.StatusUnsupportedMediaType) + http.Error(w, "Invalid Content-Type, expect `application/json`", http.StatusUnsupportedMediaType) return } - var admissionResponse *v1beta1.AdmissionResponse - ar := v1beta1.AdmissionReview{} - if _, _, err := deserializer.Decode(body, nil, &ar); err != nil { + rawAR, gvk, err := deserializer.Decode(body, nil, nil) + if err != nil { klog.Errorf("Can't decode body: %v", err) - admissionResponse = &v1beta1.AdmissionResponse{ - Result: &metav1.Status{ - Message: err.Error(), - }, + http.Error(w, fmt.Sprintf("Cannot decode body: %v", err), http.StatusBadRequest) + return + } + + // Webhook can currently receive either v1 or v1beta1 AdmissionReview objects. + // v1 is the default, internal version in use by VSI (v1beta1 support will be removed). + // If v1beta1 is received, it will be converted into v1 and back to v1beta1 in response + // (as spec states response should use same version as request) + arInVersion := admv1.SchemeGroupVersion + arIn, isV1 := rawAR.(*admv1.AdmissionReview) + if !isV1 { + arInVersion = admv1beta1.SchemeGroupVersion + arInv1beta1, isv1beta1 := rawAR.(*admv1beta1.AdmissionReview) + if !isv1beta1 { + klog.Errorf("Unsupported AdmissionReview version %v", gvk.Version) + http.Error(w, fmt.Sprintf("Unsupported AdmissionReview version %v", gvk.Version), http.StatusBadRequest) + return + } + + if klog.V(5) { // enabled by providing '-v=5' at least + klog.Infof("Received AdmissionReview '%v' Request=%+v", arInv1beta1.GroupVersionKind(), arInv1beta1.Request) + } + + // Convert v1beta1 to v1 + arIn = &admv1.AdmissionReview{} + arIn.SetGroupVersionKind(admv1.SchemeGroupVersion.WithKind("AdmissionReview")) + Convert_v1beta1_AdmissionReview_To_admission_AdmissionReview(arInv1beta1, arIn) + + if klog.V(5) { // enabled by providing '-v=5' at least + klog.Infof("Converted AdmissionReview '%v' Request=%+v", arIn.GroupVersionKind(), arIn.Request) } } else { - admissionResponse = vaultInjector.mutate(&ar) + if klog.V(5) { // enabled by providing '-v=5' at least + klog.Infof("Received AdmissionReview '%v' Request=%+v", arIn.GroupVersionKind(), arIn.Request) + } } - admissionReview := v1beta1.AdmissionReview{} - if admissionResponse != nil { - admissionReview.Response = admissionResponse - if ar.Request != nil { - admissionReview.Response.UID = ar.Request.UID + arOut := &admv1.AdmissionReview{} + arOut.SetGroupVersionKind(admv1.SchemeGroupVersion.WithKind("AdmissionReview")) + arOut.Response = vaultInjector.mutate(arIn) + + var returnedAR interface{} + returnedAR = arOut + + // If v1 received + if arInVersion.Version == admv1.SchemeGroupVersion.Version { + if klog.V(5) { // enabled by providing '-v=5' at least + klog.Infof("Returned AdmissionReview '%v' Response=%+v", arOut.GroupVersionKind(), arOut.Response) + } + } else { // If v1beta1 received: convert v1 to v1beta1 + arOutv1beta1 := &admv1beta1.AdmissionReview{} + arOutv1beta1.SetGroupVersionKind(admv1beta1.SchemeGroupVersion.WithKind("AdmissionReview")) + Convert_admission_AdmissionReview_To_v1beta1_AdmissionReview(arOut, arOutv1beta1) + returnedAR = arOutv1beta1 + + if klog.V(5) { // enabled by providing '-v=5' at least + klog.Infof("Returned AdmissionReview '%v' Response=%+v", arOutv1beta1.GroupVersionKind(), arOutv1beta1.Response) } } - resp, err := json.Marshal(admissionReview) + response, err := json.Marshal(returnedAR) if err != nil { klog.Errorf("Can't encode response: %v", err) - http.Error(w, fmt.Sprintf("could not encode response: %v", err), http.StatusInternalServerError) + http.Error(w, fmt.Sprintf("Cannot encode response: %v", err), http.StatusInternalServerError) } - klog.Infof("Ready to write reponse ...") - if _, err := w.Write(resp); err != nil { + + klog.Infof("Write reponse ...") + if _, err := w.Write(response); err != nil { klog.Errorf("Can't write response: %v", err) - http.Error(w, fmt.Sprintf("could not write response: %v", err), http.StatusInternalServerError) + http.Error(w, fmt.Sprintf("Cannot write response: %v", err), http.StatusInternalServerError) } } diff --git a/pkg/webhook/webhook-server_test.go b/pkg/webhook/webhook-server_test.go index 7195836..1db00c6 100644 --- a/pkg/webhook/webhook-server_test.go +++ b/pkg/webhook/webhook-server_test.go @@ -1,4 +1,4 @@ -// Copyright © 2019-2020 Talend - www.talend.com +// Copyright © 2019-2021 Talend - www.talend.com // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,89 +15,20 @@ package webhook import ( - "encoding/json" - "errors" "flag" - "fmt" - "io/ioutil" + "net/http" + "net/http/httptest" "os" - "path/filepath" "strconv" - cfg "talend/vault-sidecar-injector/pkg/config" - ctx "talend/vault-sidecar-injector/pkg/context" + "strings" "testing" - "k8s.io/api/admission/v1beta1" - appsv1 "k8s.io/api/apps/v1" - authenticationv1 "k8s.io/api/authentication/v1" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/klog" - - "github.com/ghodss/yaml" "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/klog" ) -var clientDeserializer = scheme.Codecs.UniversalDeserializer() - -type testResource struct { - manifest string - podTemplateSpec *corev1.PodTemplateSpec -} - -type assertFunc func(*v1beta1.AdmissionResponse) - -func TestWebhookServerOK(t *testing.T) { - err := mutateWorkloads("../../test/workloads/ok/*.yaml", - func(resp *v1beta1.AdmissionResponse) { - assert.Condition(t, func() bool { - // Handle injection cases *and* also pod submitted without `inject: "true"` annotation - if (resp.Allowed && resp.Patch != nil && resp.Result == nil) || (resp.Allowed && resp.Patch == nil && resp.Result == nil) { - return true - } - - return false - }, "Inconsistent AdmissionResponse") - - if resp.Patch != nil { - var patch []ctx.PatchOperation - if err := yaml.Unmarshal(resp.Patch, &patch); err != nil { - t.Errorf("JSON Patch unmarshal error \"%s\"", err) - } - - klog.Infof("JSON Patch=%+v", patch) - } - }) - - if err != nil { - t.Fatalf("%s", err) - } -} - -func TestWebhookServerKO(t *testing.T) { - err := mutateWorkloads("../../test/workloads/ko/*.yaml", - func(resp *v1beta1.AdmissionResponse) { - assert.Condition(t, func() bool { - // Handle error cases - if !resp.Allowed && resp.Patch == nil && resp.Result != nil { - return true - } - - return false - }, "Inconsistent AdmissionResponse") - - klog.Infof("Result=%+v", resp.Result) - }) - - if err != nil { - t.Fatalf("%s", err) - } -} - -func mutateWorkloads(manifestsPattern string, test assertFunc) error { +func TestWebhookServer(t *testing.T) { verbose, _ := strconv.ParseBool(os.Getenv("VERBOSE")) if verbose { // Set Klog verbosity level to have detailed logs from our webhook (where we use level 5+ to log such info) @@ -107,126 +38,107 @@ func mutateWorkloads(manifestsPattern string, test assertFunc) error { } // Create webhook instance - vaultInjector, err := createVaultInjector() - if err != nil { - return fmt.Errorf("Loading error: %s", err) - } - - // Get all test workloads - workloads, err := filepath.Glob(manifestsPattern) + vaultInjector, err := createTestVaultInjector() if err != nil { - return fmt.Errorf("Fail listing files: %s", err) - } - - // Loop on all test workloads: mutate and display JSON Patch structure - for _, workloadManifest := range workloads { - klog.Info("================================================================================================") - klog.Infof("Loading workload %s", workloadManifest) - - ar, err := (&testResource{manifest: workloadManifest}).load() - if err != nil { - return fmt.Errorf("Error creating AR: %s", err) - } - - // Mutate pod and test result - test(vaultInjector.mutate(ar)) + t.Fatalf("Loading error: %s", err) } - return nil -} - -func createVaultInjector() (*VaultInjector, error) { - vsiCfg, err := cfg.Load( - cfg.WhSvrParameters{ - Port: 0, MetricsPort: 0, - CACertFile: "", CertFile: "", KeyFile: "", - WebhookCfgName: "", - AnnotationKeyPrefix: "sidecar.vault.talend.org", AppLabelKey: "com.talend.application", AppServiceLabelKey: "com.talend.service", - InjectionCfgFile: "../../test/config/injectionconfig.yaml", - ProxyCfgFile: "../../test/config/proxyconfig.hcl", - TemplateBlockFile: "../../test/config/tmplblock.hcl", - TemplateDefaultFile: "../../test/config/tmpldefault.tmpl", - PodLifecycleHooksFile: "../../test/config/podlifecyclehooks.yaml", + tables := []struct { + name string + admissionReviewVersion string + vaultInjection bool + statusCode int + }{ + { + name: "AdmissionReview v1, no injection", + admissionReviewVersion: "v1", + vaultInjection: false, + statusCode: http.StatusOK, }, - ) - if err != nil { - return nil, err - } - - // Create webhook instance - return New(vsiCfg, nil), nil -} - -func (tr *testResource) load() (*v1beta1.AdmissionReview, error) { - data, err := ioutil.ReadFile(tr.manifest) - if err != nil { - return nil, err - } - - obj, _, err := clientDeserializer.Decode(data, nil, nil) - if err != nil { - return nil, err - } - - switch resource := obj.(type) { - // Beware: despite content being the same, golang does not support 'fallthrough' keyword in type switch (see https://stackoverflow.com/questions/11531264/why-isnt-fallthrough-allowed-in-a-type-switch) - case *appsv1.Deployment: // here 'resource' type is now *appsv1.Deployment - tr.podTemplateSpec = &resource.Spec.Template - case *batchv1.Job: // here 'resource' type is now *batchv1.Job - tr.podTemplateSpec = &resource.Spec.Template - default: - return nil, errors.New("Worload not supported") - } - - tr.addSATokenVolume() - return tr.createAdmissionReview() -} - -func (tr *testResource) addSATokenVolume() { - // We expect to find serviceaccount token volume. It is dynamically added to the pod by the Service Account Admission Controller. - // Add it manually here to pass internal check. - saTokenVolumeMount := corev1.VolumeMount{ - Name: "default-token-1234", - ReadOnly: true, - MountPath: k8sDefaultSATokenVolMountPath, - } - - saTokenVolume := corev1.Volume{ - Name: "default-token-1234", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "default-token", - }, + { + name: "AdmissionReview v1", + admissionReviewVersion: "v1", + vaultInjection: true, + statusCode: http.StatusOK, + }, + { + name: "AdmissionReview v1beta1", + admissionReviewVersion: "v1beta1", + vaultInjection: true, + statusCode: http.StatusOK, + }, + { + name: "AdmissionReview v1beta2", + admissionReviewVersion: "v1beta2", + vaultInjection: true, + statusCode: http.StatusBadRequest, }, } - if len(tr.podTemplateSpec.Spec.Containers) > 0 { - tr.podTemplateSpec.Spec.Containers[0].VolumeMounts = append(tr.podTemplateSpec.Spec.Containers[0].VolumeMounts, saTokenVolumeMount) - } + for _, table := range tables { + t.Run(table.name, func(t *testing.T) { + uid := string(uuid.NewUUID()) + request := httptest.NewRequest(http.MethodPost, "/mutate", strings.NewReader(`{ + "kind":"AdmissionReview", + "apiVersion":"admission.k8s.io/`+table.admissionReviewVersion+`", + "request":{ + "uid":"`+uid+`", + "kind":{ + "group":"", + "version":"v1", + "kind":"Pod" + }, + "namespace":"default", + "operation":"CREATE", + "object":{ + "apiVersion":"v1", + "kind":"Pod", + "metadata":{ + "annotations":{ + "sidecar.vault.talend.org/inject": "`+strconv.FormatBool(table.vaultInjection)+`" + }, + "labels":{ + "com.talend.application": "test", + "com.talend.service": "test-app-svc" + } + }, + "spec":{ + "containers":[ + { + "name": "testcontainer", + "image": "myfakeimage:1.0.0", + "volumeMounts":[ + { + "name": "default-token-1234", + "mountPath" : "/var/run/secrets/kubernetes.io/serviceaccount" + } + ] + } + ] + } + } + } + }`)) + request.Header.Add("Content-Type", "application/json") + responseRecorder := httptest.NewRecorder() - tr.podTemplateSpec.Spec.Volumes = append(tr.podTemplateSpec.Spec.Volumes, saTokenVolume) -} + vaultInjector.Serve(responseRecorder, request) -func (tr *testResource) createAdmissionReview() (*v1beta1.AdmissionReview, error) { - rawPod, err := json.Marshal(tr.podTemplateSpec) - if err != nil { - return nil, err - } + if klog.V(5) { + klog.Infof("HTTP Response=%+v", responseRecorder) + } - return &v1beta1.AdmissionReview{ - Request: &v1beta1.AdmissionRequest{ - Kind: metav1.GroupVersionKind{ - Version: "v1", - Kind: "Pod", - }, - Namespace: tr.podTemplateSpec.GetNamespace(), - Operation: v1beta1.Create, - UserInfo: authenticationv1.UserInfo{ - Username: "vault-sidecar-injector", - }, - Object: runtime.RawExtension{ - Raw: rawPod, - }, - }, - }, nil + assert.Equal(t, responseRecorder.Code, table.statusCode) + assert.Condition(t, func() bool { + if responseRecorder.Code == http.StatusOK { + return strings.Contains(responseRecorder.Body.String(), + `"kind":"AdmissionReview","apiVersion":"admission.k8s.io/`+table.admissionReviewVersion+`"`) && + strings.Contains(responseRecorder.Body.String(), + `"response":{"uid":"`+uid+`"`) + } else { + return true // HTTP error: return true to skip this test + } + }, "AdmissionReview version must match received version and admission response UID must match admission request UID") + }) + } }