From 7427322f730a4d478227299e65af9e1e2a4d2551 Mon Sep 17 00:00:00 2001 From: Matthias Diester Date: Thu, 30 Jan 2020 15:14:21 +0100 Subject: [PATCH] Consume Quarks Links as environment variables [#170725085](https://www.pivotaltracker.com/story/show/170725085) Using Quarks links (entanglements) on the consumer side required changes in the operator so that a link property can be used through an environment variable and as a file with just the property content. In theory, this should enable consumers to retrieve the property value in their respective project with minimal adjustments. Introduce flattened link properties in the Kubernetes secrets that contain the link properties. The name and the data of the link secret change from `link-deploymentname-name` ``` "nats.nats": "{\"nats\": { \"port\": 4442}, ... }" ``` to a new flatten look and name `link-deploymentname-type-name` ``` "nats.port": "4442" "nats.user": "admin" ``` The link type and name become part of the secret name and will not be used in the secret data as the root key anymore. Therefore, all code sections and tests that wait or rely on the old link secret name were updated to use the new names. This relies on new convenience functions to create the name for both the link secret name and the type/name pair. The `cmd_instance_group_resolver` code and tests were updated to write the flattened properties in the `provides.json`, which is later used by a Quarks Job to persist the data into the link secrets. The `job_factory` code and tests were updated to use the new "fan-out" style of the Quarks Job project that writes the content of the aforementioned `provides.json` into separate link secrets, each only containing one key/value pair. With other words, there will be one secret per each link type/name tuple. The `pod_mutator` code and tests were changed to not mount the link properties as one file (`link.yaml`) anymore, but that each entry in the link properties becomes its own file in the target container. Also, each entry in the properties will result in an environment variable starting with the prefix `LINK_` and the respective value. For example, the nats username will be exposed as `LINK_NATS_USER` in the containers. Update documentation to include the changes with respect to the newly added environment variables that are based on the link secrets. --- bin/include/dependencies | 2 +- docs/entanglements.md | 35 +++++++---- e2e/kube/bosh_link_entangled_pods_test.go | 26 ++++---- go.mod | 4 +- go.sum | 4 ++ integration/quarks_link_entangled_pod_test.go | 40 +++++-------- integration/quarks_link_from_bosh_test.go | 18 +++++- pkg/bosh/converter/job_factory.go | 6 +- pkg/bosh/converter/job_factory_test.go | 14 ++++- .../manifest/cmd_instance_group_resolver.go | 33 ++++++++++- .../cmd_instance_group_resolver_test.go | 16 ++++- pkg/bosh/manifest/job_provider_links.go | 2 +- .../controllers/quarkslink/entanglement.go | 24 +++----- .../controllers/quarkslink/pod_mutator.go | 59 +++++++++++++------ .../quarkslink/pod_mutator_test.go | 16 ++--- testing/catalog.go | 20 ++++--- 16 files changed, 208 insertions(+), 111 deletions(-) diff --git a/bin/include/dependencies b/bin/include/dependencies index bc19c41d6..f7b46b39b 100644 --- a/bin/include/dependencies +++ b/bin/include/dependencies @@ -1,6 +1,6 @@ #!/bin/bash -git_sha="6a177e8" +git_sha="62fcf84" quarks_job_release="v0.0.0-0.g$git_sha" # QUARKS_JOB_IMAGE_TAG is used for integration tests diff --git a/docs/entanglements.md b/docs/entanglements.md index 023b18ae6..00ebdefd4 100644 --- a/docs/entanglements.md +++ b/docs/entanglements.md @@ -22,7 +22,7 @@ We construct link information like so: > If multiple secrets or services are found with the same link information, the operator should error -### Example +### Example (Native -> BOSH) ```yaml kind: Secret @@ -37,7 +37,7 @@ spec: Using this secret, I should be able to use `link("nats").p("password")` in one of my BOSH templates. -``` +```yaml apiVersion: v1 kind: Service metadata: @@ -66,20 +66,35 @@ If the service is changed, or the list of pods selected by the service is change In this case, the BOSH component is a provider, and the native component is a consumer. -The operator creates link Secrets for all providers in a BOSH deployment. +The operator creates link secrets for all providers in a BOSH deployment. Each secret contains a flattened map with the provided properties: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: link-test-nats-nats +data: + nats.password: YXBwYXJlbnRseSwgeW91Cg== + nats.port: aGF2ZSB0b28K + nats.user: bXVjaCB0aW1lCg== +``` If a pod is annotated with the following: - - `quarks.cloudfoundry.org/deployment: foo` - - `quarks.cloudfoundry.org/consumes: '[{"name":"nats","type":"nats"}]'` -The operator will: - - mutate the pod and mount the secret as `/quarks/link/DEPLOYMENT//link.yaml` -Where `` is the name of the link, e.g. 'nats'. +- `quarks.cloudfoundry.org/deployment: foo` +- `quarks.cloudfoundry.org/consumes: '[{"name":"nats","type":"nats"}]'` + +The operator will mutate the pod to: + +- mount the link secrets as `/quarks/link/DEPLOYMENT/./` +- add an environment variable for each key in the secret data mapping: `LINK_` + +The `` and `` are the respective link type and name, for example the nats release uses `nats` for both the name and the type of the link. Whereas the `` describes the BOSH properties style flattened property key with its dot-style, for example `nats.password`. The key name is modified to be all upper case and without dots in the context of an environment variable, therefore `nats.password` becomes `LINK_NATS_PASSWORD` in the container. If link information changes, the operator will trigger an update (restart) of the deployment or statefulset owning the pod. This can be done by updating the template of the pod using an annotation. -## Example +### Example (BOSH -> Native) an Eirini Helm Chart @@ -94,6 +109,7 @@ The OPI process of Eirini required the NATS password and IP. spec: ``` + and a CF-Deployment with Operator Instance Groups: @@ -102,4 +118,3 @@ Instance Groups: - Gorouter - NATS provides: nats - diff --git a/e2e/kube/bosh_link_entangled_pods_test.go b/e2e/kube/bosh_link_entangled_pods_test.go index a6e994d40..e9e39eb90 100644 --- a/e2e/kube/bosh_link_entangled_pods_test.go +++ b/e2e/kube/bosh_link_entangled_pods_test.go @@ -11,18 +11,15 @@ import ( ) var _ = Describe("BOSHLinkEntanglements", func() { - const jobProperties = `{"nats":{"password":"onetwothreefour","port":4222,"user":"admin"}}` - const changedProperties = `{"nats":{"password":"qwerty1234","port":4222,"user":"admin"}}` - apply := func(p string) error { yamlPath := path.Join(examplesDir, p) return cmdHelper.Apply(namespace, yamlPath) } - checkEntanglement := func(podName, expect string) error { + checkEntanglement := func(podName, cmd, expect string) error { return kubectl.RunCommandWithCheckString( namespace, podName, - "cat /quarks/link/nats-deployment/nats/link.yaml", + cmd, expect, ) } @@ -44,7 +41,7 @@ var _ = Describe("BOSHLinkEntanglements", func() { Context("when creating a bosh deployment", func() { It("creates secrets for a all BOSH links", func() { - exist, err := kubectl.SecretExists(namespace, "link-nats-deployment-nats") + exist, err := kubectl.SecretExists(namespace, "link-nats-deployment-nats-nats") Expect(err).ToNot(HaveOccurred()) Expect(exist).To(BeTrue()) }) @@ -61,8 +58,9 @@ var _ = Describe("BOSHLinkEntanglements", func() { err := apply("quarks-link/entangled-sts.yaml") Expect(err).ToNot(HaveOccurred()) podWait(selector) - err = checkEntanglement(podName, jobProperties) - Expect(err).ToNot(HaveOccurred()) + + Expect(checkEntanglement(podName, "cat /quarks/link/nats-deployment/nats-nats/nats.password", "onetwothreefour")).ToNot(HaveOccurred()) + Expect(checkEntanglement(podName, "echo $LINK_NATS_USER", "admin")).ToNot(HaveOccurred()) }) By("restarting pods when the link secret changes", func() { @@ -76,8 +74,8 @@ var _ = Describe("BOSHLinkEntanglements", func() { Expect(err).ToNot(HaveOccurred(), "waiting for restart annotation on entangled pod") podWait(selector) - err = checkEntanglement(podName, changedProperties) - Expect(err).ToNot(HaveOccurred()) + Expect(checkEntanglement(podName, "cat /quarks/link/nats-deployment/nats-nats/nats.password", "qwerty1234")).ToNot(HaveOccurred()) + Expect(checkEntanglement(podName, "echo $LINK_NATS_USER", "admin")).ToNot(HaveOccurred()) }) }) }) @@ -97,8 +95,8 @@ var _ = Describe("BOSHLinkEntanglements", func() { podName = getPodName(selector) podWait("pod/" + podName) - err = checkEntanglement(podName, jobProperties) - Expect(err).ToNot(HaveOccurred()) + Expect(checkEntanglement(podName, "cat /quarks/link/nats-deployment/nats-nats/nats.password", "onetwothreefour")).ToNot(HaveOccurred()) + Expect(checkEntanglement(podName, "echo $LINK_NATS_USER", "admin")).ToNot(HaveOccurred()) }) By("restarting pods when the link secret changes", func() { @@ -119,8 +117,8 @@ var _ = Describe("BOSHLinkEntanglements", func() { ) Expect(err).ToNot(HaveOccurred(), "waiting for restart annotation on entangled pod") - err = checkEntanglement(podName, changedProperties) - Expect(err).ToNot(HaveOccurred()) + Expect(checkEntanglement(podName, "cat /quarks/link/nats-deployment/nats-nats/nats.password", "qwerty1234")).ToNot(HaveOccurred()) + Expect(checkEntanglement(podName, "echo $LINK_NATS_USER", "admin")).ToNot(HaveOccurred()) }) }) }) diff --git a/go.mod b/go.mod index 392d7cb6b..3f3f27c24 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module code.cloudfoundry.org/cf-operator require ( - code.cloudfoundry.org/quarks-job v0.0.0-20200128080450-0deec9592a1e - code.cloudfoundry.org/quarks-utils v0.0.0-20200128080244-7f8de3f1673c + code.cloudfoundry.org/quarks-job v0.0.0-20200130141800-25a764f89866 + code.cloudfoundry.org/quarks-utils v0.0.0-20200130110052-eac67a73088b github.com/beorn7/perks v1.0.1 // indirect github.com/bmatcuk/doublestar v1.1.1 // indirect github.com/charlievieth/fs v0.0.0-20170613215519-7dc373669fa1 // indirect diff --git a/go.sum b/go.sum index 588223960..666b3e3cf 100644 --- a/go.sum +++ b/go.sum @@ -3,10 +3,14 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= code.cloudfoundry.org/quarks-job v0.0.0-20200128080450-0deec9592a1e h1:eKxuDhITX02+q4+NagYT/eakaDk9IMNUgGt6Dlno2bk= code.cloudfoundry.org/quarks-job v0.0.0-20200128080450-0deec9592a1e/go.mod h1:fLlgnrSanve71EtkVswD8hoI6U2RYM4+0S3A2ajvTEI= +code.cloudfoundry.org/quarks-job v0.0.0-20200130141800-25a764f89866 h1:EU2BCSJjcQLpqvkcJ8ShXS+LUb3OTrsmHJqeLR9bwP8= +code.cloudfoundry.org/quarks-job v0.0.0-20200130141800-25a764f89866/go.mod h1:fLlgnrSanve71EtkVswD8hoI6U2RYM4+0S3A2ajvTEI= code.cloudfoundry.org/quarks-utils v0.0.0-20200127150718-47028dacbc7c h1:nEbvnRc7JUgzLFfopuvdPBhT4nHQulxg8iVVPxuVZOM= code.cloudfoundry.org/quarks-utils v0.0.0-20200127150718-47028dacbc7c/go.mod h1:d2OaSM1qVE/7Zo1imovL7CZCOAShFePFMI3jlpMcp14= code.cloudfoundry.org/quarks-utils v0.0.0-20200128080244-7f8de3f1673c h1:m0VovyZz1Ny2OtGn3siS6Q7JumekLW0hvIwoeKadl4c= code.cloudfoundry.org/quarks-utils v0.0.0-20200128080244-7f8de3f1673c/go.mod h1:d2OaSM1qVE/7Zo1imovL7CZCOAShFePFMI3jlpMcp14= +code.cloudfoundry.org/quarks-utils v0.0.0-20200130110052-eac67a73088b h1:vngvL8ncDz/ihUgnNZplK1gCwnxDppucvBLeYZKFqVc= +code.cloudfoundry.org/quarks-utils v0.0.0-20200130110052-eac67a73088b/go.mod h1:d2OaSM1qVE/7Zo1imovL7CZCOAShFePFMI3jlpMcp14= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= diff --git a/integration/quarks_link_entangled_pod_test.go b/integration/quarks_link_entangled_pod_test.go index 66221fcd8..4cabba26e 100644 --- a/integration/quarks_link_entangled_pod_test.go +++ b/integration/quarks_link_entangled_pod_test.go @@ -29,16 +29,6 @@ var _ = Describe("Entangled Pods PodMutator", func() { return names } - volumeKeyToPaths := func(volumes []corev1.Volume) []string { - keys := make([]string, len(volumes)) - for i, v := range volumes { - if len(v.Secret.Items) > 0 { - keys[i] = v.Secret.Items[0].Key - } - } - return keys - } - volumeMountNames := func(mounts []corev1.VolumeMount) []string { names := make([]string, len(mounts)) for i, m := range mounts { @@ -87,12 +77,11 @@ var _ = Describe("Entangled Pods PodMutator", func() { Expect(err).NotTo(HaveOccurred()) Expect(p.Spec.Volumes).To(HaveLen(2)) - Expect(volumeNames(p.Spec.Volumes)).To(ContainElement("link-nats-deployment-nats")) - Expect(volumeKeyToPaths(p.Spec.Volumes)).To(ContainElement("nats.nats")) + Expect(volumeNames(p.Spec.Volumes)).To(ContainElement("link-nats-deployment-nats-nats")) for _, c := range p.Spec.Containers { Expect(c.VolumeMounts).To(HaveLen(2)) - Expect(volumeMountNames(c.VolumeMounts)).To(ContainElement("link-nats-deployment-nats")) + Expect(volumeMountNames(c.VolumeMounts)).To(ContainElement("link-nats-deployment-nats-nats")) } }) }) @@ -105,10 +94,15 @@ var _ = Describe("Entangled Pods PodMutator", func() { tearDowns = append(tearDowns, tearDown) otherSecret := env.QuarksLinkSecret( - deploymentName, "ig", - "type", "name", - `{"foo":[1,2,3],{"password":"abc"}}`, + deploymentName, + "type", + "name", + map[string][]byte{ + "foo": []byte("[1,2,3]"), + "password": []byte("abc"), + }, ) + tearDown, err = env.CreateSecret(env.Namespace, otherSecret) Expect(err).NotTo(HaveOccurred()) tearDowns = append(tearDowns, tearDown) @@ -125,15 +119,14 @@ var _ = Describe("Entangled Pods PodMutator", func() { Expect(err).NotTo(HaveOccurred()) Expect(p.Spec.Volumes).To(HaveLen(3)) - Expect(volumeNames(p.Spec.Volumes)).To(ContainElement("link-nats-deployment-nats")) - Expect(volumeNames(p.Spec.Volumes)).To(ContainElement("link-nats-deployment-ig")) - Expect(volumeKeyToPaths(p.Spec.Volumes)).To(ContainElement("type.name")) + Expect(volumeNames(p.Spec.Volumes)).To(ContainElement("link-nats-deployment-nats-nats")) + Expect(volumeNames(p.Spec.Volumes)).To(ContainElement("link-nats-deployment-type-name")) for _, c := range p.Spec.Containers { Expect(c.VolumeMounts).To(HaveLen(3)) mounts := c.VolumeMounts - Expect(volumeMountNames(mounts)).To(ContainElement("link-nats-deployment-nats")) - Expect(volumeMountNames(mounts)).To(ContainElement("link-nats-deployment-ig")) + Expect(volumeMountNames(mounts)).To(ContainElement("link-nats-deployment-nats-nats")) + Expect(volumeMountNames(mounts)).To(ContainElement("link-nats-deployment-type-name")) } }) }) @@ -176,12 +169,11 @@ var _ = Describe("Entangled Pods PodMutator", func() { Expect(err).NotTo(HaveOccurred()) Expect(p.Spec.Volumes).To(HaveLen(2)) - Expect(volumeNames(p.Spec.Volumes)).To(ContainElement("link-nats-deployment-nats")) - Expect(volumeKeyToPaths(p.Spec.Volumes)).To(ContainElement("nats.nats")) + Expect(volumeNames(p.Spec.Volumes)).To(ContainElement("link-nats-deployment-nats-nats")) for _, c := range p.Spec.Containers { Expect(c.VolumeMounts).To(HaveLen(2)) - Expect(volumeMountNames(c.VolumeMounts)).To(ContainElement("link-nats-deployment-nats")) + Expect(volumeMountNames(c.VolumeMounts)).To(ContainElement("link-nats-deployment-nats-nats")) } }) }) diff --git a/integration/quarks_link_from_bosh_test.go b/integration/quarks_link_from_bosh_test.go index a82f7acdf..05b3c2549 100644 --- a/integration/quarks_link_from_bosh_test.go +++ b/integration/quarks_link_from_bosh_test.go @@ -7,6 +7,7 @@ import ( corev1 "k8s.io/api/core/v1" bm "code.cloudfoundry.org/cf-operator/testing/boshmanifest" + "code.cloudfoundry.org/quarks-utils/pkg/names" "code.cloudfoundry.org/quarks-utils/testing/machine" ) @@ -14,7 +15,6 @@ var _ = Describe("BOSHLinks", func() { const ( manifestRef = "manifest" deploymentName = "test" - secretName = "link-test-nats" ) var ( @@ -43,12 +43,18 @@ var _ = Describe("BOSHLinks", func() { }) It("creates a secret for each link found in jobs", func() { + secretName := names.QuarksLinkSecretName(deploymentName, "nats", "nats") + By("waiting for secrets", func() { err := env.WaitForSecret(env.Namespace, secretName) Expect(err).NotTo(HaveOccurred()) secret, err := env.GetSecret(env.Namespace, secretName) Expect(err).NotTo(HaveOccurred()) - Expect(secret.Data).Should(HaveKeyWithValue("nats.nats", []byte("{\"nats\":{\"password\":\"changeme\",\"port\":4222,\"user\":\"admin\"}}"))) + Expect(secret.Data).To(Equal(map[string][]byte{ + "nats.password": []byte("changeme"), + "nats.port": []byte("4222"), + "nats.user": []byte("admin"), + })) }) }) }) @@ -59,12 +65,18 @@ var _ = Describe("BOSHLinks", func() { }) It("creates a secret for each link found in jobs", func() { + secretName := names.QuarksLinkSecretName(deploymentName, "nats", "nuts") + By("waiting for secrets", func() { err := env.WaitForSecret(env.Namespace, secretName) Expect(err).NotTo(HaveOccurred()) secret, err := env.GetSecret(env.Namespace, secretName) Expect(err).NotTo(HaveOccurred()) - Expect(secret.Data).Should(HaveKeyWithValue("nats.nuts", []byte("{\"nats\":{\"password\":\"changeme\",\"port\":4222,\"user\":\"admin\"}}"))) + Expect(secret.Data).To(Equal(map[string][]byte{ + "nats.password": []byte("changeme"), + "nats.port": []byte("4222"), + "nats.user": []byte("admin"), + })) }) }) }) diff --git a/pkg/bosh/converter/job_factory.go b/pkg/bosh/converter/job_factory.go index 13c9f5636..1424362df 100644 --- a/pkg/bosh/converter/job_factory.go +++ b/pkg/bosh/converter/job_factory.go @@ -179,7 +179,7 @@ func (f *JobFactory) InstanceGroupManifestJob(manifest bdm.Manifest, linkInfos L if ig.Instances != 0 { // Additional secret for BOSH links per instance group containerName := names.Sanitize(ig.Name) - linkOutputs[containerName] = names.EntanglementSecretName(manifest.Name, ig.Name) + linkOutputs[containerName] = names.QuarksLinkSecretName(manifest.Name) // One container per instance group containers = append(containers, ct.newUtilContainer(ig.Name, linkInfos.VolumeMounts())) @@ -195,9 +195,11 @@ func (f *JobFactory) InstanceGroupManifestJob(manifest bdm.Manifest, linkInfos L // add the BOSH link secret to the output list of each container for container, secret := range linkOutputs { qJob.Spec.Output.OutputMap[container]["provides.json"] = qjv1a1.SecretOptions{ - Name: secret, + Name: secret, + PersistenceMethod: qjv1a1.PersistUsingFanOut, } } + return qJob, nil } diff --git a/pkg/bosh/converter/job_factory_test.go b/pkg/bosh/converter/job_factory_test.go index a03bbe1b3..521379f3f 100644 --- a/pkg/bosh/converter/job_factory_test.go +++ b/pkg/bosh/converter/job_factory_test.go @@ -90,14 +90,19 @@ var _ = Describe("JobFactory", func() { Name: "foo-deployment.ig-resolved.redis-slave", AdditionalSecretLabels: map[string]string{"quarks.cloudfoundry.org/secret-type": "ig-resolved"}, Versioned: true, + PersistenceMethod: "", }, "bpm.json": qjv1a1.SecretOptions{ Name: "foo-deployment.bpm.redis-slave", AdditionalSecretLabels: map[string]string{"quarks.cloudfoundry.org/secret-type": "bpm"}, Versioned: true, + PersistenceMethod: "", }, "provides.json": qjv1a1.SecretOptions{ - Name: "link-foo-deployment-redis-slave", + Name: "link-foo-deployment", + AdditionalSecretLabels: nil, + Versioned: false, + PersistenceMethod: "fan-out", }, }, "diego-cell": qjv1a1.FilesToSecrets{ @@ -105,14 +110,19 @@ var _ = Describe("JobFactory", func() { Name: "foo-deployment.ig-resolved.diego-cell", AdditionalSecretLabels: map[string]string{"quarks.cloudfoundry.org/secret-type": "ig-resolved"}, Versioned: true, + PersistenceMethod: "", }, "bpm.json": qjv1a1.SecretOptions{ Name: "foo-deployment.bpm.diego-cell", AdditionalSecretLabels: map[string]string{"quarks.cloudfoundry.org/secret-type": "bpm"}, Versioned: true, + PersistenceMethod: "", }, "provides.json": qjv1a1.SecretOptions{ - Name: "link-foo-deployment-diego-cell", + Name: "link-foo-deployment", + AdditionalSecretLabels: nil, + Versioned: false, + PersistenceMethod: "fan-out", }, }, }, diff --git a/pkg/bosh/manifest/cmd_instance_group_resolver.go b/pkg/bosh/manifest/cmd_instance_group_resolver.go index 9d251a9ab..3ba105d0c 100644 --- a/pkg/bosh/manifest/cmd_instance_group_resolver.go +++ b/pkg/bosh/manifest/cmd_instance_group_resolver.go @@ -151,7 +151,7 @@ func (igr *InstanceGroupResolver) SaveLinks(path string) error { var result = map[string]string{} for id, property := range properties { - jsonBytes, err := json.Marshal(property) + jsonBytes, err := json.Marshal(flattenForSecretData(property)) if err != nil { return errors.Wrapf(err, "JSON marshalling failed for ig '%s' property '%s'", igName, id) } @@ -628,3 +628,34 @@ func getQuarksLinkFromMap(m map[string]interface{}) (QuarksLink, error) { err = json.Unmarshal(data, &result) return result, err } + +func traverse(path string, obj interface{}, leafFunc func(path string, value interface{})) { + appendPath := func(new string) string { + if len(path) == 0 { + return new + } + + return fmt.Sprintf("%s.%s", path, new) + } + + switch tobj := obj.(type) { + case map[string]interface{}: + for key, value := range tobj { + traverse(appendPath(key), value, leafFunc) + } + + default: + leafFunc(path, tobj) + } +} + +func flattenForSecretData(property JobLinkProperties) map[string]string { + tmp := map[string]string{} + for k, v := range property { + traverse(k, v, func(path string, value interface{}) { + tmp[path] = fmt.Sprintf("%v", value) + }) + } + + return tmp +} diff --git a/pkg/bosh/manifest/cmd_instance_group_resolver_test.go b/pkg/bosh/manifest/cmd_instance_group_resolver_test.go index c294861bd..785947b04 100644 --- a/pkg/bosh/manifest/cmd_instance_group_resolver_test.go +++ b/pkg/bosh/manifest/cmd_instance_group_resolver_test.go @@ -318,6 +318,18 @@ var _ = Describe("InstanceGroupResolver", func() { Describe("SaveLinks", func() { Context("when jobs provide links", func() { + var fileContentOf = func(path string) map[string]string { + Expect(afero.Exists(fs, path)).To(BeTrue()) + + bytes, err := afero.ReadFile(fs, path) + Expect(err).ToNot(HaveOccurred()) + + var data map[string]string + Expect(json.Unmarshal(bytes, &data)).ToNot(HaveOccurred()) + + return data + } + BeforeEach(func() { m, err = env.BOSHManifestWithLinks() Expect(err).NotTo(HaveOccurred()) @@ -331,7 +343,9 @@ var _ = Describe("InstanceGroupResolver", func() { err = igr.SaveLinks("/mnt/quarks") Expect(err).ToNot(HaveOccurred()) - Expect(afero.Exists(fs, "/mnt/quarks/provides.json")).To(BeTrue()) + Expect(fileContentOf("/mnt/quarks/provides.json")).To(Equal(map[string]string{ + "nats-nuts": `{"nats.password":"changeme","nats.port":"4222","nats.user":"admin"}`, + })) }) }) }) diff --git a/pkg/bosh/manifest/job_provider_links.go b/pkg/bosh/manifest/job_provider_links.go index c93341b19..b6b51e270 100644 --- a/pkg/bosh/manifest/job_provider_links.go +++ b/pkg/bosh/manifest/job_provider_links.go @@ -101,7 +101,7 @@ func (jpl jobProviderLinks) Add(igName string, job Job, spec JobSpec, jobsInstan if _, ok := jpl.instanceGroups[igName]; !ok { jpl.instanceGroups[igName] = map[string]JobLinkProperties{} } - jpl.instanceGroups[igName][names.EntanglementSecretKey(linkType, linkName)] = properties + jpl.instanceGroups[igName][names.QuarksLinkSecretKey(linkType, linkName)] = properties } return nil } diff --git a/pkg/kube/controllers/quarkslink/entanglement.go b/pkg/kube/controllers/quarkslink/entanglement.go index f06341bee..4f74e37ac 100644 --- a/pkg/kube/controllers/quarkslink/entanglement.go +++ b/pkg/kube/controllers/quarkslink/entanglement.go @@ -3,7 +3,6 @@ package quarkslink import ( "encoding/json" "fmt" - "regexp" corev1 "k8s.io/api/core/v1" @@ -16,8 +15,9 @@ var ( // DeploymentKey is the key to retrieve the name of the deployment, // which provides the variables for the pod DeploymentKey = fmt.Sprintf("%s/deployment", apis.GroupName) - // ConsumesKey is the key for identifying the provider to be consumed, in the - // format of 'type.job' + + // ConsumesKey is the key for identifying the provider to be consumed, in + // the format of: '[{"name":"","type":""}]' (JSON string) ConsumesKey = fmt.Sprintf("%s/consumes", apis.GroupName) ) @@ -52,13 +52,13 @@ func newLinks(value string) (links, error) { } type link struct { - Name string `json:"name"` - LinkType string `json:"type"` - secretName string + Name string `json:"name"` + LinkType string `json:"type"` + secret *corev1.Secret } func (l link) String() string { - return names.EntanglementSecretKey(l.LinkType, l.Name) + return names.QuarksLinkSecretKey(l.LinkType, l.Name) } type links []link @@ -91,17 +91,11 @@ func (e entanglement) find(secret corev1.Secret) (link, bool) { return link{}, false } - // secret name is a valid quarks link name and matches deployment - var regex = regexp.MustCompile(fmt.Sprintf("^link-%s-[a-z0-9-]*$", e.deployment)) - if !regex.MatchString(secret.Name) { - return link{}, false - } - - // secret contains one of the requested properties for _, link := range e.links { - if _, found := secret.Data[link.String()]; found { + if secret.Name == names.QuarksLinkSecretName(e.deployment, link.LinkType, link.Name) { return link, true } } + return link{}, false } diff --git a/pkg/kube/controllers/quarkslink/pod_mutator.go b/pkg/kube/controllers/quarkslink/pod_mutator.go index 3bd8521c0..f8f812e09 100644 --- a/pkg/kube/controllers/quarkslink/pod_mutator.go +++ b/pkg/kube/controllers/quarkslink/pod_mutator.go @@ -6,6 +6,9 @@ import ( "fmt" "net/http" "path/filepath" + "regexp" + "sort" + "strings" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -79,18 +82,12 @@ func (m *PodMutator) addSecrets(ctx context.Context, namespace string, pod *core // add missing volume sources to pod for _, link := range links { - if !hasSecretVolumeSource(pod.Spec.Volumes, link.secretName) { + if !hasSecretVolumeSource(pod.Spec.Volumes, link.secret.Name) { volume := corev1.Volume{ - Name: link.secretName, + Name: link.secret.Name, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: link.secretName, - Items: []corev1.KeyToPath{ - corev1.KeyToPath{ - Key: link.String(), - Path: "link.yaml", - }, - }, + SecretName: link.secret.Name, }, }, } @@ -99,12 +96,12 @@ func (m *PodMutator) addSecrets(ctx context.Context, namespace string, pod *core // create/update volume mount on containers mount := corev1.VolumeMount{ - Name: link.secretName, + Name: link.secret.Name, ReadOnly: true, - MountPath: filepath.Join("/quarks/link", e.deployment, link.Name), + MountPath: filepath.Join("/quarks/link", e.deployment, link.String()), } for i, container := range pod.Spec.Containers { - idx := findVolumeMount(container.VolumeMounts, link.secretName) + idx := findVolumeMount(container.VolumeMounts, link.secret.Name) if idx > -1 { container.VolumeMounts[idx] = mount } else { @@ -112,6 +109,29 @@ func (m *PodMutator) addSecrets(ctx context.Context, namespace string, pod *core } pod.Spec.Containers[i] = container } + + // add link properties as environment variables + keys := []string{} + for key := range link.secret.Data { + keys = append(keys, key) + } + sort.Strings(keys) + + for contIdx := range pod.Spec.Containers { + for _, key := range keys { + pod.Spec.Containers[contIdx].Env = append(pod.Spec.Containers[contIdx].Env, + corev1.EnvVar{ + Name: fmt.Sprintf("LINK_%s", asEnvironmentVariableName(key)), + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: link.secret.Name}, + Key: key, + }, + }, + }, + ) + } + } } return nil @@ -133,13 +153,9 @@ func (m *PodMutator) findLinks(ctx context.Context, namespace string, e entangle return links, nil } - // we can't use the instance group from - // link-- for the search, because we don't - // know which ig provides the link, so filter for secrets which match - // the link name scheme and have our link 'type.name' as data key - for _, secret := range list.Items { - if link, ok := e.find(secret); ok { - link.secretName = secret.Name + for i := range list.Items { + if link, ok := e.find(list.Items[i]); ok { + link.secret = &(list.Items[i]) links = append(links, link) } } @@ -147,6 +163,11 @@ func (m *PodMutator) findLinks(ctx context.Context, namespace string, e entangle return links, nil } +func asEnvironmentVariableName(input string) string { + reg := regexp.MustCompile(`[^a-zA-Z0-9]+`) + return strings.ToUpper(reg.ReplaceAllString(input, "_")) +} + func hasSecretVolumeSource(volumes []corev1.Volume, name string) bool { for _, v := range volumes { if v.Secret != nil && v.Secret.SecretName == name { diff --git a/pkg/kube/controllers/quarkslink/pod_mutator_test.go b/pkg/kube/controllers/quarkslink/pod_mutator_test.go index 0143b144a..27a28af58 100644 --- a/pkg/kube/controllers/quarkslink/pod_mutator_test.go +++ b/pkg/kube/controllers/quarkslink/pod_mutator_test.go @@ -45,9 +45,9 @@ var _ = Describe("Mount quarks link secret on entangled pods", func() { response admission.Response ) - podPatch := `{"op":"add","path":"/spec/volumes","value":[{"name":"link-nats-deployment-nats","secret":{"items":[{"key":"nats.nats","path":"link.yaml"}],"secretName":"link-nats-deployment-nats"}}]}` - containerPatch := `{"op":"add","path":"/spec/containers/0/volumeMounts","value":[{"mountPath":"/quarks/link/nats-deployment/nats","name":"link-nats-deployment-nats","readOnly":true}]}` - secondContainerPatch := `{"op":"add","path":"/spec/containers/1/volumeMounts","value":[{"mountPath":"/quarks/link/nats-deployment/nats","name":"link-nats-deployment-nats","readOnly":true}]}` + podPatch := `{"op":"add","path":"/spec/volumes","value":[{"name":"link-nats-deployment-nats-nats","secret":{"secretName":"link-nats-deployment-nats-nats"}}]}` + containerPatch := `{"op":"add","path":"/spec/containers/0/volumeMounts","value":[{"mountPath":"/quarks/link/nats-deployment/nats-nats","name":"link-nats-deployment-nats-nats","readOnly":true}]}` + secondContainerPatch := `{"op":"add","path":"/spec/containers/1/volumeMounts","value":[{"mountPath":"/quarks/link/nats-deployment/nats-nats","name":"link-nats-deployment-nats-nats","readOnly":true}]}` jsonPatches := func(operations []jsonpatch.Operation) []string { patches := make([]string, len(operations)) @@ -118,7 +118,7 @@ var _ = Describe("Mount quarks link secret on entangled pods", func() { }) It("secret is mounted on all containers", func() { - Expect(response.Patches).To(HaveLen(3)) + Expect(response.Patches).To(HaveLen(5)) patches := jsonPatches(response.Patches) Expect(patches).To(ContainElement(podPatch)) Expect(patches).To(ContainElement(containerPatch)) @@ -157,8 +157,9 @@ var _ = Describe("Mount quarks link secret on entangled pods", func() { }) Context("when pod has existing volumes", func() { - podPatch := `{"op":"add","path":"/spec/volumes/1","value":{"name":"link-nats-deployment-nats","secret":{"items":[{"key":"nats.nats","path":"link.yaml"}],"secretName":"link-nats-deployment-nats"}}}` - containerPatch := `{"op":"add","path":"/spec/containers/0/volumeMounts/1","value":{"mountPath":"/quarks/link/nats-deployment/nats","name":"link-nats-deployment-nats","readOnly":true}}` + podPatch := `{"op":"add","path":"/spec/volumes/1","value":{"name":"link-nats-deployment-nats-nats","secret":{"secretName":"link-nats-deployment-nats-nats"}}}` + containerPatch := `{"op":"add","path":"/spec/containers/0/volumeMounts/1","value":{"mountPath":"/quarks/link/nats-deployment/nats-nats","name":"link-nats-deployment-nats-nats","readOnly":true}}` + envVarsPatch := `{"op":"add","path":"/spec/containers/0/env","value":[{"name":"LINK_NATS_PASSWORD","valueFrom":{"secretKeyRef":{"key":"nats.password","name":"link-nats-deployment-nats-nats"}}},{"name":"LINK_NATS_PORT","valueFrom":{"secretKeyRef":{"key":"nats.port","name":"link-nats-deployment-nats-nats"}}},{"name":"LINK_NATS_USER","valueFrom":{"secretKeyRef":{"key":"nats.user","name":"link-nats-deployment-nats-nats"}}}]}` BeforeEach(func() { pod = env.NatsPod("entangled-pod") @@ -175,10 +176,11 @@ var _ = Describe("Mount quarks link secret on entangled pods", func() { }) It("does add the link volume and mounts it on all containers", func() { - Expect(response.Patches).To(HaveLen(2)) + Expect(response.Patches).To(HaveLen(3)) patches := jsonPatches(response.Patches) Expect(patches).To(ContainElement(podPatch)) Expect(patches).To(ContainElement(containerPatch)) + Expect(patches).To(ContainElement(envVarsPatch)) Expect(response.AdmissionResponse.Allowed).To(BeTrue()) }) }) diff --git a/testing/catalog.go b/testing/catalog.go index 5130984b0..dd406f588 100644 --- a/testing/catalog.go +++ b/testing/catalog.go @@ -243,27 +243,29 @@ func (c *Catalog) DefaultConfigMap(name string) corev1.ConfigMap { } // QuarksLinkSecret returns a link secret, as generated for consumption by an external (non BOSH) consumer -func (c *Catalog) QuarksLinkSecret(deploymentName, igName, linkType, linkName, value string) corev1.Secret { - key := names.EntanglementSecretKey(linkType, linkName) +func (c *Catalog) QuarksLinkSecret(deploymentName, linkType, linkName string, value map[string][]byte) corev1.Secret { return corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: "link-" + deploymentName + "-" + igName, + Name: names.QuarksLinkSecretName(deploymentName, linkType, linkName), Labels: map[string]string{ manifest.LabelDeploymentName: deploymentName, }, }, - Data: map[string][]byte{ - key: []byte(value), - }, + Data: value, } } // DefaultQuarksLinkSecret has default values from the nats release func (c *Catalog) DefaultQuarksLinkSecret(deploymentName, linkType string) corev1.Secret { return c.QuarksLinkSecret( - deploymentName, linkType, // link-- - linkType, "nats", // type.name - `{"nats":{"password":"custom_password","port":4222,"user":"admin"}}`, + deploymentName, + linkType, + "nats", + map[string][]byte{ + "nats.password": []byte("custom_password"), + "nats.port": []byte("4222"), + "nats.user": []byte("admin"), + }, ) }