Merge remote-tracking branch 'origin/master' into v4.0.x

bin/build-helm

@@ -22,8 +22,29 @@ perl -pi -e "s|org: .*|org: ${DOCKER_IMAGE_ORG}|g" "${output_dir}/cf-operator/va
 perl -pi -e "s|tag: .*|tag: ${DOCKER_IMAGE_TAG}|g" "${output_dir}/cf-operator/values.yaml"
 perl -pi -e "s|version: .*|version: ${QUARKS_JOB_HELM_VERSION}|g" "${output_dir}/cf-operator/requirements.yaml"
 
+# Fetch subchart dependencies
+pushd "$output_dir/cf-operator"
+  helm dependency update
+popd
+
+# Store a list of referenced images with the chart
+quarks_job_values_file=$(mktemp)
+tar xfO ${output_dir}/cf-operator/charts/quarks-job-*.tgz quarks-job/values.yaml > "${quarks_job_values_file}"
+imagelist="$output_dir/cf-operator/imagelist.txt"
+ruby > $imagelist <<EOF
+  require 'yaml'
+  values = YAML.load_file('${output_dir}/cf-operator/values.yaml')
+  image = values['image']
+  puts "#{image['org']}/#{image['repository']}:#{image['tag']}"
+  puts values['operator']['boshDNSDockerImage']
+
+  values = YAML.load_file('${quarks_job_values_file}')
+  image = values['image']
+  puts "#{image['org']}/#{image['repository']}:#{image['tag']}"
+EOF
+rm "${quarks_job_values_file}"
+
 helm package helm/cf-operator \
-  --dependency-update \
   --app-version "$version" \
   --version "$version" \
   --destination "$output_dir"

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/spf13/afero v1.2.2
 	github.com/spf13/cobra v0.0.7
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.6.2
+	github.com/spf13/viper v1.6.3
 	github.com/viovanov/bosh-template-go v0.0.0-20190801125410-a195ef3de03a
 	go.uber.org/zap v1.14.1
 	golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586

go.sum

@@ -56,6 +56,7 @@ github.com/cloudfoundry/bosh-utils v0.0.0-20190206192830-9a0affed2bf1/go.mod h1:
 github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
@@ -408,6 +409,8 @@ github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E=
 github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
+github.com/spf13/viper v1.6.3 h1:pDDu1OyEDTKzpJwdq4TiuLyMsUgRa/BT5cn5O62NoHs=
+github.com/spf13/viper v1.6.3/go.mod h1:jUMtyi0/lB5yZH/FjyGAoH7IMNrIhlBf6pXZmbMDvzw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=

pkg/bosh/bpmconverter/resources.go

@@ -331,6 +331,12 @@ func (kc *BPMConverter) errandToQuarksJob(
 		strategy = qjv1a1.TriggerOnce
 	}
 
+	restartPolicy := corev1.RestartPolicyOnFailure
+	if instanceGroup.LifeCycle == bdm.IGTypeErrand {
+		// Manually triggered errands are not auto-restarted
+		restartPolicy = corev1.RestartPolicyNever
+	}
+
 	qJob := qjv1a1.QuarksJob{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        instanceGroup.Name,
@@ -351,7 +357,7 @@ func (kc *BPMConverter) errandToQuarksJob(
 							Annotations: instanceGroup.Env.AgentEnvBoshConfig.Agent.Settings.Annotations,
 						},
 						Spec: corev1.PodSpec{
-							RestartPolicy:  corev1.RestartPolicyOnFailure,
+							RestartPolicy:  restartPolicy,
 							Containers:     containers,
 							InitContainers: initContainers,
 							Volumes:        volumes,

pkg/bosh/bpmconverter/resources_test.go

@@ -113,6 +113,8 @@ var _ = Describe("BPM Converter", func() {
 					// Test affinity & tolerations
 					Expect(qJob.Spec.Template.Spec.Template.Spec.Affinity).To(BeNil())
 					Expect(len(qJob.Spec.Template.Spec.Template.Spec.Tolerations)).To(Equal(0))
+
+					Expect(qJob.Spec.Template.Spec.Template.Spec.RestartPolicy).To(Equal(corev1.RestartPolicyNever))
 				})
 
 				It("converts the instance group to an quarksJob when this the lifecycle is set to auto-errand", func() {
@@ -124,6 +126,8 @@ var _ = Describe("BPM Converter", func() {
 					// Test trigger strategy
 					qJob := resources.Errands[0]
 					Expect(qJob.Spec.Trigger.Strategy).To(Equal(qjv1a1.TriggerOnce))
+
+					Expect(qJob.Spec.Template.Spec.Template.Spec.RestartPolicy).To(Equal(corev1.RestartPolicyOnFailure))
 				})
 
 				It("converts the AgentEnvBoshConfig information", func() {

pkg/bosh/manifest/cmd_instance_group_resolver.go

@@ -325,11 +325,8 @@ func (igr *InstanceGroupResolver) renderBPM() error {
 			return errors.Wrapf(err, "Rendering BPM failed for instance group %s", igr.instanceGroup.Name)
 		}
 
-		if template != "" {
-			err = igr.renderJobBPM(job, jobSpecFile, template)
-			if err != nil {
-				return errors.Wrapf(err, "Rendering BPM failed for instance group %s", igr.instanceGroup.Name)
-			}
+		if err := igr.renderJobBPM(job, jobSpecFile, template); err != nil {
+			return errors.Wrapf(err, "Rendering BPM failed for instance group %s", igr.instanceGroup.Name)
 		}
 	}
 
@@ -385,50 +382,55 @@ func (igr *InstanceGroupResolver) renderJobBPM(currentJob *Job, jobSpecFile stri
 
 	jobIndexBPM := make([]bpm.Config, len(jobInstances))
 	for i, jobInstance := range jobInstances {
+		var renderedBPM = bpm.Config{}
+
 		properties := currentJob.Properties.ToMap()
 
-		renderPointer := btg.NewERBRenderer(
-			&btg.EvaluationContext{
-				Properties: properties,
-			},
-			&btg.InstanceInfo{
-				Address:    jobInstance.Address,
-				AZ:         jobInstance.AZ,
-				Bootstrap:  jobInstance.Bootstrap,
-				ID:         jobInstance.ID,
-				Index:      jobInstance.Index,
-				Deployment: igr.deploymentName,
-				Name:       jobInstance.Name,
-			},
-			jobSpecFile,
-		)
-
-		// Write to a tmp, this is following the conventions on how the
-		// https://github.com/viovanov/bosh-template-go/ processes the params
-		// when we calling the *.Render().
-		tmpfile, err := ioutil.TempFile("", "rendered.*.yml")
-		if err != nil {
-			return errors.Wrapf(err, "Creation of tmp file %s failed", tmpfile.Name())
-		}
-		defer os.Remove(tmpfile.Name())
+		if erbFilePath != "" {
+			renderPointer := btg.NewERBRenderer(
+				&btg.EvaluationContext{
+					Properties: properties,
+				},
+				&btg.InstanceInfo{
+					Address:    jobInstance.Address,
+					AZ:         jobInstance.AZ,
+					Bootstrap:  jobInstance.Bootstrap,
+					ID:         jobInstance.ID,
+					Index:      jobInstance.Index,
+					Deployment: igr.deploymentName,
+					Name:       jobInstance.Name,
+				},
+				jobSpecFile,
+			)
+
+			// Write to a tmp, this is following the conventions on how the
+			// https://github.com/viovanov/bosh-template-go/ processes the params
+			// when we calling the *.Render().
+			tmpfile, err := ioutil.TempFile("", "rendered.*.yml")
+			if err != nil {
+				return errors.Wrapf(err, "Creation of tmp file %s failed", tmpfile.Name())
+			}
+			defer os.Remove(tmpfile.Name())
 
-		if err := renderPointer.Render(erbFilePath, tmpfile.Name()); err != nil {
-			return errors.Wrapf(err, "Rendering file %s failed", erbFilePath)
-		}
+			if err := renderPointer.Render(erbFilePath, tmpfile.Name()); err != nil {
+				return errors.Wrapf(err, "Rendering file %s failed", erbFilePath)
+			}
 
-		bpmBytes, err := ioutil.ReadFile(tmpfile.Name())
-		if err != nil {
-			return errors.Wrapf(err, "Reading of tmp file %s failed", tmpfile.Name())
-		}
+			bpmBytes, err := ioutil.ReadFile(tmpfile.Name())
+			if err != nil {
+				return errors.Wrapf(err, "Reading of tmp file %s failed", tmpfile.Name())
+			}
 
-		// Parse a rendered bpm.yml into the bpm Config struct.
-		renderedBPM, err := bpm.NewConfig(bpmBytes)
-		if err != nil {
-			return errors.Wrapf(err, "Rendering bpm.yaml into bpm config %s failed", string(bpmBytes))
+			// Parse a rendered bpm.yml into the bpm Config struct.
+			renderedBPM, err = bpm.NewConfig(bpmBytes)
+			if err != nil {
+				return errors.Wrapf(err, "Rendering bpm.yaml into bpm config %s failed", string(bpmBytes))
+			}
 		}
 
 		// Merge processes if they also exist in Quarks
 		if currentJob.Properties.Quarks.BPM != nil && len(currentJob.Properties.Quarks.BPM.Processes) > 0 {
+			var err error
 			renderedBPM.Processes, err = renderedBPM.MergeProcesses(currentJob.Properties.Quarks.BPM.Processes)
 			if err != nil {
 				return errors.Wrapf(err, "failed to merge bpm information from quarks for job '%s'", currentJob.Name)

pkg/bosh/manifest/cmd_instance_group_resolver_test.go

@@ -68,6 +68,36 @@ var _ = Describe("InstanceGroupResolver", func() {
 		})
 	})
 
+	Context("real-life manifests", func() {
+		var fs = afero.NewMemMapFs()
+
+		// https://github.com/cloudfoundry-incubator/kubecf/pull/641
+		It("it avoids regression for kubecf-641", func() {
+			deploymentName := "kubecf"
+			ig := "asdatabase"
+
+			m, err = env.BOSHManifestFromKubeCF641()
+			Expect(err).NotTo(HaveOccurred())
+
+			dns, err := boshdns.NewDNS(*m)
+			Expect(err).ToNot(HaveOccurred())
+
+			igr, err = NewInstanceGroupResolver(fs, assetPath, deploymentName, *m, ig, dns)
+			Expect(err).ToNot(HaveOccurred())
+
+			resolve()
+			bpmInfo, err := igr.BPMInfo()
+			Expect(err).ToNot(HaveOccurred())
+
+			bpm := bpmInfo.Configs["postgres"]
+			Expect(bpm.Ports).To(ContainElement(bpmConfig.Port{
+				Name:     "postgres",
+				Protocol: "TCP",
+				Internal: 5432,
+			}))
+		})
+	})
+
 	Context("InstanceGroupResolver", func() {
 		var fs = afero.NewMemMapFs()
 		var deploymentName string

pkg/kube/util/boshdns/dns.go

@@ -23,8 +23,7 @@ import (
 )
 
 const (
-	cfDomain = "service.cf.internal"
-	appName  = "bosh-dns"
+	appName = "bosh-dns"
 )
 
 var (
@@ -135,7 +134,6 @@ func (dns *boshDomainNameService) DNSSetting(namespace string) (corev1.DNSPolicy
 			fmt.Sprintf("%s.svc.%s", namespace, clusterDomain),
 			fmt.Sprintf("svc.%s", clusterDomain),
 			clusterDomain,
-			cfDomain,
 		},
 		Options: []corev1.PodDNSConfigOption{{Name: "ndots", Value: &ndots}},
 	}, nil

testing/assets/jobs-src/loggregator-agent/loggr-forwarder-agent/job.MF

@@ -0,0 +1,61 @@
+---
+name: loggr-forwarder-agent
+
+templates:
+  bpm.yml.erb: config/bpm.yml
+  prom_scraper_config.yml.erb: config/prom_scraper_config.yml
+  loggregator_ca.crt.erb: config/certs/loggregator_ca.crt
+  forwarder.crt.erb: config/certs/forwarder.crt
+  forwarder.key.erb: config/certs/forwarder.key
+  metrics_ca.crt.erb: config/certs/metrics_ca.crt
+  metrics.crt.erb: config/certs/metrics.crt
+  metrics.key.erb: config/certs/metrics.key
+
+packages:
+- forwarder-agent
+
+properties:
+  port:
+    description: "Port the agent is serving gRPC via mTLS"
+    default: 3458
+  downstream_ingress_port_glob:
+    description: |
+      Files matching the glob are expected to contain ports of downstream
+      consumers that will be bound to 127.0.0.1:{port} with the provided
+      mTLS configuration. The forwarder assumes the downstream server is
+      serving Loggregator's V2 IngressService. See code.cloudfoundry.org/loggregator-api.
+    default: /var/vcap/jobs/*/config/ingress_port.yml
+
+  deployment:
+    description: "Name of deployment (added as tag on all outgoing v1 envelopes)"
+    default: ""
+  tags:
+    description: "Collection of tags to add on all outgoing v2 envelopes. Bosh deployment, job, index and IP will be merged with this property if they are not provided"
+    default: {}
+    example: {"deployment": "cf"}
+
+  tls.ca_cert:
+    description: |
+      TLS loggregator root CA certificate. It is required for key/cert
+      verification.
+  tls.cert:
+    description: "TLS certificate for forwarder signed by the loggregator CA"
+  tls.key:
+    description: "TLS private key for forwarder signed by the loggregator CA"
+  tls.cipher_suites:
+    description: |
+      An ordered list of supported SSL cipher suites. Allowed cipher suites are
+      TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 and TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384.
+    default: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"
+
+  metrics.port:
+    description: "Port the agent uses to serve metrics and debug information"
+    default: 14823
+  metrics.ca_cert:
+    description: "TLS CA cert to verify requests to metrics endpoint."
+  metrics.cert:
+    description: "TLS certificate for metrics server signed by the metrics CA"
+  metrics.key:
+    description: "TLS private key for metrics server signed by the metrics CA"
+  metrics.server_name:
+    description: "The server name used in the scrape configuration for the metrics endpoint"