Rewrites concourse-filter to not require any kind of buffer

Co-authored-by: Ryan Moran <[email protected]>

concourse_filter_suite_test.go

@@ -1,34 +1,26 @@
 package main_test
 
 import (
-	"os"
-	"os/exec"
-	"path/filepath"
-
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
 
 	"testing"
 )
 
 func TestConcourseFilter(t *testing.T) {
 	RegisterFailHandler(Fail)
-	BuildTestBinary(".", "cred-filter")
 	RunSpecs(t, "ConcourseFilter Suite")
 }
 
-func BuildTestBinary(relativePathToDir, FileName string) {
-	dir, err := os.Getwd()
-	if err != nil {
-		panic(err)
-	}
+var path string
 
-	binaryDestination := filepath.Join(dir, relativePathToDir, FileName+".exe")
-	SourceFile := filepath.Join(dir, relativePathToDir, FileName+".go")
+var _ = BeforeSuite(func() {
+	var err error
+	path, err = gexec.Build("github.com/pivotal-cf-experimental/concourse-filter")
+	Expect(err).NotTo(HaveOccurred())
+})
 
-	cmd := exec.Command("go", "build", "-o", binaryDestination, SourceFile)
-	err = cmd.Run()
-	if err != nil {
-		panic(err)
-	}
-}
+var _ = AfterSuite(func() {
+	gexec.CleanupBuildArtifacts()
+})

cred-filter.go

@@ -2,63 +2,126 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
+	"io"
+	"log"
 	"os"
-	"regexp"
+	"sort"
 	"strings"
 )
 
-func whiteList() map[string]bool {
-	r, _ := regexp.Compile("^CREDENTIAL_FILTER_WHITELIST=")
-	whiteListMap := map[string]bool{}
-	for _, envVar := range os.Environ() {
-		if r.MatchString(envVar) {
-			pair := strings.Split(envVar, "=")
-			envVarWhitelist := pair[1]
-			for _, key := range strings.Split(envVarWhitelist, ",") {
-				whiteListMap[key] = true
-			}
-		}
+func main() {
+	source := os.Stdin
+	destination := os.Stdout
+	if len(os.Args) > 1 && os.Args[1] == "-stderr" {
+		destination = os.Stderr
 	}
-	return whiteListMap
+
+	redacted, maxSize := RedactedList()
+
+	err := Stream(source, destination, redacted, maxSize)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+type RedactedVariable struct {
+	Name  string
+	Value []byte
 }
 
-//newEnvStringReplacer creates a string replacer for env variable text
-func newEnvStringReplacer() *strings.Replacer {
-	var envVars []string
+func RedactedList() ([]RedactedVariable, int) {
+	whiteList := map[string]struct{}{}
+	for _, value := range strings.Split(os.Getenv("CREDENTIAL_FILTER_WHITELIST"), ",") {
+		whiteList[value] = struct{}{}
+	}
+
+	var redacted []RedactedVariable
+
+	for _, variable := range os.Environ() {
+		pair := strings.Split(variable, "=")
 
-	whiteList := whiteList()
+		if pair[1] == "" {
+			continue
+		}
 
-	for _, envVar := range os.Environ() {
-		pair := strings.Split(envVar, "=")
-		envVarName := pair[0]
-		envVarValue := pair[1]
-		if !whiteList[envVarName] && envVarValue != "" {
-			envVars = append(envVars, envVarValue)
-			redactedOutput := "[redacted " + envVarName + "]"
-			envVars = append(envVars, redactedOutput)
+		if _, ok := whiteList[pair[0]]; ok {
+			continue
 		}
+
+		redacted = append(redacted, RedactedVariable{
+			Name:  pair[0],
+			Value: []byte(pair[1]),
+		})
 	}
 
-	return strings.NewReplacer(envVars...)
-}
+	sort.Slice(redacted, func(i, j int) bool {
+		return len(redacted[i].Value) > len(redacted[j].Value)
+	})
 
-func main() {
-	envStringReplacer := newEnvStringReplacer()
-	buffer := make([]byte, 257*1024)
+	var maxSize int
+	if len(redacted) > 0 {
+		maxSize = len(redacted[0].Value)
+	}
 
-	scanner := bufio.NewScanner(os.Stdin)
-	scanner.Buffer(buffer, 257*1024)
+	return redacted, maxSize
+}
 
-	for scanner.Scan() {
-		output := os.Stdout
-		if len(os.Args) > 1 && os.Args[1] == "-stderr" {
-			output = os.Stderr
+func Stream(source io.Reader, destination io.Writer, redacted []RedactedVariable, maxSize int) error {
+	if maxSize == 0 {
+		_, err := io.Copy(destination, source)
+		if err != nil {
+			return fmt.Errorf("failed to copy source to destination: %w", err)
 		}
-		fmt.Fprintln(output, envStringReplacer.Replace(scanner.Text()))
+
+		return nil
 	}
-	if err := scanner.Err(); err != nil {
-		fmt.Fprintln(os.Stderr, "error:", err)
-		os.Exit(1)
+
+	reader := bufio.NewReader(source)
+
+	preview, err := reader.Peek(maxSize)
+	if err != nil && err != io.EOF {
+		return fmt.Errorf("failed to preview source: %w", err)
+	}
+
+	for {
+		var match bool
+
+		for _, r := range redacted {
+			if bytes.HasPrefix(preview, r.Value) {
+				_, err = reader.Discard(len(r.Value))
+				if err != nil {
+					return fmt.Errorf("failed to discard from source: %w", err)
+				}
+
+				fmt.Fprintf(destination, "[redacted %s]", r.Name)
+
+				match = true
+
+				break
+			}
+		}
+
+		if !match {
+			b, err := reader.ReadByte()
+			if err != nil {
+				if err == io.EOF {
+					return nil
+				}
+
+				return fmt.Errorf("failed to read byte from source: %w", err)
+			}
+
+			_, err = destination.Write([]byte{b})
+			if err != nil {
+				return fmt.Errorf("failed to write byte to destination: %w", err)
+			}
+		}
+
+		preview, err = reader.Peek(maxSize)
+		if err != nil && err != io.EOF {
+			return fmt.Errorf("failed to preview source: %w", err)
+		}
 	}
 }

cred-filter_test.go

@@ -1,73 +1,89 @@
 package main_test
 
 import (
-	"bytes"
 	"os/exec"
 	"strings"
 
+	"github.com/onsi/gomega/gexec"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
-func runBinary(stdin string, args, env []string) (string, string, error) {
-	var stdout, stderr bytes.Buffer
-	cmd := exec.Command("./cred-filter.exe", args...)
-	cmd.Stdin = strings.NewReader(stdin)
-	cmd.Env = env
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
-
-	err := cmd.Run()
-	return stdout.String(), stderr.String(), err
-}
-
 var _ = Describe("CredFilter", func() {
-	var args []string
-	BeforeEach(func() {
-		args = []string{}
-	})
-
 	Context("Output to stderr instead", func() {
 		It("sends output to stderr instead", func() {
-			args = []string{"-stderr"}
-			output, stderr, err := runBinary("boring text", args, []string{})
-			Expect(err).To(BeNil())
-			Expect(output).To(Equal(""))
-			Expect(stderr).To(Equal("boring text\n"))
+			command := exec.Command(path, "-stderr")
+			command.Env = []string{"BORING=boring"}
+			command.Stdin = strings.NewReader("boring text")
+
+			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
+			Expect(err).NotTo(HaveOccurred())
+			Eventually(session).Should(gexec.Exit(0))
+
+			Expect(session.Out.Contents()).To(BeEmpty())
+			Expect(string(session.Err.Contents())).To(Equal("[redacted BORING] text"))
 		})
 	})
 
 	Context("No sensitive credentials available", func() {
 		It("outputs as is", func() {
-			env := []string{}
-			output, stderr, err := runBinary("boring text", args, env)
-			Expect(err).To(BeNil())
-			Expect(output).To(Equal("boring text\n"))
-			Expect(stderr).To(Equal(""))
+			command := exec.Command(path)
+			command.Env = []string{}
+			command.Stdin = strings.NewReader("boring text")
+
+			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
+			Expect(err).NotTo(HaveOccurred())
+			Eventually(session).Should(gexec.Exit(0))
+
+			Expect(string(session.Out.Contents())).To(Equal("boring text"))
+			Expect(session.Err.Contents()).To(BeEmpty())
 		})
 	})
+
 	Context("Sensitive credentials available", func() {
 		It("filters out those credentials", func() {
-			env := []string{"SECRET=secret", "INFO=info"}
-			output, _, err := runBinary("super secret info\nnew line", args, env)
-			Expect(err).To(BeNil())
-			Expect(output).To(Equal("super [redacted SECRET] [redacted INFO]\nnew line\n"))
+			command := exec.Command(path)
+			command.Env = []string{"SECRET=secret", "INFO=info"}
+			command.Stdin = strings.NewReader("super secret info\nnew line")
+
+			session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
+			Expect(err).NotTo(HaveOccurred())
+			Eventually(session).Should(gexec.Exit(0))
+
+			Expect(string(session.Out.Contents())).To(Equal("super [redacted SECRET] [redacted INFO]\nnew line"))
+			Expect(session.Err.Contents()).To(BeEmpty())
 		})
+
 		Context("sensitive credential env var is whitelisted", func() {
 			It("filters out non-white-listed credentials", func() {
-				env := []string{"SECRET=secret", "INFO=info", "CREDENTIAL_FILTER_WHITELIST=OTHER1,INFO,OTHER2"}
-				output, _, err := runBinary("super secret info", args, env)
-				Expect(err).To(BeNil())
-				Expect(output).To(Equal("super [redacted SECRET] info\n"))
+				command := exec.Command(path)
+				command.Env = []string{"SECRET=secret", "INFO=info", "CREDENTIAL_FILTER_WHITELIST=OTHER1,INFO,OTHER2"}
+				command.Stdin = strings.NewReader("super secret info")
+
+				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
+				Expect(err).NotTo(HaveOccurred())
+				Eventually(session).Should(gexec.Exit(0))
+
+				Expect(string(session.Out.Contents())).To(Equal("super [redacted SECRET] info"))
+				Expect(session.Err.Contents()).To(BeEmpty())
 			})
 		})
+
 		Context("the buffer can handle a 256k string", func() {
 			It("doesn't crash", func() {
-				env := []string{"SECRET=secret", "INFO=info", "CREDENTIAL_FILTER_WHITELIST=OTHER1,INFO,OTHER2"}
-				input := make([]byte, 256*1024)
+				command := exec.Command(path)
+				command.Env = []string{"SECRET=secret", "INFO=info", "CREDENTIAL_FILTER_WHITELIST=OTHER1,INFO,OTHER2"}
+
+				input := string(make([]byte, 256*1024))
+				command.Stdin = strings.NewReader(input)
+
+				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
+				Expect(err).NotTo(HaveOccurred())
+				Eventually(session).Should(gexec.Exit(0))
 
-				_, _, err := runBinary(string(input[:]), args, env)
-				Expect(err).To(BeNil())
+				Expect(string(session.Out.Contents())).To(Equal(input))
+				Expect(session.Err.Contents()).To(BeEmpty())
 			})
 		})
 	})

go.mod

@@ -0,0 +1,8 @@
+module github.com/pivotal-cf-experimental/concourse-filter
+
+go 1.13
+
+require (
+	github.com/onsi/ginkgo v1.11.0
+	github.com/onsi/gomega v1.8.1
+)

go.sum

@@ -0,0 +1,25 @@
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34=
+github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=