- Adds rudimentary support for KV version 2 secret fetching.

- Update some upstream packages via go mod tidy; go mod vendor

go.mod

@@ -2,26 +2,19 @@ module github.com/cruise-automation/daytona
 
 require (
 	cloud.google.com/go v0.40.0
-	github.com/aws/aws-sdk-go v1.19.39
+	github.com/aws/aws-sdk-go v1.25.37
 	github.com/briankassouf/jose v0.9.2-0.20180619214549-d2569464773f
 	github.com/cenkalti/backoff v2.2.1+incompatible
-	github.com/frankban/quicktest v1.4.1 // indirect
-	github.com/google/go-cmp v0.3.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1
 	github.com/hashicorp/go-gcp-common v0.5.0
-	github.com/hashicorp/golang-lru v0.5.3 // indirect
-	github.com/hashicorp/vault/api v1.0.5-0.20190909201928-35325e2c3262
+	github.com/hashicorp/vault v1.3.3
+	github.com/hashicorp/vault-plugin-secrets-kv v0.5.4
+	github.com/hashicorp/vault/api v1.0.5-0.20200215224050-f6547fa8e820
+	github.com/hashicorp/vault/sdk v0.1.14-0.20200305172021-03a3749f220d
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/pierrec/lz4 v2.2.6+incompatible // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.3.0
-	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 // indirect
-	golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a // indirect
 	google.golang.org/api v0.6.0
-	google.golang.org/appengine v1.6.0 // indirect
-	google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64 // indirect
 )
 
 go 1.13

pkg/secrets/helpers_test.go

@@ -0,0 +1,48 @@
+package secrets
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	kv "github.com/hashicorp/vault-plugin-secrets-kv"
+	"github.com/hashicorp/vault/api"
+	vaulthttp "github.com/hashicorp/vault/http"
+	"github.com/hashicorp/vault/sdk/logical"
+	"github.com/hashicorp/vault/vault"
+)
+
+func startVaultTestCluster(t *testing.T) (*vault.TestCluster, *api.Client) {
+	coreConfig := &vault.CoreConfig{
+		LogicalBackends: map[string]logical.Factory{
+			"kv": kv.Factory,
+		},
+	}
+	cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
+		HandlerFunc: vaulthttp.Handler,
+		NumCores:    1,
+	})
+
+	cluster.Start()
+
+	core := cluster.Cores[0]
+	vault.TestWaitActive(t, core.Core)
+	client := core.Client
+
+	err := client.Sys().Mount("kv", &api.MountInput{
+		Type: "kv-v2",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return cluster, client
+}
+
+func generateTestFile(prefix string) (*os.File, error) {
+	file, err := ioutil.TempFile(os.TempDir(), prefix)
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}

pkg/secrets/reader.go

@@ -18,6 +18,7 @@ package secrets
 
 import (
 	"context"
+	"log"
 
 	"github.com/hashicorp/vault/api"
 )
@@ -90,6 +91,7 @@ func (pr *ParallelReader) worker() {
 		case <-pr.ctx.Done():
 			return
 		case keyPath := <-pr.keyPathInChan:
+			log.Printf("Reading path %s\n", keyPath)
 			secret, err := pr.logicalClient.Read(keyPath)
 			pr.secretOutChan <- &SecretResult{
 				KeyPath: keyPath,
@@ -98,4 +100,4 @@ func (pr *ParallelReader) worker() {
 			}
 		}
 	}
-}
+}

pkg/secrets/secrets.go

@@ -65,7 +65,7 @@ func SecretFetcher(client *api.Client, config cfg.Config) {
 	for _, env := range envs {
 		// VAULT_SECRET_WHATEVER=secret/application/thing
 		// VAULT_SECRETS_WHATEVER=secret/application/things
-		// envKey=secretPath
+		// envKey=apex
 		pair := strings.Split(env, "=")
 		envKey := pair[0]
 		apex := os.Getenv(envKey)
@@ -90,6 +90,8 @@ func SecretFetcher(client *api.Client, config cfg.Config) {
 			continue
 		}
 
+		log.Printf("Found %s, attempting to read secrets from %s", envKey, apex)
+
 		// look for a corresponding secretDestinationPrefix key.
 		// sometimes these can be cased inconsistently so we have to attempt normalization.
 		// e.g.  VAULT_SECRET_APPLICATIONA --> DAYTONA_SECRET_DESTINATION_applicationa
@@ -208,6 +210,13 @@ func (sd *SecretDefinition) addSecrets(client *api.Client, secretResult *SecretR
 	}
 	secretData := secret.Data
 
+	// kv version 2 nests secret data beneath two data keys
+	// this makes a best effort to extract it
+	nestedData, ok := secretData["data"]
+	if ok {
+		secretData = nestedData.(map[string]interface{})
+	}
+
 	// Return last error encountered during processing, if any
 	var lastErr error
 
@@ -228,6 +237,20 @@ func (sd *SecretDefinition) addSecrets(client *api.Client, secretResult *SecretR
 func (sd *SecretDefinition) Walk(client *api.Client) error {
 	paths := make([]string, 0)
 
+	// kv version 2 secret backends list keys using a
+	// <mount_name>/metadata/<your supplied path> convention
+	// this extracts the mount and path values so they can be
+	// replaced by 'data' and re-assbled below when being
+	// added to the path slice
+	var isKV2 bool
+	var mount, sPath string
+	kv2Split := strings.Split(sd.secretApex, "metadata")
+	if len(kv2Split) == 2 {
+		isKV2 = true
+		mount = kv2Split[0]
+		sPath = kv2Split[1]
+	}
+
 	list, err := client.Logical().List(sd.secretApex)
 	if err != nil {
 		return fmt.Errorf("there was a problem listing %s: %s", sd.secretApex, err)
@@ -246,7 +269,11 @@ func (sd *SecretDefinition) Walk(client *api.Client) error {
 		if !ok {
 			return fmt.Errorf("non-string secret name: %#v", key)
 		}
-		paths = append(paths, path.Join(sd.secretApex, key))
+		apex := sd.secretApex
+		if isKV2 {
+			apex = mount + "data" + sPath
+		}
+		paths = append(paths, path.Join(apex, key))
 	}
 	sd.paths = paths
 	return nil

pkg/secrets/secrets_test.go

@@ -318,3 +318,71 @@ func TestSecretAWalk(t *testing.T) {
 	assert.Equal(t, "bbbb", destSecrets["credentials_api_b"])
 	assert.Equal(t, "password", destSecrets["other"])
 }
+
+func TestKV2(t *testing.T) {
+	cluster, client := startVaultTestCluster(t)
+	defer cluster.Cleanup()
+
+	var config cfg.Config
+
+	_, err := client.Logical().Write(fmt.Sprintf("kv/data/single"), map[string]interface{}{
+		"data": map[string]interface{}{
+			"value": "just a regular ole value",
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 1; i < 4; i++ {
+		_, err := client.Logical().Write(fmt.Sprintf("kv/data/multiple/thing%v", i), map[string]interface{}{
+			"data": map[string]interface{}{
+				"value": fmt.Sprintf("%v", i),
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	testFiles := make([]*os.File, 0, 2)
+	testFileNames := []string{"secret-kv2-multiple-dest-", "secret-kv2-single-"}
+
+	for _, fileName := range testFileNames {
+		f, err := generateTestFile(fileName)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.Remove(f.Name())
+		testFiles = append(testFiles, f)
+	}
+
+	os.Setenv("VAULT_SECRETS_THING", "kv/metadata/multiple")
+	os.Setenv("DAYTONA_SECRET_DESTINATION_THING", testFiles[0].Name())
+	os.Setenv("VAULT_SECRET_SINGLE", "kv/data/single")
+	os.Setenv("DAYTONA_SECRET_DESTINATION_SINGLE", testFiles[1].Name())
+	defer os.Unsetenv("VAULT_SECRETS_THING")
+	defer os.Unsetenv("DAYTONA_SECRET_DESTINATION_THING")
+	defer os.Unsetenv("VAULT_SECRET_SINGLE")
+	defer os.Unsetenv("DAYTONA_SECRET_DESTINATION_SINGLE")
+
+	SecretFetcher(client, config)
+
+	type expected struct {
+		Name    string
+		Payload string
+	}
+
+	expectedData := []expected{
+		expected{Name: testFiles[0].Name(), Payload: `{"thing1":"1","thing2":"2","thing3":"3"}`},
+		expected{Name: testFiles[1].Name(), Payload: "just a regular ole value"},
+	}
+
+	for _, expected := range expectedData {
+		data, err := ioutil.ReadFile(expected.Name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, expected.Payload, string(data))
+	}
+}