Add support for Vault KV secrets engine v2 (#86)

* Fix vault couldn't get a value

* Add support for Vault KV secrets engine v2

* Allow simpler paths like the Vault CLI does

* Fix Vault tests fail when run twice

Due to data versioning the deferred deletion doesn't completely remove
the secret - only the data.

* Improve path change

Co-authored-by: Jean-Philippe Moal <[email protected]>

Co-authored-by: mopemope <[email protected]>
Co-authored-by: Jean-Philippe Moal <[email protected]>

.travis.yml

@@ -31,3 +31,4 @@ before_script:
   - docker run -d -p 2379:2379 quay.io/coreos/etcd /usr/local/bin/etcd -advertise-client-urls http://0.0.0.0:2379 -listen-client-urls http://0.0.0.0:2379
   - docker run -d -p 8500:8500 --name consul consul
   - docker run -d -p 8200:8200 --cap-add=IPC_LOCK -e 'VAULT_DEV_ROOT_TOKEN_ID=root' -e 'VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' vault:0.9.6
+  - docker run -d -p 8222:8200 --cap-add=IPC_LOCK -e 'VAULT_DEV_ROOT_TOKEN_ID=root' -e 'VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' vault:0.10.0

backend/vault/vault.go

@@ -3,8 +3,10 @@ package vault
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/hashicorp/vault/api"
+
 	"github.com/heetch/confita/backend"
 )
 
@@ -13,17 +15,37 @@ type Backend struct {
 	client *api.Logical
 	path   string
 	secret *api.Secret
+	// KV secrets engine v2
+	v2 bool
 }
 
 // NewBackend creates a configuration loader that loads from Vault
 // all the keys from the given path and holds them in memory.
+// Use this when using Vault KV secrets engine v1.
 func NewBackend(client *api.Logical, path string) *Backend {
 	return &Backend{
 		client: client,
 		path:   path,
 	}
 }
 
+// NewBackendV2 creates a configuration loader that loads from Vault
+// all the keys from the given path and holds them in memory.
+// Use this when using Vault KV secrets engine v2.
+func NewBackendV2(client *api.Logical, path string) *Backend {
+	path = strings.TrimPrefix(path, "/")
+	// The KV secrets engine v2 uses the "secrets/data" prefix in the path,
+	// but we want to support regular paths as well, just like the Vault CLI does.
+	if strings.HasPrefix(path, "secret/") && !strings.HasPrefix(path, "secret/data/") {
+		path = strings.Replace(path, "secret/", "secret/data/", 1)
+	}
+	return &Backend{
+		client: client,
+		path:   path,
+		v2:     true,
+	}
+}
+
 // Get loads the given key from Vault.
 func (b *Backend) Get(ctx context.Context, key string) ([]byte, error) {
 	var err error
@@ -39,8 +61,17 @@ func (b *Backend) Get(ctx context.Context, key string) ([]byte, error) {
 		}
 	}
 
-	if v, ok := b.secret.Data[key]; ok {
-		return []byte(v.(string)), nil
+	if b.v2 {
+		if data, ok := b.secret.Data["data"]; ok {
+			data := data.(map[string]interface{})
+			if v, ok := data[key]; ok {
+				return []byte(v.(string)), nil
+			}
+		}
+	} else {
+		if v, ok := b.secret.Data[key]; ok {
+			return []byte(v.(string)), nil
+		}
 	}
 
 	return nil, backend.ErrNotFound

backend/vault/vault_test.go

@@ -2,15 +2,23 @@ package vault
 
 import (
 	"context"
+	"math/rand"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/vault/api"
-	"github.com/heetch/confita/backend"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/heetch/confita/backend"
 )
 
+const letterBytes = "abcdefghijklmnopqrstuvwxyz"
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
 func TestVaultBackend(t *testing.T) {
 	os.Setenv("VAULT_ADDR", "http://127.0.0.1:8200")
 	client, err := api.NewClient(api.DefaultConfig())
@@ -34,16 +42,16 @@ func TestVaultBackend(t *testing.T) {
 
 		_, err = c.Write(path,
 			map[string]interface{}{
-				"foo":    "bar",
-				"cheese": "nan",
+				"foo":  "bar",
+				"data": "nan",
 			})
 		require.NoError(t, err)
 
 		val, err := b.Get(context.Background(), "foo")
 		require.NoError(t, err)
 		assert.Equal(t, "bar", string(val))
 
-		val, err = b.Get(context.Background(), "cheese")
+		val, err = b.Get(context.Background(), "data")
 		require.NoError(t, err)
 		assert.Equal(t, "nan", string(val))
 	})
@@ -54,3 +62,76 @@ func TestVaultBackend(t *testing.T) {
 		require.EqualError(t, err, backend.ErrNotFound.Error())
 	})
 }
+
+func TestVaultBackendV2(t *testing.T) {
+	os.Setenv("VAULT_ADDR", "http://127.0.0.1:8222")
+	client, err := api.NewClient(api.DefaultConfig())
+	require.NoError(t, err)
+
+	client.SetToken("root")
+	c := client.Logical()
+
+	randPathSuffix := randStringBytes(5)
+	path := "secret/data/" + randPathSuffix
+
+	defer c.Delete(path)
+
+	t.Run("SecretPathNotFound", func(t *testing.T) {
+		b := NewBackendV2(c, path)
+		_, err := b.Get(context.Background(), "foo")
+		require.EqualError(t, err, "secret not found at the following path: "+path)
+	})
+
+	okTests := []struct {
+		name string
+		path string
+	}{
+		{
+			"OK v2 data path",
+			path,
+		},
+		{
+			"OK old path",
+			"secret/" + randPathSuffix,
+		},
+	}
+	for _, okTest := range okTests {
+		t.Run(okTest.name, func(t *testing.T) {
+			b := NewBackendV2(c, okTest.path)
+
+			// For writing we use the Consul client directly,
+			// so we need to use the full proper path.
+			_, err = c.Write(path,
+				map[string]interface{}{
+					"data": map[string]string{
+						"foo":  "bar",
+						"data": "nan",
+					},
+				})
+			require.NoError(t, err)
+
+			val, err := b.Get(context.Background(), "foo")
+			require.NoError(t, err)
+			assert.Equal(t, "bar", string(val))
+
+			val, err = b.Get(context.Background(), "data")
+			require.NoError(t, err)
+			assert.Equal(t, "nan", string(val))
+		})
+	}
+
+	t.Run("NotFound", func(t *testing.T) {
+		b := NewBackendV2(c, path)
+		_, err := b.Get(context.Background(), "badKey")
+		require.EqualError(t, err, backend.ErrNotFound.Error())
+	})
+
+}
+
+func randStringBytes(n int) string {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return string(b)
+}