Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow env mapping for secrets #100

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Allow env mapping for secrets #100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,9 @@ The following authentication methods are supported:

`<STORAGE PATH PREFIX>_<secretID-SUFFIX>=<SECRET-APEX>`

- `VAULT_SECRET_` (`<STORAGE PATH PREFIX>`): Singular Secret Storage Path Prefix
- `VAULT_SECRETS_` (`<STORAGE PATH PREFIX>`): Plural (more than 1 secret beneath the specified path) Secret Storage Path Prefix
- `VAULT_SECRET_`(`<STORAGE PATH PREFIX>`): Singular Secret Storage Path Prefix
- `VAULT_SECRET_`(`<STORAGE PATH PREFIX>`)(`<ENV_VAR_NAME>`): Add the name of the resulting env variable
- `VAULT_SECRETS_`(`<STORAGE PATH PREFIX>`): Plural (more than 1 secret beneath the specified path) Secret Storage Path Prefix
- `secretID-SUFFIX`: The unique secret identifier that can be used to tie a Secret Storage Path Prefix to a corresponding Destination Prefix. The uniqueness of this value provides the ability to supply multiple secret paths.
- `SECRET-APEX`: When used with **Singular** definitions, the Vault path where the secret exists in Vault that can be read. When used with **Plural** definitions, the Vault path where the secrets exist in Vault that can be listed and then read. This will fetch all secrets within the given Vault directory.

Expand Down
64 changes: 54 additions & 10 deletions pkg/secrets/secrets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ type SecretDefinition struct {
paths []string
secrets map[string]string
plural bool

// secretEnvMapping contains a map to correlate secret keys to env variable names
secretEnvMapping map[string]string
}

// SecretFetcher inspects the environment for variables that
Expand Down Expand Up @@ -81,9 +84,10 @@ func SecretFetcher(client *api.Client, config cfg.Config) {
}

def := &SecretDefinition{
envkey: envKey,
secretApex: apex,
secrets: make(map[string]string),
envkey: envKey,
secretApex: apex,
secrets: make(map[string]string),
secretEnvMapping: make(map[string]string),
}

switch {
Expand All @@ -100,6 +104,16 @@ func SecretFetcher(client *api.Client, config cfg.Config) {
continue
}

// read all VAULT_VALUE_KEY_<def.secretID> variables for the secretEnvMapping
for _, env2 := range envs {
pair := strings.Split(env2, "=")
envKey := pair[0]
prefix := fmt.Sprintf("%s%s_", secretValueKeyPrefix, def.secretID)
if strings.HasPrefix(envKey, prefix) {
def.secretEnvMapping[os.Getenv(envKey)] = strings.TrimPrefix(envKey, prefix)
}
}

// look for a corresponding secretDestinationPrefix key.
// sometimes these can be cased inconsistently so we have to attempt normalization.
// e.g. VAULT_SECRET_APPLICATIONA --> DAYTONA_SECRET_DESTINATION_applicationa
Expand Down Expand Up @@ -135,7 +149,7 @@ func SecretFetcher(client *api.Client, config cfg.Config) {
// output the secret definitions
for _, def := range defs {
if config.SecretEnv {
err := setEnvSecrets(def.secrets)
err := setEnvSecrets(def)
if err != nil {
log.Error().Err(err).Msg("failed to set env var")
}
Expand Down Expand Up @@ -200,7 +214,7 @@ func SecretFetcher(client *api.Client, config cfg.Config) {
// all of the path's values. Otherwise, each secret is written to
// its configured destination
func writeSecretsToDestination(def *SecretDefinition) error {
if def.plural {
if def.plural || len(def.secretEnvMapping) > 1 {
err := writeJSONSecrets(def.secrets, def.outputDestination)
if err != nil {
return err
Expand Down Expand Up @@ -231,13 +245,17 @@ func writeJSONSecrets(secrets map[string]string, filepath string) error {

// setEnvSecrets sets the supplied map of strings to the configured environment
// variables
func setEnvSecrets(secrets map[string]string) error {
for k, v := range secrets {
err := os.Setenv(k, v)
func setEnvSecrets(sd *SecretDefinition) error {
for k, v := range sd.secrets {
envName := k
if envMapEntry, ok := sd.secretEnvMapping[k]; ok {
envName = envMapEntry
}
err := os.Setenv(envName, v)
if err != nil {
return fmt.Errorf("error from os.Setenv: %s", err)
}
log.Info().Str("var", strings.ToUpper(k)).Msg("Set env var")
log.Info().Str("var", envName).Msg("Set env var")
}
return nil
}
Expand Down Expand Up @@ -285,7 +303,7 @@ func (sd *SecretDefinition) addSecrets(secretResult *SecretResult) error {
return fmt.Errorf("vault listed a secret %s %s, but failed trying to read it; likely the rate-limiting retry attempts were exceeded", keyName, keyPath)
}

if !sd.plural && sd.outputDestination != "" {
if !sd.plural && sd.outputDestination != "" && len(sd.secretEnvMapping) == 0 {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

my initial approach was just to remove sd.outputDestination but that would make many tests fail, which means that the current API would be broken.

Instead I decided to make this non-breaking change, which doesn't fix the bug, but at least allows for setting environment vairables.

singleValueKey := defaultKeyName
if envKey := os.Getenv(secretValueKeyPrefix + sd.secretID); envKey != "" {
log.Info().Str("key", secretValueKeyPrefix+sd.secretID).Str("value", singleValueKey).Msg("Found an explicit vault value key, will read this value key instead of using the default")
Expand All @@ -302,6 +320,15 @@ func (sd *SecretDefinition) addSecrets(secretResult *SecretResult) error {
return err
}
}
if !sd.plural && len(sd.secretEnvMapping) > 0 {
for k, _ := range sd.secretEnvMapping {
log.Info().Str("key", k).Msg("Found an explicit vault value key, will read this value key instead of using the default")
if err := sd.copyValue(secretData, k); err != nil {
return err
}
}
return nil
}

for k, v := range secretData {
secretValue, err := valueConverter(v)
Expand All @@ -321,6 +348,23 @@ func (sd *SecretDefinition) addSecrets(secretResult *SecretResult) error {
return nil
}

// copyValues copies a value from the secretData object returned by vault and writes it into the secrets map of the
// SecretDefintion
func (sd *SecretDefinition) copyValue(secretData map[string]interface{}, key string) error {
v, ok := secretData[key]
if !ok {
return fmt.Errorf("key not found in vault secret: %s", key)
}
secretValue, err := valueConverter(v)
if err != nil {
return err
}
sd.Lock()
sd.secrets[key] = secretValue
sd.Unlock()
return nil
}

// Walk walks a SecretDefintions SecretApex. This is used for iteration
// of the provided apex path
func (sd *SecretDefinition) Walk(client *api.Client) error {
Expand Down
126 changes: 123 additions & 3 deletions pkg/secrets/secrets_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"net/http"
"net/http/httptest"
"os"
"reflect"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -288,7 +289,7 @@ func TestSecretAWalk(t *testing.T) {
os.Setenv("VAULT_SECRETS_COMMON", "secret/path/common")
os.Setenv("DAYTONA_SECRET_DESTINATION_COMMON", destinationPrefixFile.Name())
defer os.Unsetenv("VAULT_SECRETS_COMMON")
defer os.Unsetenv("VAULT_SECRETS_GENERIC")
defer os.Unsetenv("DAYTONA_SECRET_DESTINATION_COMMON")

config.SecretPayloadPath = file.Name()
SecretFetcher(client, config)
Expand Down Expand Up @@ -440,7 +441,7 @@ func TestUnmatchedPluralDesintation(t *testing.T) {
os.Setenv("DAYTONA_SECRET_DESTINATION_jacka", f2.Name())

defer os.Unsetenv("VAULT_SECRETS_APEX")
defer os.Setenv("DAYTONA_SECRET_DESTINATION_tha", f1.Name())
defer os.Unsetenv("DAYTONA_SECRET_DESTINATION_tha")
defer os.Unsetenv("DAYTONA_SECRET_DESTINATION_jacka")

SecretFetcher(client, config)
Expand Down Expand Up @@ -788,7 +789,7 @@ func TestSecretSingularDestinationKeyOverride(t *testing.T) {

os.Setenv("VAULT_SECRET_APPLICATIONA", "secret/applicationA")
os.Setenv("DAYTONA_SECRET_DESTINATION_APPLICATIONA", file.Name())
os.Setenv("VAULT_VALUE_KEY_APPLICATIONA", "password")
os.Setenv("VAULT_VALUE_KEY_APPLICATIONA_PASSWORD", "password")

defer os.Unsetenv("VAULT_SECRET_APPLICATIONA")
defer os.Unsetenv("DAYTONA_SECRET_DESTINATION_APPLICATIONA")
Expand All @@ -804,3 +805,122 @@ func TestSecretSingularDestinationKeyOverride(t *testing.T) {

assert.Equal(t, "nonstandard", string(data))
}

func TestSecretFetcher(t *testing.T) {
tests := []struct {
name string
envVars map[string]string
assertEnv map[string]string
}{
{
name: "Singular key override",
envVars: map[string]string{
"VAULT_SECRET_APPLICATIONA": "secret/applicationA",
"VAULT_VALUE_KEY_APPLICATIONA_MYPASSWORD": "password",
},
assertEnv: map[string]string{
"MYPASSWORD": "p@ssw0rd",
},
},
{
name: "Multiple key override",
envVars: map[string]string{
"VAULT_SECRET_APPLICATIONA": "secret/applicationA",
"VAULT_VALUE_KEY_APPLICATIONA_MYUSER": "username",
"VAULT_VALUE_KEY_APPLICATIONA_MYPASSWORD": "password",
},
assertEnv: map[string]string{
"MYUSER": "alice",
"MYPASSWORD": "p@ssw0rd",
},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var config cfg.Config

ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, `
{
"auth": null,
"data": {
"username": "alice",
"password": "p@ssw0rd"
},
"lease_duration": 3600,
"lease_id": "",
"renewable": false
}
`)
}))
defer ts.Close()

for key, value := range test.envVars {
os.Setenv(key, value)
defer os.Unsetenv(key)
}

config.Workers = 3
config.SecretEnv = true
client, err := testhelpers.GetTestClient(ts.URL)
if err != nil {
t.Fatal(err)
}
SecretFetcher(client, config)

for key, value := range test.assertEnv {
assert.Equal(t, value, os.Getenv(key))
defer os.Unsetenv(key)
}
})
}
}

func TestCopyValue(t *testing.T) {
tests := []struct {
name string
secretData map[string]interface{}
key string
expectedErr error
expectedMap map[string]string
}{
{
name: "key found",
secretData: map[string]interface{}{
"username": "alice",
"password": "p@ssw0rd",
},
key: "username",
expectedErr: nil,
expectedMap: map[string]string{
"username": "alice",
},
},
{
name: "key not found",
secretData: map[string]interface{}{
"username": "alice",
"password": "p@ssw0rd",
},
key: "email",
expectedErr: fmt.Errorf("key not found in vault secret: email"),
expectedMap: nil,
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
sd := &SecretDefinition{
secrets: make(map[string]string),
}
err := sd.copyValue(test.secretData, test.key)
if err != nil && err.Error() != test.expectedErr.Error() {
t.Errorf("expected error '%v' but got '%v'", test.expectedErr, err)
}
if err == nil && !reflect.DeepEqual(sd.secrets, test.expectedMap) {
t.Errorf("expected secrets map '%v' but got '%v'", test.expectedMap, sd.secrets)
}
})
}
}