Skip to content

Commit

Permalink
Allow env mapping for secrets
Browse files Browse the repository at this point in the history 
This was previously not possible.
This is a non breaking way of fixing cruise-automation#99.
Other ways would introduce breaking changes to variable names.

Additional features:
* allow to rename secret keys in env variables (see https://github.com/cruise-automation/daytona/issues/99\#issuecomment-1492390145)
* allow to store selected secret keys as env variables and in a file
  • Loading branch information
@jonnylangefeld
jonnylangefeld committed Apr 4, 2023
1 parent f9f7dce commit 4bd0456
Show file tree
Hide file tree
Showing 3 changed files with 180 additions and 19 deletions.
5 changes: 3 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,9 @@ The following authentication methods are supported:
`<STORAGE PATH PREFIX>_<secretID-SUFFIX>=<SECRET-APEX>`
- `VAULT_SECRET_` (`<STORAGE PATH PREFIX>`): Singular Secret Storage Path Prefix
- `VAULT_SECRETS_` (`<STORAGE PATH PREFIX>`): Plural (more than 1 secret beneath the specified path) Secret Storage Path Prefix
- `VAULT_SECRET_`(`<STORAGE PATH PREFIX>`): Singular Secret Storage Path Prefix
- `VAULT_SECRET_`(`<STORAGE PATH PREFIX>`)(`<ENV_VAR_NAME>`): Add the name of the resulting env variable
- `VAULT_SECRETS_`(`<STORAGE PATH PREFIX>`): Plural (more than 1 secret beneath the specified path) Secret Storage Path Prefix
- `secretID-SUFFIX`: The unique secret identifier that can be used to tie a Secret Storage Path Prefix to a corresponding Destination Prefix. The uniqueness of this value provides the ability to supply multiple secret paths.
- `SECRET-APEX`: When used with **Singular** definitions, the Vault path where the secret exists in Vault that can be read. When used with **Plural** definitions, the Vault path where the secrets exist in Vault that can be listed and then read. This will fetch all secrets within the given Vault directory.
Expand Down
68 changes: 54 additions & 14 deletions pkg/secrets/secrets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ type SecretDefinition struct {
paths []string
secrets map[string]string
plural bool

// secretEnvMapping contains a map to correlate secret keys to env variable names
secretEnvMapping map[string]string
}

// SecretFetcher inspects the environment for variables that
Expand Down Expand Up @@ -81,9 +84,10 @@ func SecretFetcher(client *api.Client, config cfg.Config) {
}

def := &SecretDefinition{
envkey: envKey,
secretApex: apex,
secrets: make(map[string]string),
envkey: envKey,
secretApex: apex,
secrets: make(map[string]string),
secretEnvMapping: make(map[string]string),
}

switch {
Expand All @@ -100,6 +104,16 @@ func SecretFetcher(client *api.Client, config cfg.Config) {
continue
}

// read all VAULT_VALUE_KEY_<def.secretID> variables for the secretEnvMapping
for _, env2 := range envs {
pair := strings.Split(env2, "=")
envKey := pair[0]
prefix := fmt.Sprintf("%s%s_", secretValueKeyPrefix, def.secretID)
if strings.HasPrefix(envKey, prefix) {
def.secretEnvMapping[os.Getenv(envKey)] = strings.TrimPrefix(envKey, prefix)
}
}

// look for a corresponding secretDestinationPrefix key.
// sometimes these can be cased inconsistently so we have to attempt normalization.
// e.g. VAULT_SECRET_APPLICATIONA --> DAYTONA_SECRET_DESTINATION_applicationa
Expand Down Expand Up @@ -135,7 +149,7 @@ func SecretFetcher(client *api.Client, config cfg.Config) {
// output the secret definitions
for _, def := range defs {
if config.SecretEnv {
err := setEnvSecrets(def.secrets)
err := setEnvSecrets(def)
if err != nil {
log.Error().Err(err).Msg("failed to set env var")
}
Expand Down Expand Up @@ -200,7 +214,7 @@ func SecretFetcher(client *api.Client, config cfg.Config) {
// all of the path's values. Otherwise, each secret is written to
// its configured destination
func writeSecretsToDestination(def *SecretDefinition) error {
if def.plural {
if def.plural || len(def.secretEnvMapping) > 1 {
err := writeJSONSecrets(def.secrets, def.outputDestination)
if err != nil {
return err
Expand Down Expand Up @@ -231,13 +245,17 @@ func writeJSONSecrets(secrets map[string]string, filepath string) error {

// setEnvSecrets sets the supplied map of strings to the configured environment
// variables
func setEnvSecrets(secrets map[string]string) error {
for k, v := range secrets {
err := os.Setenv(k, v)
func setEnvSecrets(sd *SecretDefinition) error {
for k, v := range sd.secrets {
envName := k
if envMapEntry, ok := sd.secretEnvMapping[k]; ok {
envName = envMapEntry
}
err := os.Setenv(envName, v)
if err != nil {
return fmt.Errorf("error from os.Setenv: %s", err)
}
log.Info().Str("var", strings.ToUpper(k)).Msg("Set env var")
log.Info().Str("var", envName).Msg("Set env var")
}
return nil
}
Expand Down Expand Up @@ -285,12 +303,8 @@ func (sd *SecretDefinition) addSecrets(secretResult *SecretResult) error {
return fmt.Errorf("vault listed a secret %s %s, but failed trying to read it; likely the rate-limiting retry attempts were exceeded", keyName, keyPath)
}

if !sd.plural && sd.outputDestination != "" {
if !sd.plural && sd.outputDestination != "" && len(sd.secretEnvMapping) == 0 {
singleValueKey := defaultKeyName
if envKey := os.Getenv(secretValueKeyPrefix + sd.secretID); envKey != "" {
log.Info().Str("key", secretValueKeyPrefix+sd.secretID).Str("value", singleValueKey).Msg("Found an explicit vault value key, will read this value key instead of using the default")
singleValueKey = envKey
}
v, ok := secretData[singleValueKey]
if ok {
secretValue, err := valueConverter(v)
Expand All @@ -302,6 +316,15 @@ func (sd *SecretDefinition) addSecrets(secretResult *SecretResult) error {
return err
}
}
if !sd.plural && len(sd.secretEnvMapping) > 0 {
for k, _ := range sd.secretEnvMapping {
log.Info().Str("key", k).Msg("Found an explicit vault value key, will read this value key instead of using the default")
if err := sd.copyValue(secretData, k); err != nil {
return err
}
}
return nil
}

for k, v := range secretData {
secretValue, err := valueConverter(v)
Expand All @@ -321,6 +344,23 @@ func (sd *SecretDefinition) addSecrets(secretResult *SecretResult) error {
return nil
}

// copyValues copies a value from the secretData object returned by vault and writes it into the secrets map of the
// SecretDefintion
func (sd *SecretDefinition) copyValue(secretData map[string]interface{}, key string) error {
v, ok := secretData[key]
if !ok {
return fmt.Errorf("key not found in vault secret: %s", key)
}
secretValue, err := valueConverter(v)
if err != nil {
return err
}
sd.Lock()
sd.secrets[key] = secretValue
sd.Unlock()
return nil
}

// Walk walks a SecretDefintions SecretApex. This is used for iteration
// of the provided apex path
func (sd *SecretDefinition) Walk(client *api.Client) error {
Expand Down
126 changes: 123 additions & 3 deletions pkg/secrets/secrets_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"net/http"
"net/http/httptest"
"os"
"reflect"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -440,7 +441,7 @@ func TestUnmatchedPluralDesintation(t *testing.T) {
os.Setenv("DAYTONA_SECRET_DESTINATION_jacka", f2.Name())

defer os.Unsetenv("VAULT_SECRETS_APEX")
defer os.Setenv("DAYTONA_SECRET_DESTINATION_tha", f1.Name())
defer os.Unsetenv("DAYTONA_SECRET_DESTINATION_tha")
defer os.Unsetenv("DAYTONA_SECRET_DESTINATION_jacka")

SecretFetcher(client, config)
Expand Down Expand Up @@ -788,11 +789,11 @@ func TestSecretSingularDestinationKeyOverride(t *testing.T) {

os.Setenv("VAULT_SECRET_APPLICATIONA", "secret/applicationA")
os.Setenv("DAYTONA_SECRET_DESTINATION_APPLICATIONA", file.Name())
os.Setenv("VAULT_VALUE_KEY_APPLICATIONA", "password")
os.Setenv("VAULT_VALUE_KEY_APPLICATIONA_PASSWORD", "password")

defer os.Unsetenv("VAULT_SECRET_APPLICATIONA")
defer os.Unsetenv("DAYTONA_SECRET_DESTINATION_APPLICATIONA")
defer os.Unsetenv("VAULT_VALUE_KEY_APPLICATIONA")
defer os.Unsetenv("VAULT_VALUE_KEY_APPLICATIONA_PASSWORD")

config.Workers = 3
SecretFetcher(client, config)
Expand All @@ -804,3 +805,122 @@ func TestSecretSingularDestinationKeyOverride(t *testing.T) {

assert.Equal(t, "nonstandard", string(data))
}

func TestSecretFetcher(t *testing.T) {
tests := []struct {
name string
envVars map[string]string
assertEnv map[string]string
}{
{
name: "Singular key override",
envVars: map[string]string{
"VAULT_SECRET_APPLICATIONA": "secret/applicationA",
"VAULT_VALUE_KEY_APPLICATIONA_MYPASSWORD": "password",
},
assertEnv: map[string]string{
"MYPASSWORD": "p@ssw0rd",
},
},
{
name: "Multiple key override",
envVars: map[string]string{
"VAULT_SECRET_APPLICATIONA": "secret/applicationA",
"VAULT_VALUE_KEY_APPLICATIONA_MYUSER": "username",
"VAULT_VALUE_KEY_APPLICATIONA_MYPASSWORD": "password",
},
assertEnv: map[string]string{
"MYUSER": "alice",
"MYPASSWORD": "p@ssw0rd",
},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var config cfg.Config

ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, `
{
"auth": null,
"data": {
"username": "alice",
"password": "p@ssw0rd"
},
"lease_duration": 3600,
"lease_id": "",
"renewable": false
}
`)
}))
defer ts.Close()

for key, value := range test.envVars {
os.Setenv(key, value)
defer os.Unsetenv(key)
}

config.Workers = 3
config.SecretEnv = true
client, err := testhelpers.GetTestClient(ts.URL)
if err != nil {
t.Fatal(err)
}
SecretFetcher(client, config)

for key, value := range test.assertEnv {
assert.Equal(t, value, os.Getenv(key))
defer os.Unsetenv(key)
}
})
}
}

func TestCopyValue(t *testing.T) {
tests := []struct {
name string
secretData map[string]interface{}
key string
expectedErr error
expectedMap map[string]string
}{
{
name: "key found",
secretData: map[string]interface{}{
"username": "alice",
"password": "p@ssw0rd",
},
key: "username",
expectedErr: nil,
expectedMap: map[string]string{
"username": "alice",
},
},
{
name: "key not found",
secretData: map[string]interface{}{
"username": "alice",
"password": "p@ssw0rd",
},
key: "email",
expectedErr: fmt.Errorf("key not found in vault secret: email"),
expectedMap: nil,
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
sd := &SecretDefinition{
secrets: make(map[string]string),
}
err := sd.copyValue(test.secretData, test.key)
if err != nil && err.Error() != test.expectedErr.Error() {
t.Errorf("expected error '%v' but got '%v'", test.expectedErr, err)
}
if err == nil && !reflect.DeepEqual(sd.secrets, test.expectedMap) {
t.Errorf("expected secrets map '%v' but got '%v'", test.expectedMap, sd.secrets)
}
})
}
}

0 comments on commit 4bd0456

Please sign in to comment.