From c500607a067be3dd9ef46558bf39ada3a41c3767 Mon Sep 17 00:00:00 2001 From: James Hunt Date: Sat, 30 Jun 2018 18:02:13 -0400 Subject: [PATCH] Rename safe -> vault (plugin) and include in releases --- .gitignore | 1 + ci/release_notes.md | 5 + plugin/safe/plugin.go | 250 -------------------------------- plugin/vault/plugin.go | 314 +++++++++++++++++++++++++++++++++++++++++ plugins | 1 + t/api | 6 +- 6 files changed, 324 insertions(+), 253 deletions(-) create mode 100644 ci/release_notes.md delete mode 100644 plugin/safe/plugin.go create mode 100644 plugin/vault/plugin.go diff --git a/.gitignore b/.gitignore index f6f5fd4b7..647172e16 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ /s3 /swift /scality +/vault /webdav /xtrabackup diff --git a/ci/release_notes.md b/ci/release_notes.md new file mode 100644 index 000000000..101521353 --- /dev/null +++ b/ci/release_notes.md @@ -0,0 +1,5 @@ +# Improvements + +- New `vault` plugin for backing up Safe or Vault installations. + You can optionally restrict the subtree that gets backed up and + restored, in case you share the Vault with others. diff --git a/plugin/safe/plugin.go b/plugin/safe/plugin.go deleted file mode 100644 index f646b3f5b..000000000 --- a/plugin/safe/plugin.go +++ /dev/null @@ -1,250 +0,0 @@ -package main - -import ( - "encoding/json" - "io/ioutil" - "os" - - fmt "github.com/jhunt/go-ansi" - - "github.com/starkandwayne/safe/vault" - "github.com/starkandwayne/shield/plugin" -) - -var () - -func main() { - p := SafePlugin{ - Name: "Safe Backup Plugin", - Author: "Stark & Wayne", - Version: "0.0.1", - Features: plugin.PluginFeatures{ - Target: "yes", - Store: "no", - }, - Example: ` -{ - "vault_url" : "https://safe.myorg.mycompany.com", # REQUIRED - "auth_token" : "b8714fec-0df9-3f66-d262-35a57e414120", # REQUIRED - "skip_ssl_validation" : true, # REQUIRED -} -`, - Defaults: ` -{ - "skip_ssl_validation" : false, -} -`, - Fields: []plugin.Field{ - plugin.Field{ - Mode: "target", - Name: "vault_url", - Type: "string", - Title: "Vault URL", - Help: "The url address of your Vault server, including the protocol.", - Example: "https://safe.myorg.mycompany.com", - Required: true, - }, - plugin.Field{ - Mode: "target", - Name: "auth_token", - Type: "string", - Title: "Auth Token", - Help: "The auth token for a user with privileges to read and write the entire secret/ tree", - Required: true, - }, - plugin.Field{ - Mode: "target", - Name: "skip_ssl_validation", - Type: "bool", - Title: "Skip SSL Validation", - Help: "Set to true if using a self-signed cert", - Default: "false", - }, - }, - } - - plugin.Run(p) -} - -type SafePlugin plugin.PluginInfo - -func (p SafePlugin) Meta() plugin.PluginInfo { - return plugin.PluginInfo(p) -} - -func (p SafePlugin) Validate(endpoint plugin.ShieldEndpoint) error { - var ( - s string - err error - fail bool - ) - - s, err = endpoint.StringValue("vault_url") - if err != nil { - fmt.Printf("@R{\u2717 vault_url %s}\n", err) - fail = true - } else { - fmt.Printf("@G{\u2713 vault_url} @C{%s}\n", s) - } - - s, err = endpoint.StringValue("auth_token") - if err != nil { - fmt.Printf("@R{\u2717 auth_token %s}\n", err) - fail = true - } else { - fmt.Printf("@G{\u2713 auth_token} @C{%s}\n", plugin.Redact(s)) - } - - skipValidation, err := endpoint.BooleanValueDefault("skip_ssl_validation", false) - if err != nil { - fmt.Printf("@R{\u2717 skip_ssl_validation %s}\n", err) - fail = true - } else { - fmt.Printf("@G{\u2713 skip_ssl_validation} @C{%t}\n", skipValidation) - } - - if fail { - return fmt.Errorf("safe: invalid configuration") - } - return nil -} - -func (p SafePlugin) Backup(endpoint plugin.ShieldEndpoint) error { - - v, err := connect(endpoint) - if err != nil { - return err - } - - plugin.DEBUG("Reading secrets from the Vault...") - data := make(SafeContents) - if err = data.ReadFromVault(v, "secret"); err != nil { - return err - } - - return data.WriteToStdout() -} - -func (p SafePlugin) Restore(endpoint plugin.ShieldEndpoint) error { - - v, err := connect(endpoint) - if err != nil { - return err - } - - plugin.DEBUG("Reading secrets backup from stdin...") - data := make(SafeContents) - err = data.ReadFromStdin() - if err != nil { - return err - } - - plugin.DEBUG("Grabbing Safe metadata for current Vault") - sealKeysData := make(SafeContents) - sealKeysErr := sealKeysData.ReadFromVault(v, "secret/vault/seal/keys") - if sealKeysErr != nil && !vault.IsNotFound(sealKeysErr) { - return sealKeysErr - } - - plugin.DEBUG("Cleaning out the old secrets from the Vault...") - if err = v.DeleteTree("secret"); err != nil && !vault.IsNotFound(err) { - return err - } - - plugin.DEBUG("Restoring backup contents to the Vault") - err = data.WriteToVault(v) - if err != nil { - return err - } - - if sealKeysErr == nil { - plugin.DEBUG("Rewriting Safe metadata to the Vault") - return sealKeysData.WriteToVault(v) - } - return nil -} - -func (p SafePlugin) Store(endpoint plugin.ShieldEndpoint) (string, int64, error) { - return "", 0, plugin.UNIMPLEMENTED -} - -func (p SafePlugin) Retrieve(endpoint plugin.ShieldEndpoint, file string) error { - return plugin.UNIMPLEMENTED -} - -func (p SafePlugin) Purge(endpoint plugin.ShieldEndpoint, file string) error { - return plugin.UNIMPLEMENTED -} - -func connect(endpoint plugin.ShieldEndpoint) (*vault.Vault, error) { - url, err := endpoint.StringValue("vault_url") - if err != nil { - return nil, err - } - plugin.DEBUG("VAULT_URL: '%s'", url) - - token, err := endpoint.StringValue("auth_token") - if err != nil { - return nil, err - } - plugin.DEBUG("AUTH_TOKEN: '%s'", token) - - skipSslValidation, err := endpoint.BooleanValueDefault("skip_ssl_validation", false) - if err != nil { - return nil, err - } - if skipSslValidation { - plugin.DEBUG("Skipping SSL validation") - os.Setenv("VAULT_SKIP_VERIFY", "1") - } - v, err := vault.NewVault(url, token, true) - return v, err -} - -type SafeContents map[string]*vault.Secret - -func (sc SafeContents) ReadFromVault(v *vault.Vault, path string) error { - - tree, err := v.Tree(path, vault.TreeOptions{ - StripSlashes: true, - }) - if err != nil { - return err - } - for _, sub := range tree.Paths("/") { - s, err := v.Read(sub) - if err != nil { - return err - } - sc[sub] = s - } - return nil -} - -func (sc SafeContents) ReadFromStdin() error { - b, err := ioutil.ReadAll(os.Stdin) - if err != nil { - return err - } - return json.Unmarshal(b, &sc) -} - -func (sc SafeContents) WriteToVault(v *vault.Vault) error { - for path, s := range sc { - err := v.Write(path, s) - if err != nil { - return err - } - plugin.DEBUG(" -- wrote contents to %s", path) - } - return nil -} - -func (sc SafeContents) WriteToStdout() error { - b, err := json.Marshal(sc) - if err != nil { - return err - } - fmt.Printf("%s\n", string(b)) - return nil -} diff --git a/plugin/vault/plugin.go b/plugin/vault/plugin.go new file mode 100644 index 000000000..af2a006e3 --- /dev/null +++ b/plugin/vault/plugin.go @@ -0,0 +1,314 @@ +package main + +import ( + "encoding/json" + "io/ioutil" + "os" + "strings" + + fmt "github.com/jhunt/go-ansi" + + "github.com/starkandwayne/safe/vault" + "github.com/starkandwayne/shield/plugin" +) + +var () + +func main() { + p := VaultPlugin{ + Name: "Vault Backup Plugin", + Author: "Stark & Wayne", + Version: "0.0.1", + Features: plugin.PluginFeatures{ + Target: "yes", + Store: "no", + }, + Example: ` +{ + "url" : "https://vault.myorg.mycompany.com", # REQUIRED + "token" : "b8714fec-0df9-3f66-d262-35a57e414120", # REQUIRED + "skip_ssl_validation" : true, # REQUIRED + + "subtree" : "secret/some/sub/tree", # OPTIONAL +} +`, + Defaults: ` +{ + "subtree" : "secret", + "skip_ssl_validation" : false +} +`, + Fields: []plugin.Field{ + plugin.Field{ + Mode: "target", + Name: "url", + Type: "string", + Title: "Vault URL", + Help: "The url address of your Vault server, including the protocol.", + Example: "https://vault.myorg.mycompany.com", + Required: true, + }, + plugin.Field{ + Mode: "target", + Name: "token", + Type: "string", + Title: "Auth Token", + Help: "The auth token for a user with privileges to read and write the entire secret/ tree.", + Required: true, + }, + plugin.Field{ + Mode: "target", + Name: "subtree", + Type: "string", + Title: "Vault Path Subtree", + Help: "A subtree to limit the backup operation to.", + Default: "", + }, + plugin.Field{ + Mode: "target", + Name: "skip_ssl_validation", + Type: "bool", + Title: "Skip SSL Validation", + Help: "If your Vault certificate is invalid, expired, or signed by an unknown Certificate Authority, you can disable SSL validation. This is not recommended from a security standpoint, however.", + Default: "false", + }, + }, + } + + plugin.Run(p) +} + +type VaultPlugin plugin.PluginInfo + +func (p VaultPlugin) Meta() plugin.PluginInfo { + return plugin.PluginInfo(p) +} + +func (p VaultPlugin) Validate(endpoint plugin.ShieldEndpoint) error { + var ( + s string + err error + fail bool + ) + + s, err = endpoint.StringValue("url") + if err != nil { + fmt.Printf("@R{\u2717 url %s}\n", err) + fail = true + } else { + fmt.Printf("@G{\u2713 url} @C{%s}\n", s) + } + + s, err = endpoint.StringValue("token") + if err != nil { + fmt.Printf("@R{\u2717 token %s}\n", err) + fail = true + } else { + fmt.Printf("@G{\u2713 token} @C{%s}\n", plugin.Redact(s)) + } + + s, err = endpoint.StringValueDefault("subtree", "") + if err != nil { + fmt.Printf("@R{\u2717 subtree %s}\n", err) + fail = true + } else if s == "" { + fmt.Printf("@G{\u2713 subtree} @C{secret}/* (everything)\n") + } else { + fmt.Printf("@G{\u2713 subtree} @C{%s}/*\n", s) + } + + yes, err := endpoint.BooleanValueDefault("skip_ssl_validation", false) + if err != nil { + fmt.Printf("@R{\u2717 skip_ssl_validation %s}\n", err) + fail = true + } else { + fmt.Printf("@G{\u2713 skip_ssl_validation} @C{%t}\n", yes) + } + + if fail { + return fmt.Errorf("vault: invalid configuration") + } + return nil +} + +func (p VaultPlugin) Backup(endpoint plugin.ShieldEndpoint) error { + var data Vault + + v, subtree, err := connect(endpoint) + if err != nil { + return err + } + + if subtree == "" { + subtree = "secret" + } + + plugin.DEBUG("Reading %s/* from the Vault...", subtree) + data = make(Vault) + if err = data.Export(v, subtree); err != nil { + return err + } + + plugin.DEBUG("Exported %d paths from the Vault successfully.", len(data)) + return data.Write() +} + +func (p VaultPlugin) Restore(endpoint plugin.ShieldEndpoint) error { + var ( + preserve bool + data, prev Vault + ) + + v, subtree, err := connect(endpoint) + if err != nil { + return err + } + + if subtree == "" { + subtree = "secret" + preserve = true + } + + plugin.DEBUG("Reading contents of backup archive...") + data = make(Vault) + if err = data.Read(subtree); err != nil { + return err + } + + if preserve { + prev = make(Vault) + plugin.DEBUG("Saving seal keys for current Vault...") + err = prev.Export(v, "secret/vault/seal/keys") + if err != nil { + if !vault.IsNotFound(err) { + return err + } + prev = nil + } + } + + plugin.DEBUG("Deleting pre-existing contents of %s/* from Vault...", subtree) + if err = v.DeleteTree(subtree); err != nil && !vault.IsNotFound(err) { + return err + } + + plugin.DEBUG("Restoring %d paths to the Vault...", len(data)) + if err = data.Import(v); err != nil { + return err + } + + if prev != nil { + plugin.DEBUG("Replacing seal keys for current Vault (overwriting those from the backup archive)...") + return prev.Import(v) + } + return nil +} + +func (p VaultPlugin) Store(endpoint plugin.ShieldEndpoint) (string, int64, error) { + return "", 0, plugin.UNIMPLEMENTED +} + +func (p VaultPlugin) Retrieve(endpoint plugin.ShieldEndpoint, file string) error { + return plugin.UNIMPLEMENTED +} + +func (p VaultPlugin) Purge(endpoint plugin.ShieldEndpoint, file string) error { + return plugin.UNIMPLEMENTED +} + +func connect(endpoint plugin.ShieldEndpoint) (*vault.Vault, string, error) { + url, err := endpoint.StringValue("url") + if err != nil { + return nil, "", err + } + plugin.DEBUG("VAULT_URL: '%s'", url) + + token, err := endpoint.StringValue("token") + if err != nil { + return nil, "", err + } + plugin.DEBUG("AUTH_TOKEN: '%s'", token) + + skipSslValidation, err := endpoint.BooleanValueDefault("skip_ssl_validation", false) + if err != nil { + return nil, "", err + } + if skipSslValidation { + plugin.DEBUG("Skipping SSL validation") + os.Setenv("VAULT_SKIP_VERIFY", "1") + } + + subtree, err := endpoint.StringValueDefault("subtree", "") + if err != nil { + return nil, "", err + } + + v, err := vault.NewVault(url, token, true) + return v, subtree, err +} + +type Vault map[string]*vault.Secret + +func (v Vault) Export(from *vault.Vault, path string) error { + tree, err := from.Tree(path, vault.TreeOptions{ + StripSlashes: true, + }) + if err != nil { + return err + } + + for _, path := range tree.Paths("/") { + s, err := from.Read(path) + if err != nil { + return err + } + plugin.DEBUG(" -- read %s", path) + v[path] = s + } + return nil +} + +func (v Vault) Import(to *vault.Vault) error { + for path, s := range v { + err := to.Write(path, s) + if err != nil { + return err + } + plugin.DEBUG(" -- wrote contents to %s", path) + } + return nil +} + +func (v Vault) Read(subtree string) error { + b, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return err + } + + if err := json.Unmarshal(b, &v); err != nil { + return err + } + + if !strings.HasSuffix(subtree, "/") { + subtree += "/" + } + + for key := range v { + if !strings.HasPrefix(key, subtree) { + plugin.DEBUG(" -- IGNORING %s (not under %s*)", key, subtree) + delete(v, key) + } + } + + return nil +} + +func (v Vault) Write() error { + b, err := json.Marshal(v) + if err != nil { + return err + } + + fmt.Printf("%s\n", string(b)) + return nil +} diff --git a/plugins b/plugins index 6b356b943..abdd35425 100644 --- a/plugins +++ b/plugins @@ -14,5 +14,6 @@ rabbitmq-broker redis-broker s3 swift +vault webdav xtrabackup diff --git a/t/api b/t/api index 63a7c9a18..a40e53863 100755 --- a/t/api +++ b/t/api @@ -61,8 +61,8 @@ EOF set -e echo ">> Setting up a local (loopback) Vault" export PATH=$PATH:$WORKDIR - vault version - vault server -config ${WORKDIR}/etc/vault.conf 2>&1 & + $(pwd)/bin/vault version + $(pwd)/bin/vault server -config ${WORKDIR}/etc/vault.conf 2>&1 & set +e } @@ -210,7 +210,7 @@ while [[ $# -ne 0 ]]; do TESTS="$TESTS $1 " shift done -echo "running: $TESTS" | sed -e 's/ */ /' +echo "running: ${TESTS:-(all tests)}" | sed -e 's/ */ /' # }}} spin_vault 2>&1 > ${WORKDIR}/vault.log