From f850baa2741a048bbde88df5746e22577a3287bd Mon Sep 17 00:00:00 2001 From: Dave Henderson Date: Sun, 3 Feb 2019 23:19:54 -0500 Subject: [PATCH] Adding support for .env files Signed-off-by: Dave Henderson --- Gopkg.lock | 9 + data/data.go | 7 + data/data_test.go | 22 ++ data/datasource.go | 3 + data/mimetypes.go | 1 + docs/content/datasources.md | 31 +- tests/integration/datasources_file_test.go | 20 ++ vendor/github.com/joho/godotenv/LICENCE | 23 ++ vendor/github.com/joho/godotenv/godotenv.go | 346 ++++++++++++++++++++ 9 files changed, 458 insertions(+), 4 deletions(-) create mode 100644 vendor/github.com/joho/godotenv/LICENCE create mode 100644 vendor/github.com/joho/godotenv/godotenv.go diff --git a/Gopkg.lock b/Gopkg.lock index b64def7e1..53d0666ac 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -304,6 +304,14 @@ pruneopts = "NUT" revision = "0b12d6b5" +[[projects]] + digest = "1:da62aa6632d04e080b8a8b85a59ed9ed1550842a0099a55f3ae3a20d02a3745a" + name = "github.com/joho/godotenv" + packages = ["."] + pruneopts = "NUT" + revision = "23d116af351c84513e1946b527c88823e476be13" + version = "v1.3.0" + [[projects]] digest = "1:7b21c7fc5551b46d1308b4ffa9e9e49b66c7a8b0ba88c0130474b0e7a20d859f" name = "github.com/kr/pretty" @@ -537,6 +545,7 @@ "github.com/hashicorp/go-sockaddr/template", "github.com/hashicorp/vault/api", "github.com/imdario/mergo", + "github.com/joho/godotenv", "github.com/pkg/errors", "github.com/spf13/afero", "github.com/spf13/cobra", diff --git a/data/data.go b/data/data.go index c8cb9d8e2..61d7808a7 100644 --- a/data/data.go +++ b/data/data.go @@ -7,6 +7,8 @@ import ( "reflect" "strings" + "github.com/joho/godotenv" + "github.com/Shopify/ejson" ejsonJson "github.com/Shopify/ejson/json" "github.com/hairyhenderson/gomplate/env" @@ -100,6 +102,11 @@ func TOML(in string) (interface{}, error) { return unmarshalObj(obj, in, toml.Unmarshal) } +// dotEnv - Unmarshal a dotenv file +func dotEnv(in string) (interface{}, error) { + return godotenv.Unmarshal(in) +} + func parseCSV(args ...string) ([][]string, []string, error) { in, delim, hdr := csvParseArgs(args...) c := csv.NewReader(strings.NewReader(in)) diff --git a/data/data_test.go b/data/data_test.go index cd9dea74d..c640d78ef 100644 --- a/data/data_test.go +++ b/data/data_test.go @@ -460,3 +460,25 @@ func TestDecryptEJSON(t *testing.T) { assert.NoError(t, err) assert.EqualValues(t, expected, actual) } + +func TestDotEnv(t *testing.T) { + in := `FOO=a regular unquoted value +export BAR=another value, exports are ignored + +# comments are totally ignored, as are blank lines +FOO.BAR = "values can be double-quoted, and shell\nescapes are supported" + +BAZ = "variable expansion: ${FOO}" +QUX='single quotes ignore $variables' +` + expected := map[string]string{ + "FOO": "a regular unquoted value", + "BAR": "another value, exports are ignored", + "FOO.BAR": "values can be double-quoted, and shell\nescapes are supported", + "BAZ": "variable expansion: a regular unquoted value", + "QUX": "single quotes ignore $variables", + } + out, err := dotEnv(in) + assert.NoError(t, err) + assert.EqualValues(t, expected, out) +} diff --git a/data/datasource.go b/data/datasource.go index 9c1ca2742..47ce6b81e 100644 --- a/data/datasource.go +++ b/data/datasource.go @@ -36,6 +36,7 @@ func init() { regExtension(".yaml", yamlMimetype) regExtension(".csv", csvMimetype) regExtension(".toml", tomlMimetype) + regExtension(".env", envMimetype) } // registerReaders registers the source-reader functions @@ -338,6 +339,8 @@ func parseData(mimeType, s string) (out interface{}, err error) { out, err = CSV(s) case tomlMimetype: out, err = TOML(s) + case envMimetype: + out, err = dotEnv(s) case textMimetype: out = s default: diff --git a/data/mimetypes.go b/data/mimetypes.go index 99e952ce2..4117aa436 100644 --- a/data/mimetypes.go +++ b/data/mimetypes.go @@ -7,4 +7,5 @@ const ( jsonArrayMimetype = "application/array+json" tomlMimetype = "application/toml" yamlMimetype = "application/yaml" + envMimetype = "application/x-env" ) diff --git a/docs/content/datasources.md b/docs/content/datasources.md index 24be5ad74..d662c1247 100644 --- a/docs/content/datasources.md +++ b/docs/content/datasources.md @@ -100,12 +100,13 @@ These are the supported types: | Format | MIME Type | Extension(s) | Notes | |--------|-----------|-------|------| -| CSV | `text/csv` | | Uses the [`data.CSV`][] function to present the file as a 2-dimensional row-first string array | -| JSON | `application/json` | | [JSON][] _objects_ are assumed, and arrays or other values are not parsed with this type. Uses the [`data.JSON`][] function for parsing. [EJSON][] (encrypted JSON) is supported and will be decrypted. | +| CSV | `text/csv` | `.csv` | Uses the [`data.CSV`][] function to present the file as a 2-dimensional row-first string array | +| JSON | `application/json` | `.json` | [JSON][] _objects_ are assumed, and arrays or other values are not parsed with this type. Uses the [`data.JSON`][] function for parsing. [EJSON][] (encrypted JSON) is supported and will be decrypted. | | JSON Array | `application/array+json` | | A special type for parsing datasources containing just JSON arrays. Uses the [`data.JSONArray`][] function for parsing | | Plain Text | `text/plain` | | Unstructured, and as such only intended for use with the [`include`][] function | -| TOML | `application/toml` | | Parses [TOML][] with the [`data.TOML`][] function | -| YAML | `application/yaml` | | Parses [YAML][] with the [`data.YAML`][] function | +| TOML | `application/toml` | `.toml` | Parses [TOML][] with the [`data.TOML`][] function | +| YAML | `application/yaml` | `.yml`, `.yaml` | Parses [YAML][] with the [`data.YAML`][] function | +| [.env](#the-env-file-format) | `application/x-env` | `.env` | Basically just a file of `key=value` pairs separated by newlines, usually intended for sourcing into a shell. Common in [Docker Compose](https://docs.docker.com/compose/env-file/), [Ruby](https://github.com/bkeepers/dotenv), and [Node.js](https://github.com/motdotla/dotenv) applications. See [below](#the-env-file-format) for more information. | ### Overriding MIME Types @@ -119,6 +120,28 @@ $ gomplate -d data=file:///tmp/data.txt?type=application/json -i '{{ (ds "data") bar ``` +### The `.env` file format + +Many applications and frameworks support the use of a ".env" file for providing environment variables. It can also be considerd a simple key/value file format, and as such can be used as a datasource in gomplate. + +To [override](#overriding-mime-types), use the unregistered `application/x-env` MIME type. + +Here's a sample explaining the syntax: + +```bash +FOO=a regular unquoted value +export BAR=another value, exports are ignored + +# comments are totally ignored, as are blank lines +FOO.BAR = "values can be double-quoted, and\tshell\nescapes are supported" + +BAZ="variable expansion: ${FOO}" +QUX='single quotes ignore $variables and newlines' +``` + +The [`github.com/joho/godotenv`](https://github.com/joho/godotenv) package is used for parsing - see the full details there. + + ## Using `aws+smp` datasources The `aws+smp://` scheme can be used to retrieve data from the [AWS Systems Manager](https://aws.amazon.com/systems-manager/) (née AWS EC2 Simple Systems Manager) [Parameter Store](https://aws.amazon.com/systems-manager/features/#Parameter_Store). This hierarchically organized key/value store allows you to store text, lists or encrypted secrets for easy retrieval by AWS resources. See [the AWS Systems Manager documentation](https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-paramstore-su-create.html#sysman-paramstore-su-create-about) for details on creating these parameters. diff --git a/tests/integration/datasources_file_test.go b/tests/integration/datasources_file_test.go index 7da6b90e3..243c394b7 100644 --- a/tests/integration/datasources_file_test.go +++ b/tests/integration/datasources_file_test.go @@ -30,6 +30,15 @@ func (s *FileDatasourcesSuite) SetUpSuite(c *C) { A1,B1 A2,"foo"" bar" +`, + "test.env": `FOO=a regular unquoted value +export BAR=another value, exports are ignored + +# comments are totally ignored, as are blank lines +FOO.BAR = "values can be double-quoted, and shell\nescapes are supported" + +BAZ = "variable expansion: ${FOO}" +QUX='single quotes ignore $variables' `, }), fs.WithDir("sortorder", fs.WithFiles(map[string]string{ @@ -144,4 +153,15 @@ bar`}) zonef = "false" } `}) + result = icmd.RunCommand(GomplateBin, + "-d", "envfile="+s.tmpDir.Join("test.env"), + "-i", `{{ (ds "envfile") | data.ToJSONPretty " " }}`, + ) + result.Assert(c, icmd.Expected{ExitCode: 0, Out: `{ + "BAR": "another value, exports are ignored", + "BAZ": "variable expansion: a regular unquoted value", + "FOO": "a regular unquoted value", + "FOO.BAR": "values can be double-quoted, and shell\nescapes are supported", + "QUX": "single quotes ignore $variables" +}`}) } diff --git a/vendor/github.com/joho/godotenv/LICENCE b/vendor/github.com/joho/godotenv/LICENCE new file mode 100644 index 000000000..e7ddd51be --- /dev/null +++ b/vendor/github.com/joho/godotenv/LICENCE @@ -0,0 +1,23 @@ +Copyright (c) 2013 John Barton + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/vendor/github.com/joho/godotenv/godotenv.go b/vendor/github.com/joho/godotenv/godotenv.go new file mode 100644 index 000000000..29b436c77 --- /dev/null +++ b/vendor/github.com/joho/godotenv/godotenv.go @@ -0,0 +1,346 @@ +// Package godotenv is a go port of the ruby dotenv library (https://github.com/bkeepers/dotenv) +// +// Examples/readme can be found on the github page at https://github.com/joho/godotenv +// +// The TL;DR is that you make a .env file that looks something like +// +// SOME_ENV_VAR=somevalue +// +// and then in your go code you can call +// +// godotenv.Load() +// +// and all the env vars declared in .env will be available through os.Getenv("SOME_ENV_VAR") +package godotenv + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "os/exec" + "regexp" + "sort" + "strings" +) + +const doubleQuoteSpecialChars = "\\\n\r\"!$`" + +// Load will read your env file(s) and load them into ENV for this process. +// +// Call this function as close as possible to the start of your program (ideally in main) +// +// If you call Load without any args it will default to loading .env in the current path +// +// You can otherwise tell it which files to load (there can be more than one) like +// +// godotenv.Load("fileone", "filetwo") +// +// It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults +func Load(filenames ...string) (err error) { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + err = loadFile(filename, false) + if err != nil { + return // return early on a spazout + } + } + return +} + +// Overload will read your env file(s) and load them into ENV for this process. +// +// Call this function as close as possible to the start of your program (ideally in main) +// +// If you call Overload without any args it will default to loading .env in the current path +// +// You can otherwise tell it which files to load (there can be more than one) like +// +// godotenv.Overload("fileone", "filetwo") +// +// It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefilly set all vars. +func Overload(filenames ...string) (err error) { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + err = loadFile(filename, true) + if err != nil { + return // return early on a spazout + } + } + return +} + +// Read all env (with same file loading semantics as Load) but return values as +// a map rather than automatically writing values into env +func Read(filenames ...string) (envMap map[string]string, err error) { + filenames = filenamesOrDefault(filenames) + envMap = make(map[string]string) + + for _, filename := range filenames { + individualEnvMap, individualErr := readFile(filename) + + if individualErr != nil { + err = individualErr + return // return early on a spazout + } + + for key, value := range individualEnvMap { + envMap[key] = value + } + } + + return +} + +// Parse reads an env file from io.Reader, returning a map of keys and values. +func Parse(r io.Reader) (envMap map[string]string, err error) { + envMap = make(map[string]string) + + var lines []string + scanner := bufio.NewScanner(r) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + + if err = scanner.Err(); err != nil { + return + } + + for _, fullLine := range lines { + if !isIgnoredLine(fullLine) { + var key, value string + key, value, err = parseLine(fullLine, envMap) + + if err != nil { + return + } + envMap[key] = value + } + } + return +} + +//Unmarshal reads an env file from a string, returning a map of keys and values. +func Unmarshal(str string) (envMap map[string]string, err error) { + return Parse(strings.NewReader(str)) +} + +// Exec loads env vars from the specified filenames (empty map falls back to default) +// then executes the cmd specified. +// +// Simply hooks up os.Stdin/err/out to the command and calls Run() +// +// If you want more fine grained control over your command it's recommended +// that you use `Load()` or `Read()` and the `os/exec` package yourself. +func Exec(filenames []string, cmd string, cmdArgs []string) error { + Load(filenames...) + + command := exec.Command(cmd, cmdArgs...) + command.Stdin = os.Stdin + command.Stdout = os.Stdout + command.Stderr = os.Stderr + return command.Run() +} + +// Write serializes the given environment and writes it to a file +func Write(envMap map[string]string, filename string) error { + content, error := Marshal(envMap) + if error != nil { + return error + } + file, error := os.Create(filename) + if error != nil { + return error + } + _, err := file.WriteString(content) + return err +} + +// Marshal outputs the given environment as a dotenv-formatted environment file. +// Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped. +func Marshal(envMap map[string]string) (string, error) { + lines := make([]string, 0, len(envMap)) + for k, v := range envMap { + lines = append(lines, fmt.Sprintf(`%s="%s"`, k, doubleQuoteEscape(v))) + } + sort.Strings(lines) + return strings.Join(lines, "\n"), nil +} + +func filenamesOrDefault(filenames []string) []string { + if len(filenames) == 0 { + return []string{".env"} + } + return filenames +} + +func loadFile(filename string, overload bool) error { + envMap, err := readFile(filename) + if err != nil { + return err + } + + currentEnv := map[string]bool{} + rawEnv := os.Environ() + for _, rawEnvLine := range rawEnv { + key := strings.Split(rawEnvLine, "=")[0] + currentEnv[key] = true + } + + for key, value := range envMap { + if !currentEnv[key] || overload { + os.Setenv(key, value) + } + } + + return nil +} + +func readFile(filename string) (envMap map[string]string, err error) { + file, err := os.Open(filename) + if err != nil { + return + } + defer file.Close() + + return Parse(file) +} + +func parseLine(line string, envMap map[string]string) (key string, value string, err error) { + if len(line) == 0 { + err = errors.New("zero length string") + return + } + + // ditch the comments (but keep quoted hashes) + if strings.Contains(line, "#") { + segmentsBetweenHashes := strings.Split(line, "#") + quotesAreOpen := false + var segmentsToKeep []string + for _, segment := range segmentsBetweenHashes { + if strings.Count(segment, "\"") == 1 || strings.Count(segment, "'") == 1 { + if quotesAreOpen { + quotesAreOpen = false + segmentsToKeep = append(segmentsToKeep, segment) + } else { + quotesAreOpen = true + } + } + + if len(segmentsToKeep) == 0 || quotesAreOpen { + segmentsToKeep = append(segmentsToKeep, segment) + } + } + + line = strings.Join(segmentsToKeep, "#") + } + + firstEquals := strings.Index(line, "=") + firstColon := strings.Index(line, ":") + splitString := strings.SplitN(line, "=", 2) + if firstColon != -1 && (firstColon < firstEquals || firstEquals == -1) { + //this is a yaml-style line + splitString = strings.SplitN(line, ":", 2) + } + + if len(splitString) != 2 { + err = errors.New("Can't separate key from value") + return + } + + // Parse the key + key = splitString[0] + if strings.HasPrefix(key, "export") { + key = strings.TrimPrefix(key, "export") + } + key = strings.Trim(key, " ") + + // Parse the value + value = parseValue(splitString[1], envMap) + return +} + +func parseValue(value string, envMap map[string]string) string { + + // trim + value = strings.Trim(value, " ") + + // check if we've got quoted values or possible escapes + if len(value) > 1 { + rs := regexp.MustCompile(`\A'(.*)'\z`) + singleQuotes := rs.FindStringSubmatch(value) + + rd := regexp.MustCompile(`\A"(.*)"\z`) + doubleQuotes := rd.FindStringSubmatch(value) + + if singleQuotes != nil || doubleQuotes != nil { + // pull the quotes off the edges + value = value[1 : len(value)-1] + } + + if doubleQuotes != nil { + // expand newlines + escapeRegex := regexp.MustCompile(`\\.`) + value = escapeRegex.ReplaceAllStringFunc(value, func(match string) string { + c := strings.TrimPrefix(match, `\`) + switch c { + case "n": + return "\n" + case "r": + return "\r" + default: + return match + } + }) + // unescape characters + e := regexp.MustCompile(`\\([^$])`) + value = e.ReplaceAllString(value, "$1") + } + + if singleQuotes == nil { + value = expandVariables(value, envMap) + } + } + + return value +} + +func expandVariables(v string, m map[string]string) string { + r := regexp.MustCompile(`(\\)?(\$)(\()?\{?([A-Z0-9_]+)?\}?`) + + return r.ReplaceAllStringFunc(v, func(s string) string { + submatch := r.FindStringSubmatch(s) + + if submatch == nil { + return s + } + if submatch[1] == "\\" || submatch[2] == "(" { + return submatch[0][1:] + } else if submatch[4] != "" { + return m[submatch[4]] + } + return s + }) +} + +func isIgnoredLine(line string) bool { + trimmedLine := strings.Trim(line, " \n\t") + return len(trimmedLine) == 0 || strings.HasPrefix(trimmedLine, "#") +} + +func doubleQuoteEscape(line string) string { + for _, c := range doubleQuoteSpecialChars { + toReplace := "\\" + string(c) + if c == '\n' { + toReplace = `\n` + } + if c == '\r' { + toReplace = `\r` + } + line = strings.Replace(line, string(c), toReplace, -1) + } + return line +}