From 4490d125a2c40c89f7161645ef5d6ddb5c462d37 Mon Sep 17 00:00:00 2001 From: Elijah Wilson Date: Sat, 15 Jun 2019 15:07:52 +0200 Subject: [PATCH] AWS support (#1) * add AWS support * tests and such --- .circleci/config.yml | 15 +++++ .github/FUNDING.yml | 8 --- .travis.yml | 14 ----- Makefile | 12 ++++ README.md | 40 ++++++++---- aws.go | 98 ++++++++++++++++++++++++++++++ aws_test.go | 141 +++++++++++++++++++++++++++++++++++++++++++ config.go | 26 ++++++++ example_test.go | 25 +++++++- go.mod | 8 ++- go.sum | 18 ++++++ 11 files changed, 368 insertions(+), 37 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 .github/FUNDING.yml delete mode 100644 .travis.yml create mode 100644 Makefile create mode 100644 aws.go create mode 100644 aws_test.go create mode 100644 go.sum diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..67a9bc5 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,15 @@ +# Golang CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-go/ for more details +version: 2 +jobs: + build: + docker: + # specify the version + - image: circleci/golang:1.12 + working_directory: /go/src/github.com/hookactions/config + environment: # environment variables for the build itself + GO111MODULE: 'on' + steps: + - checkout + - run: make test diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index aa79bdc..0000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,8 +0,0 @@ -# These are supported funding model platforms - -github: [JeremyLoy] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -# patreon: # Replace with a single Patreon username -# open_collective: # Replace with a single Open Collective username -# ko_fi: # Replace with a single Ko-fi username -# tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -# custom: # Replace with a single custom sponsorship URL diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c71767b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ -language: go - -go: - - 1.x - -before_install: - - go get github.com/mattn/goveralls - - go get golang.org/x/tools/cmd/goimports - - go get -u golang.org/x/lint/golint -script: - - go vet - - golint -set_exit_status - - test -z "$(goimports -l .)" - - $HOME/gopath/bin/goveralls -service=travis-ci \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fb81160 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +test: + go get golang.org/x/tools/cmd/goimports + go get -u golang.org/x/lint/golint + + go vet ./... + + golint -set_exit_status + test -z "$(goimports -l .)" + + go test -v ./... -race -cover + + go mod tidy diff --git a/README.md b/README.md index ec16c58..c2694f6 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,22 @@ # Config -[![Documentation](https://godoc.org/github.com/JeremyLoy/config?status.svg)](http://godoc.org/github.com/JeremyLoy/config) -[![Mentioned in Awesome Go](https://awesome.re/mentioned-badge-flat.svg)](https://github.com/avelino/awesome-go) -[![Build Status](https://travis-ci.org/JeremyLoy/config.svg?branch=master)](https://travis-ci.org/JeremyLoy/config) -[![Go Report Card](https://goreportcard.com/badge/github.com/JeremyLoy/config)](https://goreportcard.com/report/github.com/JeremyLoy/config) -[![Coverage Status](https://coveralls.io/repos/github/JeremyLoy/config/badge.svg?branch=master)](https://coveralls.io/github/JeremyLoy/config?branch=master) -[![GitHub issues](https://img.shields.io/github/issues/JeremyLoy/config.svg)](https://github.com/JeremyLoy/config/issues) -[![license](https://img.shields.io/github/license/JeremyLoy/config.svg?maxAge=2592000)](https://github.com/JeremyLoy/config/LICENSE) -[![Release](https://img.shields.io/github/release/JeremyLoy/config.svg?label=Release)](https://github.com/JeremyLoy/config/releases) +[![Documentation](https://godoc.org/github.com/hookactions/config?status.svg)](http://godoc.org/github.com/hookactions/config) +[![CircleCI](https://circleci.com/gh/hookactions/config.svg?style=svg)](https://circleci.com/gh/hookactions/config) +[![Go Report Card](https://goreportcard.com/badge/github.com/hookactions/config)](https://goreportcard.com/report/github.com/hookactions/config) +[![license](https://img.shields.io/github/license/hookactions/config.svg?maxAge=2592000)](https://github.com/hookactions/config/LICENSE) +[![Release](https://img.shields.io/github/release/hookactions/config.svg?label=Release)](https://github.com/hookactions/config/releases) Manage your application config as a typesafe struct in as little as two function calls. ```go +package main + +import ( + "context" + "fmt" + + "github.com/hookactions/config" +) + type MyConfig struct { DatabaseUrl string `config:"DATABASE_URL"` FeatureFlag bool `config:"FEATURE_FLAG"` @@ -20,11 +26,22 @@ type MyConfig struct { var c MyConfig config.FromEnv().To(&c) + +fmt.Printf("%v\n", c) + +// Supports AWS Secret Manager and Parameter store +// sm://my_value +// ssm://my_value + +p, _ := config.NewAWSSecretManagerValuePreProcessor(context.Background(), true) +config.WithValuePreProcessor(p).FromEnv().To(&c) + +fmt.Printf("%v\n", c) ``` ## How It Works -Its just simple, pure stdlib. +Its just simple, pure stdlib with optional AWS support. * A field's type determines what [strconv](https://golang.org/pkg/strconv/) function is called. * All string conversion rules are as defined in the [strconv](https://golang.org/pkg/strconv/) package @@ -47,9 +64,6 @@ Its just simple, pure stdlib. * only 2 lines to configure. * Composeable: * Merge local files and environment variables for effortless local development. -* small: - * only stdlib - * < 180 LoC ## Design Philosophy @@ -64,4 +78,4 @@ Feel free to use it on its own, or alongside other libraries. * No maps. The only feature of maps not handled by structs for this usecase is dynamic keys. -* No pointer members. If you really need one, just take the address of parts of your struct. \ No newline at end of file +* No pointer members. If you really need one, just take the address of parts of your struct. diff --git a/aws.go b/aws.go new file mode 100644 index 0000000..b17662b --- /dev/null +++ b/aws.go @@ -0,0 +1,98 @@ +package config + +import ( + "context" + "regexp" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/external" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager/secretsmanageriface" + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/ssm/ssmiface" + "github.com/pkg/errors" +) + +var ( + secretsManagerStringRe = regexp.MustCompile("^sm://") + parameterStoreStringRe = regexp.MustCompile("^ssm://") +) + +func checkPrefixAndStrip(re *regexp.Regexp, s string) (string, bool) { + if re.MatchString(s) { + return re.ReplaceAllString(s, ""), true + } + return s, false +} + +// NewAWSSecretManagerValuePreProcessor creates a new AWSSecretManagerValuePreProcessor with the given context and whether to decrypt parameter store values or not. +// This will load the aws config from external.LoadDefaultAWSConfig() +func NewAWSSecretManagerValuePreProcessor(ctx context.Context, decryptParameterStoreValues bool) (*AWSSecretManagerValuePreProcessor, error) { + awsConfig, err := external.LoadDefaultAWSConfig() + if err != nil { + return nil, errors.Wrap(err, "config/aws: error loading default aws config") + } + + return &AWSSecretManagerValuePreProcessor{ + decryptParameterStoreValues: decryptParameterStoreValues, + + secretsManager: secretsmanager.New(awsConfig), + parameterStore: ssm.New(awsConfig), + ctx: ctx, + }, nil +} + +// AWSSecretManagerValuePreProcessor is a ValuePreProcessor for AWS. +// Supports Secrets Manager and Parameter Store. +type AWSSecretManagerValuePreProcessor struct { + decryptParameterStoreValues bool + + secretsManager secretsmanageriface.ClientAPI + parameterStore ssmiface.ClientAPI + ctx context.Context +} + +// PreProcessValue pre-processes a config key/value pair. +func (p *AWSSecretManagerValuePreProcessor) PreProcessValue(key, value string) string { + return p.processConfigItem(p.ctx, key, value) +} + +func (p *AWSSecretManagerValuePreProcessor) processConfigItem(ctx context.Context, key string, value string) string { + if v, ok := checkPrefixAndStrip(secretsManagerStringRe, value); ok { + return p.loadStringValueFromSecretsManager(ctx, v) + } else if v, ok := checkPrefixAndStrip(parameterStoreStringRe, v); ok { + return p.loadStringValueFromParameterStore(ctx, v, p.decryptParameterStoreValues) + } + return value +} + +func (p *AWSSecretManagerValuePreProcessor) loadStringValueFromSecretsManager(ctx context.Context, name string) string { + resp, err := p.requestSecret(ctx, name) + if err != nil { + panic("config/aws/loadStringValueFromSecretsManager: error loading secret, " + err.Error()) + } + + return *resp.SecretString +} + +func (p *AWSSecretManagerValuePreProcessor) requestSecret(ctx context.Context, name string) (*secretsmanager.GetSecretValueResponse, error) { + input := &secretsmanager.GetSecretValueInput{SecretId: aws.String(name)} + return p.secretsManager.GetSecretValueRequest(input).Send(ctx) +} + +func (p *AWSSecretManagerValuePreProcessor) loadStringValueFromParameterStore(ctx context.Context, name string, decrypt bool) string { + resp, err := p.requestParameter(ctx, name, decrypt) + if err != nil { + panic("config/aws/loadStringValueFromParameterStore: error loading value, " + err.Error()) + } + + return *resp.Parameter.Value +} + +func (p *AWSSecretManagerValuePreProcessor) requestParameter(ctx context.Context, name string, decrypt bool) (*ssm.GetParameterResponse, error) { + input := &ssm.GetParameterInput{Name: aws.String(name), WithDecryption: aws.Bool(decrypt)} + return p.parameterStore.GetParameterRequest(input).Send(ctx) +} + +// compile time assertion +var _ ValuePreProcessor = (*AWSSecretManagerValuePreProcessor)(nil) diff --git a/aws_test.go b/aws_test.go new file mode 100644 index 0000000..36c3a61 --- /dev/null +++ b/aws_test.go @@ -0,0 +1,141 @@ +package config + +import ( + "context" + "encoding/base64" + "net/http" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager/secretsmanageriface" + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/ssm/ssmiface" + "github.com/stretchr/testify/assert" +) + +type mockSecretManagerClient struct { + secretsmanageriface.ClientAPI + + checkInput func(*secretsmanager.GetSecretValueInput) + stringValue *string + binaryValue []byte +} + +func (m *mockSecretManagerClient) GetSecretValueRequest(in *secretsmanager.GetSecretValueInput) secretsmanager.GetSecretValueRequest { + if m.checkInput != nil { + m.checkInput(in) + } + + req := &aws.Request{ + Data: &secretsmanager.GetSecretValueOutput{ + SecretString: m.stringValue, + SecretBinary: m.binaryValue, + }, + HTTPRequest: new(http.Request), + } + return secretsmanager.GetSecretValueRequest{Request: req, Input: in, Copy: m.GetSecretValueRequest} +} + +type mockParameterStoreClient struct { + ssmiface.ClientAPI + + checkInput func(*ssm.GetParameterInput) + stringValue *string + binaryValue []byte +} + +func (m *mockParameterStoreClient) GetParameterRequest(in *ssm.GetParameterInput) ssm.GetParameterRequest { + if m.checkInput != nil { + m.checkInput(in) + } + + var value *string + + if m.stringValue != nil { + value = m.stringValue + } else if m.binaryValue != nil { + value = aws.String(base64.StdEncoding.EncodeToString(m.binaryValue)) + } + + req := &aws.Request{ + Data: &ssm.GetParameterOutput{ + Parameter: &ssm.Parameter{ + Value: value, + }, + }, + HTTPRequest: new(http.Request), + } + return ssm.GetParameterRequest{Request: req, Input: in, Copy: m.GetParameterRequest} +} + +func TestAWSSecretManagerValuePreProcessor_PreProcessValue(t *testing.T) { + ctx := context.Background() + + t.Run("NonPrefixedValues", func(t *testing.T) { + p := AWSSecretManagerValuePreProcessor{} + + assert.Equal(t, "bar", p.PreProcessValue("FOO_1", "bar")) + assert.Equal(t, "test", p.PreProcessValue("FOO_BAR_BAZ", "test")) + }) + + t.Run("SecretsManager", func(t *testing.T) { + manager := &mockSecretManagerClient{} + + p := &AWSSecretManagerValuePreProcessor{ + decryptParameterStoreValues: true, + secretsManager: manager, + ctx: ctx, + } + + t.Run("Simple", func(t *testing.T) { + manager.checkInput = func(input *secretsmanager.GetSecretValueInput) { + assert.Equal(t, "foo_bar", *input.SecretId) + } + manager.stringValue = aws.String("baz") + + assert.Equal(t, "baz", p.PreProcessValue("FOO", "sm://foo_bar")) + }) + + // "complex" in the sense that this would break using strings.TrimPrefix(...) + t.Run("Complex", func(t *testing.T) { + manager.checkInput = func(input *secretsmanager.GetSecretValueInput) { + assert.Equal(t, "small_foo_bar", *input.SecretId) + } + manager.stringValue = aws.String("baz") + + assert.Equal(t, "baz", p.PreProcessValue("FOO", "sm://small_foo_bar")) + }) + }) + + t.Run("ParameterStore", func(t *testing.T) { + storeClient := &mockParameterStoreClient{} + + p := &AWSSecretManagerValuePreProcessor{ + decryptParameterStoreValues: true, + parameterStore: storeClient, + ctx: ctx, + } + + t.Run("Simple", func(t *testing.T) { + storeClient.checkInput = func(input *ssm.GetParameterInput) { + assert.Equal(t, "foo_bar", *input.Name) + assert.True(t, *input.WithDecryption) + } + storeClient.stringValue = aws.String("baz") + + assert.Equal(t, "baz", p.PreProcessValue("FOO", "ssm://foo_bar")) + }) + + // "complex" in the sense that this would break using strings.TrimPrefix(...) + t.Run("Complex", func(t *testing.T) { + storeClient.checkInput = func(input *ssm.GetParameterInput) { + assert.Equal(t, "ssmall_foo_bar", *input.Name) + assert.True(t, *input.WithDecryption) + } + storeClient.stringValue = aws.String("baz") + + assert.Equal(t, "baz", p.PreProcessValue("FOO", "ssm://ssmall_foo_bar")) + }) + }) +} diff --git a/config.go b/config.go index 8e6c405..6ab09b8 100644 --- a/config.go +++ b/config.go @@ -37,10 +37,32 @@ const ( sliceDelim = " " ) +// ValuePreProcessor is an interface for pre-processing values +type ValuePreProcessor interface { + // PreProcessValue pre-processes a key/value pair for the config. + PreProcessValue(key, value string) string +} + // Builder contains the current configuration state. type Builder struct { structDelim, sliceDelim string configMap map[string]string + valuePreProcessor ValuePreProcessor +} + +// WithValuePreProcessor creates a new builder with a ValuePreProcessor. +// This will be called when adding values to the builder's configMap. +// An example use would be retrieving values from AWS secret manager. +func WithValuePreProcessor(p ValuePreProcessor) *Builder { + return newBuilder().WithValuePreProcessor(p) +} + +// WithValuePreProcessor adds a ValuePreProcessor to the builder. +// This will be called when adding values to the builder's configMap. +// An example use would be retrieving values from AWS secret manager. +func (c *Builder) WithValuePreProcessor(p ValuePreProcessor) *Builder { + c.valuePreProcessor = p + return c } func newBuilder() *Builder { @@ -99,6 +121,10 @@ func (c *Builder) FromEnv() *Builder { func (c *Builder) mergeConfig(in map[string]string) { for k, v := range in { + if c.valuePreProcessor != nil { + v = c.valuePreProcessor.PreProcessValue(k, v) + } + c.configMap[k] = v } } diff --git a/example_test.go b/example_test.go index 01ce2cb..986207b 100644 --- a/example_test.go +++ b/example_test.go @@ -7,7 +7,7 @@ import ( "os" "strings" - "github.com/JeremyLoy/config" + "github.com/hookactions/config" ) type MySubConfig struct { @@ -87,3 +87,26 @@ func Example_structTags() { // Output: // db:// } + +type MyPreProcessor struct{} + +func (p *MyPreProcessor) PreProcessValue(key, value string) string { + return strings.ToUpper(value) +} + +func ExampleWithValuePreProcessor() { + type MyConfig struct { + Foo string + } + + os.Clearenv() + os.Setenv("FOO", "bar") + + var c MyConfig + config.WithValuePreProcessor(&MyPreProcessor{}).FromEnv().To(&c) + + fmt.Println(c.Foo) + + // Output: + // BAR +} diff --git a/go.mod b/go.mod index c7d8557..cd86b39 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,9 @@ -module github.com/JeremyLoy/config +module github.com/hookactions/config go 1.12 + +require ( + github.com/aws/aws-sdk-go-v2 v0.9.0 + github.com/pkg/errors v0.8.0 + github.com/stretchr/testify v1.2.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a4d857b --- /dev/null +++ b/go.sum @@ -0,0 +1,18 @@ +github.com/aws/aws-sdk-go-v2 v0.9.0 h1:dWtJKGRFv3UZkMBQaIzMsF0/y4ge3iQPWTzeC4r/vl4= +github.com/aws/aws-sdk-go-v2 v0.9.0/go.mod h1:sa1GePZ/LfBGI4dSq30f6uR4Tthll8axxtEPvlpXZ8U= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=