diff --git a/s3secrets-helper/go.mod b/s3secrets-helper/go.mod index 4baa6e0..be3badd 100644 --- a/s3secrets-helper/go.mod +++ b/s3secrets-helper/go.mod @@ -2,4 +2,11 @@ module github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v go 1.15 -require github.com/aws/aws-sdk-go v1.35.14 +require ( + github.com/aws/aws-sdk-go-v2 v1.9.2 // indirect + github.com/aws/aws-sdk-go-v2/config v1.8.3 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.16.1 + github.com/aws/smithy-go v1.8.0 +) diff --git a/s3secrets-helper/go.sum b/s3secrets-helper/go.sum index 1005132..38c93dd 100644 --- a/s3secrets-helper/go.sum +++ b/s3secrets-helper/go.sum @@ -1,16 +1,44 @@ -github.com/aws/aws-sdk-go v1.35.14 h1:nucVVXXjAr9UkmYCBWxQWRuYa5KOlaXjuJGg2ulW0K0= -github.com/aws/aws-sdk-go v1.35.14/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= -github.com/buildkite/elastic-ci-stack-s3-secrets-hooks v1.3.0 h1:CRdPqLjYECJY0cgs24E4ZwUJ2qlmcavN7W1jo+5ujnQ= +github.com/aws/aws-sdk-go-v2 v1.9.2 h1:dUFQcMNZMLON4BOe273pl0filK9RqyQMhCK/6xssL6s= +github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2/config v1.8.3 h1:o5583X4qUfuRrOGOgmOcDgvr5gJVSu57NK08cWAhIDk= +github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= +github.com/aws/aws-sdk-go-v2/credentials v1.4.3 h1:LTdD5QhK073MpElh9umLLP97wxphkgVC/OjQaEbBwZA= +github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0 h1:9tfxW/icbSu98C2pcNynm5jmDwU3/741F11688B6QnU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.4 h1:TnU1cY51027j/MQeFy7DIgk1UuzJY+wLFYqXceY/fiE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.4/go.mod h1:Ex7XQmbFmgFHrjUX6TN3mApKW5Hglyga+F7wZHTtYhA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4 h1:leSJ6vCqtPpTmBIgE7044B1wql1E4n//McF+mEgNrYg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 h1:gceOysEWNNwLd6cki65IMBZ4WAM0MwgBQq2n7kejoT8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0/go.mod h1:v8ygadNyATSm6elwJ/4gzJwcFhri9RqS8skgHKiwXPU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2 h1:r7jel2aa4d9Duys7wEmWqDd5ebpC9w6Kxu6wIjjp18E= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.2 h1:RnZjLgtCGLsF2xYYksy0yrx6xPvKG9BYv29VfK4p/J8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.7.2/go.mod h1:np7TMuJNT83O0oDOSF8i4dF3dvGqA6hPYYo6YYkzgRA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.16.1 h1:z+P3r4LrwdudLKBoEVWxIORrk4sVg4/iqpG3+CS53AY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.16.1/go.mod h1:CQe/KvWV1AqRc65KqeJjrLzr5X2ijnFTTVzJW0VBRCI= +github.com/aws/aws-sdk-go-v2/service/sso v1.4.2 h1:pZwkxZbspdqRGzddDB92bkZBoB7lg85sMRE7OqdB3V0= +github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= +github.com/aws/aws-sdk-go-v2/service/sts v1.7.2 h1:ol2Y5DWqnJeKqNd8th7JWzBtqu63xpOfs1Is+n1t8/4= +github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= +github.com/aws/smithy-go v1.8.0 h1:AEwwwXQZtUwP5Mz506FeXXrKBe0jA8gVM+1gEcSRooc= +github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/s3secrets-helper/s3/s3.go b/s3secrets-helper/s3/s3.go index f1dd0c6..a201e54 100644 --- a/s3secrets-helper/s3/s3.go +++ b/s3secrets-helper/s3/s3.go @@ -1,61 +1,73 @@ package s3 import ( + "context" + "errors" + "fmt" "log" "io/ioutil" "os" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/ec2metadata" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/buildkite/elastic-ci-stack-s3-secrets-hooks/s3secrets-helper/v2/sentinel" ) -const envDefaultRegion = "AWS_DEFAULT_REGION" - type Client struct { - s3 *s3.S3 + s3 *s3.Client bucket string } -func New(log *log.Logger, bucket string) (*Client, error) { - sess, err := session.NewSession() - if err != nil { - return nil, err +func getRegion(ctx context.Context) (string, error) { + if region := os.Getenv("AWS_DEFAULT_REGION"); len(region) > 0 { + return region, nil } - currentRegion := os.Getenv(envDefaultRegion) - // Discover our executing region using the IMDS - if currentRegion == "" { - idms := ec2metadata.New(sess) - currentRegion, _ = idms.Region() - } - // Fall back to us-east-1 :( - if currentRegion == "" { - currentRegion = "us-east-1" + imdsClient := imds.New(imds.Options{}) + if result, err := imdsClient.GetRegion(ctx, nil); err == nil { + if len(result.Region) > 0 { + return result.Region, nil + } } - log.Printf("Discovered current region as %q\n", currentRegion) + return "", errors.New("Unknown current region") +} + +func New(log *log.Logger, bucket string) (*Client, error) { + ctx := context.Background() // Using the current region (or a guess) find where the bucket lives - bucketRegion, err := s3manager.GetBucketRegion(aws.BackgroundContext(), sess, bucket, currentRegion) + + region, err := getRegion(ctx) if err != nil { - return nil, err + // Ignore error and fallback to us-east-1 for bucket lookup + region = "us-east-1" } - log.Printf("Discovered bucket region as %q\n", bucketRegion) + config, err := config.LoadDefaultConfig(ctx, + config.WithRegion(region), + ) + if err != nil { + return nil, fmt.Errorf("Could not load the AWS SDK config (%v)", err) + } - sess, err = session.NewSession(&aws.Config{ - Region: &bucketRegion, - }) + log.Printf("Discovered current region as %q\n", config.Region) + + bucketRegion, err := manager.GetBucketRegion(ctx, s3.NewFromConfig(config), bucket) if err != nil { - return nil, err + return nil, fmt.Errorf("Could not discover the region for bucket %q: (%v)", bucket, err) } + + log.Printf("Discovered bucket region as %q\n", bucketRegion) + + config.Region = bucketRegion + return &Client{ - s3: s3.New(sess), + s3: s3.NewFromConfig(config), bucket: bucket, }, nil } @@ -69,23 +81,26 @@ func (c *Client) Bucket() (string) { // sentinel.ErrNotFound and sentinel.ErrForbidden are returned for those cases. // Other errors are returned verbatim. func (c *Client) Get(key string) ([]byte, error) { - out, err := c.s3.GetObject(&s3.GetObjectInput{ + out, err := c.s3.GetObject(context.TODO(), &s3.GetObjectInput{ Bucket: &c.bucket, Key: &key, }) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - case "NoSuchKey": - return nil, sentinel.ErrNotFound - case "Forbidden": + var noSuchKey *types.NoSuchKey + if errors.As(err, &noSuchKey) { + return nil, sentinel.ErrNotFound + } + + // Possible values can be found at https://docs.aws.amazon.com/AmazonS3/latest/API/API_Error.html + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + code := apiErr.ErrorCode() + if code == "AccessDenied" { return nil, sentinel.ErrForbidden - default: - return nil, aerr } - } else { - return nil, err } + + return nil, fmt.Errorf("Could not GetObject (%s) in bucket (%s). Ensure your IAM Identity has s3:GetObject permission for this key and bucket. (%v)", key, c.bucket, err) } defer out.Body.Close() // we probably should return io.Reader or io.ReadCloser rather than []byte, @@ -98,18 +113,8 @@ func (c *Client) Get(key string) ([]byte, error) { // 404 Not Found and 403 Forbidden return false without error. // Other errors result in false with an error. func (c *Client) BucketExists() (bool, error) { - if _, err := c.s3.HeadBucket(&s3.HeadBucketInput{Bucket: &c.bucket}); err != nil { - if aerr, ok := err.(awserr.Error); ok { - switch aerr.Code() { - // https://github.com/aws/aws-sdk-go/issues/2593#issuecomment-491436818 - case "NoSuchBucket", "NotFound": - return false, nil - default: // e.g. NoCredentialProviders, Forbidden - return false, aerr - } - } else { - return false, err - } + if _, err := c.s3.HeadBucket(context.TODO(), &s3.HeadBucketInput{Bucket: &c.bucket}); err != nil { + return false, fmt.Errorf("Could not HeadBucket (%s). Ensure your IAM Identity has s3:ListBucket permission for this bucket. (%v)", c.bucket, err) } return true, nil } diff --git a/s3secrets-helper/sentinel/errors.go b/s3secrets-helper/sentinel/errors.go index ccff7d3..8a5396b 100644 --- a/s3secrets-helper/sentinel/errors.go +++ b/s3secrets-helper/sentinel/errors.go @@ -6,8 +6,8 @@ import "errors" var ( // ErrNotFound indicates something was not found - ErrNotFound = errors.New("Forbidden") + ErrNotFound = errors.New("NotFound") // ErrForbidden indicates something was forbidden - ErrForbidden = errors.New("NotFound") + ErrForbidden = errors.New("Forbidden") )