Skip to content

Commit

Permalink
refactor(backend): use aws-sdk-go-v2 for object uploads (#1675)
Browse files Browse the repository at this point in the history 
- refactor attachment upload to use `aws-sdk-go-v2` instead of
`aws-sdk-go`
- refactor build mapping upload to use `aws-sdk-go-v2` instead of
`aws-sdk-go`

fixes #1669

---------

Signed-off-by: detj <[email protected]>
  • Loading branch information
@detj
detj authored Jan 1, 2025
1 parent 0a62043 commit c06fd96
Show file tree
Hide file tree
Showing 7 changed files with 177 additions and 102 deletions.
117 changes: 71 additions & 46 deletions backend/api/event/attachment.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package event

import (
"context"
"errors"
"fmt"
"io"
"mime"
"net/url"
Expand All @@ -11,11 +13,8 @@ import (

"backend/api/server"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
)

Expand Down Expand Up @@ -48,86 +47,112 @@ func (a Attachment) Validate() error {
return nil
}

// Upload uploads raw file bytes to an S3 compatible storage system.
func (a *Attachment) Upload() (output *s3manager.UploadOutput, err error) {
// Upload uploads raw file bytes to an S3 compatible storage system
// and returns the uploaded file's remote location.
func (a *Attachment) Upload(ctx context.Context) (location string, err error) {
config := server.Server.Config
awsConfig := &aws.Config{
Region: aws.String(config.AttachmentsBucketRegion),
Credentials: credentials.NewStaticCredentials(config.AttachmentsAccessKey, config.AttachmentsSecretAccessKey, ""),
var credentialsProvider aws.CredentialsProviderFunc = func(ctx context.Context) (aws.Credentials, error) {
return aws.Credentials{
AccessKeyID: config.AttachmentsAccessKey,
SecretAccessKey: config.AttachmentsSecretAccessKey,
}, nil
}

// if a custom endpoint was set, then most likely,
// we are in local development mode and should force
// path style instead of S3 virtual path styles.
if config.AWSEndpoint != "" {
awsConfig.S3ForcePathStyle = aws.Bool(true)
awsConfig.Endpoint = aws.String(config.AWSEndpoint)
awsConfig := &aws.Config{
Region: config.AttachmentsBucketRegion,
Credentials: credentialsProvider,
}

client := s3.NewFromConfig(*awsConfig, func(o *s3.Options) {
endpoint := config.AWSEndpoint
if endpoint != "" {
o.BaseEndpoint = aws.String(endpoint)
o.UsePathStyle = *aws.Bool(true)
}
})

// set mime type from extension
ext := filepath.Ext(a.Key)
contentType := "application/octet-stream"
if ext != "" {
contentType = mime.TypeByExtension(ext)
}

awsSession := session.Must(session.NewSession(awsConfig))
uploader := s3manager.NewUploader(awsSession)
output, err = uploader.Upload(&s3manager.UploadInput{
putObjectInput := &s3.PutObjectInput{
Bucket: aws.String(config.AttachmentsBucket),
Key: aws.String(a.Key),
Body: a.Reader,
Metadata: map[string]*string{
"original_file_name": aws.String(a.Name),
Metadata: map[string]string{
"original_file_name": a.Name,
},
ContentType: aws.String(contentType),
})
}

// for now, we construct the location manually
// implement a better solution later using
// EndpointResolverV2 with custom resolvers
// for non-AWS clouds like GCS
if config.AWSEndpoint != "" {
location = fmt.Sprintf("%s/%s/%s", config.AWSEndpoint, config.AttachmentsBucket, a.Key)
} else {
location = fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", config.AttachmentsBucket, config.AttachmentsBucketRegion, a.Key)
}

// ignore the putObjectOutput, don't need
// it for now
_, err = client.PutObject(ctx, putObjectInput)
if err != nil {
return
}

return
}

// PreSignURL generates a S3-compatible
// pre-signed URL for the attachment.
func (a *Attachment) PreSignURL() (err error) {
func (a *Attachment) PreSignURL(ctx context.Context) (err error) {
config := server.Server.Config
awsConfig := &aws.Config{
Region: aws.String(config.AttachmentsBucketRegion),
Credentials: credentials.NewStaticCredentials(config.AttachmentsAccessKey, config.AttachmentsSecretAccessKey, ""),
}

shouldProxy := true

if config.AttachmentOrigin != "" {
shouldProxy = false
}

// if a custom endpoint was set, then most likely,
// external object store is not native S3 like,
// hence should force path style instead of S3 virtual
// path styles.
if config.AWSEndpoint != "" {
awsConfig.S3ForcePathStyle = aws.Bool(true)
var credentialsProvider aws.CredentialsProviderFunc = func(ctx context.Context) (aws.Credentials, error) {
return aws.Credentials{
AccessKeyID: config.AttachmentsAccessKey,
SecretAccessKey: config.AttachmentsSecretAccessKey,
}, nil
}

if shouldProxy {
awsConfig.Endpoint = aws.String(config.AWSEndpoint)
} else {
awsConfig.Endpoint = aws.String(config.AttachmentOrigin)
}
awsConfig := &aws.Config{
Region: config.AttachmentsBucketRegion,
Credentials: credentialsProvider,
}

awsSession := session.Must(session.NewSession(awsConfig))
client := s3.NewFromConfig(*awsConfig, func(o *s3.Options) {
endpoint := config.AWSEndpoint
if endpoint != "" {
o.BaseEndpoint = aws.String(endpoint)
o.UsePathStyle = *aws.Bool(true)
}
})

presignClient := s3.NewPresignClient(client, func(o *s3.PresignOptions) {
o.Expires = 48 * time.Hour
})

svc := s3.New(awsSession)
req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
getObjectInput := &s3.GetObjectInput{
Bucket: aws.String(config.AttachmentsBucket),
Key: aws.String(a.Key),
})
}

urlStr, err := req.Presign(48 * time.Hour)
req, err := presignClient.PresignGetObject(ctx, getObjectInput)
if err != nil {
return err
return
}

urlStr := req.URL

if shouldProxy {
endpoint, err := url.JoinPath(config.APIOrigin, "attachments")
if err != nil {
Expand Down
14 changes: 11 additions & 3 deletions backend/api/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ go 1.22

require (
github.com/ClickHouse/clickhouse-go/v2 v2.25.0
github.com/aws/aws-sdk-go v1.50.19
github.com/aws/aws-sdk-go-v2 v1.32.7
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1
github.com/gin-contrib/cors v1.7.2
github.com/gin-gonic/gin v1.10.0
github.com/golang-jwt/jwt/v5 v5.2.0
Expand All @@ -29,6 +30,15 @@ require (
cloud.google.com/go/compute/metadata v0.5.0 // indirect
github.com/ClickHouse/ch-go v0.61.5 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect
github.com/aws/smithy-go v1.22.1 // indirect
github.com/bytedance/sonic v1.12.3 // indirect
github.com/bytedance/sonic/loader v0.2.0 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
Expand All @@ -53,7 +63,6 @@ require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
Expand Down Expand Up @@ -84,6 +93,5 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240709173604-40e1e62336c5 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
31 changes: 22 additions & 9 deletions backend/api/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,28 @@ github.com/ClickHouse/clickhouse-go/v2 v2.25.0 h1:rKscwqgQHzWBTZySZDcHKxgs0Ad+xF
github.com/ClickHouse/clickhouse-go/v2 v2.25.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/aws/aws-sdk-go v1.50.19 h1:YSIDKRSkh/TW0RPWoocdLqtC/T5W6IGBVhFs6P7Qcac=
github.com/aws/aws-sdk-go v1.50.19/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw=
github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c=
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE=
github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q=
github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU=
github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
Expand Down Expand Up @@ -115,10 +135,6 @@ github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
Expand Down Expand Up @@ -329,9 +345,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Expand Down
6 changes: 3 additions & 3 deletions backend/api/measure/app.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2933,7 +2933,7 @@ func GetCrashDetailCrashes(c *gin.Context) {
for i := range eventExceptions {
if len(eventExceptions[i].Attachments) > 0 {
for j := range eventExceptions[i].Attachments {
if err := eventExceptions[i].Attachments[j].PreSignURL(); err != nil {
if err := eventExceptions[i].Attachments[j].PreSignURL(ctx); err != nil {
msg := `failed to generate URLs for attachment`
fmt.Println(msg, err)
c.JSON(http.StatusInternalServerError, gin.H{
Expand Down Expand Up @@ -3859,7 +3859,7 @@ func GetANRDetailANRs(c *gin.Context) {
for i := range eventANRs {
if len(eventANRs[i].Attachments) > 0 {
for j := range eventANRs[i].Attachments {
if err := eventANRs[i].Attachments[j].PreSignURL(); err != nil {
if err := eventANRs[i].Attachments[j].PreSignURL(ctx); err != nil {
msg := `failed to generate URLs for attachment`
fmt.Println(msg, err)
c.JSON(http.StatusInternalServerError, gin.H{
Expand Down Expand Up @@ -4777,7 +4777,7 @@ func GetSession(c *gin.Context) {
continue
}
for j := range session.Events[i].Attachments {
if err := session.Events[i].Attachments[j].PreSignURL(); err != nil {
if err := session.Events[i].Attachments[j].PreSignURL(ctx); err != nil {
msg := `failed to generate URLs for attachment`
fmt.Println(msg, err)
c.JSON(http.StatusInternalServerError, gin.H{
Expand Down
8 changes: 4 additions & 4 deletions backend/api/measure/event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func (s status) String() string {
}

// uploadAttachments prepares and uploads each attachment.
func (e *eventreq) uploadAttachments() error {
func (e *eventreq) uploadAttachments(ctx context.Context) error {
for id, attachment := range e.attachments {
ext := filepath.Ext(attachment.header.Filename)
key := attachment.id.String() + ext
Expand All @@ -107,14 +107,14 @@ func (e *eventreq) uploadAttachments() error {

eventAttachment.Reader = file

output, err := eventAttachment.Upload()
location, err := eventAttachment.Upload(ctx)
if err != nil {
return err
}

attachment.uploaded = true
attachment.key = key
attachment.location = output.Location
attachment.location = location
}

return nil
Expand Down Expand Up @@ -2157,7 +2157,7 @@ func PutEvents(c *gin.Context) {

defer uploadAttachmentSpan.End()

if err := eventReq.uploadAttachments(); err != nil {
if err := eventReq.uploadAttachments(ctx); err != nil {
msg := `failed to upload attachments`
fmt.Println(msg, err)
c.JSON(http.StatusInternalServerError, gin.H{
Expand Down
Loading

0 comments on commit c06fd96

Please sign in to comment.