From a095087fa40d66bd0c7a81521c2ecc9cfb59dd8f Mon Sep 17 00:00:00 2001 From: detj Date: Wed, 1 Jan 2025 08:09:50 +0530 Subject: [PATCH 1/3] refactor(backend): use aws-sdk-go-v2 for attachments - refactor attachment upload to use `aws-sdk-go-v2` instead of `aws-sdk-go` fixes #1669 Signed-off-by: detj --- backend/api/event/attachment.go | 117 +++++++++++++++++++------------- backend/api/go.mod | 11 +++ backend/api/go.sum | 22 ++++++ backend/api/measure/app.go | 6 +- backend/api/measure/event.go | 8 +-- 5 files changed, 111 insertions(+), 53 deletions(-) diff --git a/backend/api/event/attachment.go b/backend/api/event/attachment.go index a6040d419..b547b286f 100644 --- a/backend/api/event/attachment.go +++ b/backend/api/event/attachment.go @@ -1,7 +1,9 @@ package event import ( + "context" "errors" + "fmt" "io" "mime" "net/url" @@ -11,11 +13,8 @@ import ( "backend/api/server" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/google/uuid" ) @@ -48,22 +47,30 @@ func (a Attachment) Validate() error { return nil } -// Upload uploads raw file bytes to an S3 compatible storage system. -func (a *Attachment) Upload() (output *s3manager.UploadOutput, err error) { +// Upload uploads raw file bytes to an S3 compatible storage system +// and returns the uploaded file's remote location. +func (a *Attachment) Upload(ctx context.Context) (location string, err error) { config := server.Server.Config - awsConfig := &aws.Config{ - Region: aws.String(config.AttachmentsBucketRegion), - Credentials: credentials.NewStaticCredentials(config.AttachmentsAccessKey, config.AttachmentsSecretAccessKey, ""), + var credentialsProvider aws.CredentialsProviderFunc = func(ctx context.Context) (aws.Credentials, error) { + return aws.Credentials{ + AccessKeyID: config.AttachmentsAccessKey, + SecretAccessKey: config.AttachmentsSecretAccessKey, + }, nil } - // if a custom endpoint was set, then most likely, - // we are in local development mode and should force - // path style instead of S3 virtual path styles. - if config.AWSEndpoint != "" { - awsConfig.S3ForcePathStyle = aws.Bool(true) - awsConfig.Endpoint = aws.String(config.AWSEndpoint) + awsConfig := &aws.Config{ + Region: config.AttachmentsBucketRegion, + Credentials: credentialsProvider, } + client := s3.NewFromConfig(*awsConfig, func(o *s3.Options) { + endpoint := config.AWSEndpoint + if endpoint != "" { + o.BaseEndpoint = aws.String(endpoint) + o.UsePathStyle = *aws.Bool(true) + } + }) + // set mime type from extension ext := filepath.Ext(a.Key) contentType := "application/octet-stream" @@ -71,63 +78,81 @@ func (a *Attachment) Upload() (output *s3manager.UploadOutput, err error) { contentType = mime.TypeByExtension(ext) } - awsSession := session.Must(session.NewSession(awsConfig)) - uploader := s3manager.NewUploader(awsSession) - output, err = uploader.Upload(&s3manager.UploadInput{ + putObjectInput := &s3.PutObjectInput{ Bucket: aws.String(config.AttachmentsBucket), Key: aws.String(a.Key), Body: a.Reader, - Metadata: map[string]*string{ - "original_file_name": aws.String(a.Name), + Metadata: map[string]string{ + "original_file_name": a.Name, }, ContentType: aws.String(contentType), - }) + } + + // for now, we construct the location manually + // implement a better solution later using + // EndpointResolverV2 with custom resolvers + // for non-AWS clouds like GCS + if config.AWSEndpoint != "" { + location = fmt.Sprintf("%s/%s/%s", config.AWSEndpoint, config.AttachmentsBucket, a.Key) + } else { + location = fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", config.AttachmentsBucket, config.AttachmentsBucketRegion, a.Key) + } + + // ignore the putObjectOutput, don't need + // it for now + _, err = client.PutObject(ctx, putObjectInput) + if err != nil { + return + } return } // PreSignURL generates a S3-compatible // pre-signed URL for the attachment. -func (a *Attachment) PreSignURL() (err error) { +func (a *Attachment) PreSignURL(ctx context.Context) (err error) { config := server.Server.Config - awsConfig := &aws.Config{ - Region: aws.String(config.AttachmentsBucketRegion), - Credentials: credentials.NewStaticCredentials(config.AttachmentsAccessKey, config.AttachmentsSecretAccessKey, ""), - } - shouldProxy := true - if config.AttachmentOrigin != "" { shouldProxy = false } - // if a custom endpoint was set, then most likely, - // external object store is not native S3 like, - // hence should force path style instead of S3 virtual - // path styles. - if config.AWSEndpoint != "" { - awsConfig.S3ForcePathStyle = aws.Bool(true) + var credentialsProvider aws.CredentialsProviderFunc = func(ctx context.Context) (aws.Credentials, error) { + return aws.Credentials{ + AccessKeyID: config.AttachmentsAccessKey, + SecretAccessKey: config.AttachmentsSecretAccessKey, + }, nil + } - if shouldProxy { - awsConfig.Endpoint = aws.String(config.AWSEndpoint) - } else { - awsConfig.Endpoint = aws.String(config.AttachmentOrigin) - } + awsConfig := &aws.Config{ + Region: config.AttachmentsBucketRegion, + Credentials: credentialsProvider, } - awsSession := session.Must(session.NewSession(awsConfig)) + client := s3.NewFromConfig(*awsConfig, func(o *s3.Options) { + endpoint := config.AWSEndpoint + if endpoint != "" { + o.BaseEndpoint = aws.String(endpoint) + o.UsePathStyle = *aws.Bool(true) + } + }) + + presignClient := s3.NewPresignClient(client, func(o *s3.PresignOptions) { + o.Expires = 48 * time.Hour + }) - svc := s3.New(awsSession) - req, _ := svc.GetObjectRequest(&s3.GetObjectInput{ + getObjectInput := &s3.GetObjectInput{ Bucket: aws.String(config.AttachmentsBucket), Key: aws.String(a.Key), - }) + } - urlStr, err := req.Presign(48 * time.Hour) + req, err := presignClient.PresignGetObject(ctx, getObjectInput) if err != nil { - return err + return } + urlStr := req.URL + if shouldProxy { endpoint, err := url.JoinPath(config.APIOrigin, "attachments") if err != nil { diff --git a/backend/api/go.mod b/backend/api/go.mod index 1eb615f97..15f90e8b2 100644 --- a/backend/api/go.mod +++ b/backend/api/go.mod @@ -5,6 +5,8 @@ go 1.22 require ( github.com/ClickHouse/clickhouse-go/v2 v2.25.0 github.com/aws/aws-sdk-go v1.50.19 + github.com/aws/aws-sdk-go-v2 v1.32.7 + github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt/v5 v5.2.0 @@ -29,6 +31,15 @@ require ( cloud.google.com/go/compute/metadata v0.5.0 // indirect github.com/ClickHouse/ch-go v0.61.5 // indirect github.com/andybalholm/brotli v1.1.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect + github.com/aws/smithy-go v1.22.1 // indirect github.com/bytedance/sonic v1.12.3 // indirect github.com/bytedance/sonic/loader v0.2.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect diff --git a/backend/api/go.sum b/backend/api/go.sum index 322358b16..fa9188b01 100644 --- a/backend/api/go.sum +++ b/backend/api/go.sum @@ -14,6 +14,28 @@ github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/aws/aws-sdk-go v1.50.19 h1:YSIDKRSkh/TW0RPWoocdLqtC/T5W6IGBVhFs6P7Qcac= github.com/aws/aws-sdk-go v1.50.19/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw= +github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= diff --git a/backend/api/measure/app.go b/backend/api/measure/app.go index 17ba78441..24b72eb73 100644 --- a/backend/api/measure/app.go +++ b/backend/api/measure/app.go @@ -2933,7 +2933,7 @@ func GetCrashDetailCrashes(c *gin.Context) { for i := range eventExceptions { if len(eventExceptions[i].Attachments) > 0 { for j := range eventExceptions[i].Attachments { - if err := eventExceptions[i].Attachments[j].PreSignURL(); err != nil { + if err := eventExceptions[i].Attachments[j].PreSignURL(ctx); err != nil { msg := `failed to generate URLs for attachment` fmt.Println(msg, err) c.JSON(http.StatusInternalServerError, gin.H{ @@ -3859,7 +3859,7 @@ func GetANRDetailANRs(c *gin.Context) { for i := range eventANRs { if len(eventANRs[i].Attachments) > 0 { for j := range eventANRs[i].Attachments { - if err := eventANRs[i].Attachments[j].PreSignURL(); err != nil { + if err := eventANRs[i].Attachments[j].PreSignURL(ctx); err != nil { msg := `failed to generate URLs for attachment` fmt.Println(msg, err) c.JSON(http.StatusInternalServerError, gin.H{ @@ -4777,7 +4777,7 @@ func GetSession(c *gin.Context) { continue } for j := range session.Events[i].Attachments { - if err := session.Events[i].Attachments[j].PreSignURL(); err != nil { + if err := session.Events[i].Attachments[j].PreSignURL(ctx); err != nil { msg := `failed to generate URLs for attachment` fmt.Println(msg, err) c.JSON(http.StatusInternalServerError, gin.H{ diff --git a/backend/api/measure/event.go b/backend/api/measure/event.go index 996eabaf8..43267acfd 100644 --- a/backend/api/measure/event.go +++ b/backend/api/measure/event.go @@ -89,7 +89,7 @@ func (s status) String() string { } // uploadAttachments prepares and uploads each attachment. -func (e *eventreq) uploadAttachments() error { +func (e *eventreq) uploadAttachments(ctx context.Context) error { for id, attachment := range e.attachments { ext := filepath.Ext(attachment.header.Filename) key := attachment.id.String() + ext @@ -107,14 +107,14 @@ func (e *eventreq) uploadAttachments() error { eventAttachment.Reader = file - output, err := eventAttachment.Upload() + location, err := eventAttachment.Upload(ctx) if err != nil { return err } attachment.uploaded = true attachment.key = key - attachment.location = output.Location + attachment.location = location } return nil @@ -2157,7 +2157,7 @@ func PutEvents(c *gin.Context) { defer uploadAttachmentSpan.End() - if err := eventReq.uploadAttachments(); err != nil { + if err := eventReq.uploadAttachments(ctx); err != nil { msg := `failed to upload attachments` fmt.Println(msg, err) c.JSON(http.StatusInternalServerError, gin.H{ From f059a3e85d23e143ecab6c00069ddea198c7974a Mon Sep 17 00:00:00 2001 From: detj Date: Wed, 1 Jan 2025 12:43:19 +0530 Subject: [PATCH 2/3] refactor(backend): update build mapping upload Signed-off-by: detj --- backend/api/go.mod | 3 - backend/api/go.sum | 9 --- backend/api/measure/mapping.go | 129 ++++++++++++++++++++++++--------- go.work.sum | 9 ++- 4 files changed, 101 insertions(+), 49 deletions(-) diff --git a/backend/api/go.mod b/backend/api/go.mod index 15f90e8b2..36195895b 100644 --- a/backend/api/go.mod +++ b/backend/api/go.mod @@ -4,7 +4,6 @@ go 1.22 require ( github.com/ClickHouse/clickhouse-go/v2 v2.25.0 - github.com/aws/aws-sdk-go v1.50.19 github.com/aws/aws-sdk-go-v2 v1.32.7 github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 github.com/gin-contrib/cors v1.7.2 @@ -64,7 +63,6 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect @@ -95,6 +93,5 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240709173604-40e1e62336c5 // indirect google.golang.org/protobuf v1.34.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/backend/api/go.sum b/backend/api/go.sum index fa9188b01..a8b5bd495 100644 --- a/backend/api/go.sum +++ b/backend/api/go.sum @@ -12,8 +12,6 @@ github.com/ClickHouse/clickhouse-go/v2 v2.25.0 h1:rKscwqgQHzWBTZySZDcHKxgs0Ad+xF github.com/ClickHouse/clickhouse-go/v2 v2.25.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/aws/aws-sdk-go v1.50.19 h1:YSIDKRSkh/TW0RPWoocdLqtC/T5W6IGBVhFs6P7Qcac= -github.com/aws/aws-sdk-go v1.50.19/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw= github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= @@ -137,10 +135,6 @@ github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -351,9 +345,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/backend/api/measure/mapping.go b/backend/api/measure/mapping.go index a9a9a030f..ac8839792 100644 --- a/backend/api/measure/mapping.go +++ b/backend/api/measure/mapping.go @@ -13,10 +13,9 @@ import ( "backend/api/cipher" "backend/api/server" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/gin-gonic/gin" "github.com/gin-gonic/gin/binding" "github.com/google/uuid" @@ -170,39 +169,109 @@ func (bm *BuildMapping) checksum() error { return nil } -func (bm *BuildMapping) upload() (*s3manager.UploadOutput, error) { +// func (bm *BuildMapping) upload() (*s3manager.UploadOutput, error) { +// file, err := bm.File.Open() +// if err != nil { +// return nil, err +// } + +// config := server.Server.Config +// awsConfig := &aws.Config{ +// Region: aws.String(config.SymbolsBucketRegion), +// Credentials: credentials.NewStaticCredentials(config.SymbolsAccessKey, config.SymbolsSecretAccessKey, ""), +// } + +// // if a custom endpoint was set, then most likely, +// // we are in local development mode and should force +// // path style instead of S3 virtual path styles. +// if config.AWSEndpoint != "" { +// awsConfig.S3ForcePathStyle = aws.Bool(true) +// awsConfig.Endpoint = aws.String(config.AWSEndpoint) +// } + +// if bm.Key == "" { +// bm.Key = bm.GetKey() +// } + +// metadata := map[string]*string{ +// "original_file_name": aws.String(bm.File.Filename), +// "app_id": aws.String(bm.AppID.String()), +// "version_name": aws.String(bm.VersionName), +// "version_code": aws.String(bm.VersionCode), +// "mapping_type": aws.String(bm.MappingType), +// } + +// return uploadToStorage(awsConfig, config.SymbolsBucket, bm.Key, file, metadata) +// } + +func (bm *BuildMapping) uploadTwo(ctx context.Context) (location *string, err error) { file, err := bm.File.Open() if err != nil { return nil, err } config := server.Server.Config - awsConfig := &aws.Config{ - Region: aws.String(config.SymbolsBucketRegion), - Credentials: credentials.NewStaticCredentials(config.SymbolsAccessKey, config.SymbolsSecretAccessKey, ""), + var credentialsProvider aws.CredentialsProviderFunc = func(ctx context.Context) (aws.Credentials, error) { + return aws.Credentials{ + AccessKeyID: config.SymbolsAccessKey, + SecretAccessKey: config.SymbolsSecretAccessKey, + }, nil } - // if a custom endpoint was set, then most likely, - // we are in local development mode and should force - // path style instead of S3 virtual path styles. - if config.AWSEndpoint != "" { - awsConfig.S3ForcePathStyle = aws.Bool(true) - awsConfig.Endpoint = aws.String(config.AWSEndpoint) + awsConfig := &aws.Config{ + Region: config.SymbolsBucketRegion, + Credentials: credentialsProvider, } + client := s3.NewFromConfig(*awsConfig, func(o *s3.Options) { + endpoint := config.AWSEndpoint + if endpoint != "" { + o.BaseEndpoint = aws.String(endpoint) + o.UsePathStyle = *aws.Bool(true) + } + }) + if bm.Key == "" { bm.Key = bm.GetKey() } - metadata := map[string]*string{ - "original_file_name": aws.String(bm.File.Filename), - "app_id": aws.String(bm.AppID.String()), - "version_name": aws.String(bm.VersionName), - "version_code": aws.String(bm.VersionCode), - "mapping_type": aws.String(bm.MappingType), + metadata := map[string]string{ + "original_file_name": bm.File.Filename, + "app_id": bm.AppID.String(), + "version_name": bm.VersionName, + "version_code": bm.VersionCode, + "mapping_type": bm.MappingType, } - return uploadToStorage(awsConfig, config.SymbolsBucket, bm.Key, file, metadata) + putObjectInput := &s3.PutObjectInput{ + Bucket: aws.String(config.SymbolsBucket), + Key: aws.String(bm.Key), + Body: file, + Metadata: metadata, + } + + loc := "" + + // for now, we construct the location manually + // implement a better solution later using + // EndpointResolverV2 with custom resolvers + // for non-AWS clouds like GCS + if config.AWSEndpoint != "" { + loc = fmt.Sprintf("%s/%s/%s", config.AWSEndpoint, config.SymbolsBucket, bm.Key) + } else { + loc = fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", config.SymbolsBucket, config.SymbolsBucketRegion, bm.Key) + } + + location = &loc + + // ignore the putObjectOutput, don't need + // it for now + _, err = client.PutObject(ctx, putObjectInput) + if err != nil { + return + } + + return } type BuildSize struct { @@ -317,8 +386,8 @@ func PutBuild(c *gin.Context) { // start span to trace mapping file upload mappingFileUploadTracer := otel.Tracer("mapping-file-upload-tracer") _, mappingFileUploadSpan := mappingFileUploadTracer.Start(ctx, "mapping-file-upload") - result, err := bm.upload() - if err != nil { + location, err := bm.uploadTwo(ctx) + if err != nil || location == nil { fmt.Printf("failed to upload mapping file, key: %s with error, %v\n", bm.Key, err) c.JSON(http.StatusInternalServerError, gin.H{ "error": fmt.Sprintf(`failed to upload mapping file: "%s"`, bm.File.Filename), @@ -328,7 +397,7 @@ func PutBuild(c *gin.Context) { } mappingFileUploadSpan.End() - bm.Location = result.Location + bm.Location = *location } if existingId != nil { @@ -372,15 +441,3 @@ func PutBuild(c *gin.Context) { "ok": `uploaded build info`, }) } - -func uploadToStorage(awsConfig *aws.Config, bucket, key string, file io.Reader, metadata map[string]*string) (*s3manager.UploadOutput, error) { - awsSession := session.Must(session.NewSession(awsConfig)) - uploader := s3manager.NewUploader(awsSession) - - return uploader.Upload(&s3manager.UploadInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - Body: file, - Metadata: metadata, - }) -} diff --git a/go.work.sum b/go.work.sum index 330179ee8..a351c2690 100644 --- a/go.work.sum +++ b/go.work.sum @@ -22,6 +22,7 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/akavel/rsrc v0.10.2/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aws/aws-sdk-go v1.50.19/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= @@ -63,7 +64,6 @@ github.com/containernetworking/plugins v1.2.0/go.mod h1:/VjX4uHecW5vVimFa1wkG4s+ github.com/containers/ocicrypt v1.1.6/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= @@ -129,6 +129,10 @@ github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47 github.com/intel/goresctrl v0.3.0/go.mod h1:fdz3mD85cmP9sHD8JUlrNWAxvwM86CrbmVXltEKd7zk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/josephspurrier/goversioninfo v1.4.0/go.mod h1:JWzv5rKQr+MmW+LvM412ToT/IkYDZjaclF2pKDss8IY= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -332,6 +336,9 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= k8s.io/api v0.26.2/go.mod h1:1kjMQsFE+QHPfskEcVNgL3+Hp88B80uj0QtSOlj8itU= k8s.io/apimachinery v0.26.2/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= From 0126a6cb21288ab33a80186811580fb256c6b5a4 Mon Sep 17 00:00:00 2001 From: detj Date: Wed, 1 Jan 2025 12:45:46 +0530 Subject: [PATCH 3/3] refactor(backend): remove unused code Signed-off-by: detj --- backend/api/measure/mapping.go | 39 ++-------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/backend/api/measure/mapping.go b/backend/api/measure/mapping.go index ac8839792..2d389ef33 100644 --- a/backend/api/measure/mapping.go +++ b/backend/api/measure/mapping.go @@ -169,42 +169,7 @@ func (bm *BuildMapping) checksum() error { return nil } -// func (bm *BuildMapping) upload() (*s3manager.UploadOutput, error) { -// file, err := bm.File.Open() -// if err != nil { -// return nil, err -// } - -// config := server.Server.Config -// awsConfig := &aws.Config{ -// Region: aws.String(config.SymbolsBucketRegion), -// Credentials: credentials.NewStaticCredentials(config.SymbolsAccessKey, config.SymbolsSecretAccessKey, ""), -// } - -// // if a custom endpoint was set, then most likely, -// // we are in local development mode and should force -// // path style instead of S3 virtual path styles. -// if config.AWSEndpoint != "" { -// awsConfig.S3ForcePathStyle = aws.Bool(true) -// awsConfig.Endpoint = aws.String(config.AWSEndpoint) -// } - -// if bm.Key == "" { -// bm.Key = bm.GetKey() -// } - -// metadata := map[string]*string{ -// "original_file_name": aws.String(bm.File.Filename), -// "app_id": aws.String(bm.AppID.String()), -// "version_name": aws.String(bm.VersionName), -// "version_code": aws.String(bm.VersionCode), -// "mapping_type": aws.String(bm.MappingType), -// } - -// return uploadToStorage(awsConfig, config.SymbolsBucket, bm.Key, file, metadata) -// } - -func (bm *BuildMapping) uploadTwo(ctx context.Context) (location *string, err error) { +func (bm *BuildMapping) upload(ctx context.Context) (location *string, err error) { file, err := bm.File.Open() if err != nil { return nil, err @@ -386,7 +351,7 @@ func PutBuild(c *gin.Context) { // start span to trace mapping file upload mappingFileUploadTracer := otel.Tracer("mapping-file-upload-tracer") _, mappingFileUploadSpan := mappingFileUploadTracer.Start(ctx, "mapping-file-upload") - location, err := bm.uploadTwo(ctx) + location, err := bm.upload(ctx) if err != nil || location == nil { fmt.Printf("failed to upload mapping file, key: %s with error, %v\n", bm.Key, err) c.JSON(http.StatusInternalServerError, gin.H{