refactor(backend): use aws-sdk-go-v2 for object uploads

backend/api/event/attachment.go

@@ -1,7 +1,9 @@
 package event
 
 import (
+	"context"
 	"errors"
+	"fmt"
 	"io"
 	"mime"
 	"net/url"
@@ -11,11 +13,8 @@ import (
 
 	"backend/api/server"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/google/uuid"
 )
 
@@ -48,86 +47,112 @@ func (a Attachment) Validate() error {
 	return nil
 }
 
-// Upload uploads raw file bytes to an S3 compatible storage system.
-func (a *Attachment) Upload() (output *s3manager.UploadOutput, err error) {
+// Upload uploads raw file bytes to an S3 compatible storage system
+// and returns the uploaded file's remote location.
+func (a *Attachment) Upload(ctx context.Context) (location string, err error) {
 	config := server.Server.Config
-	awsConfig := &aws.Config{
-		Region:      aws.String(config.AttachmentsBucketRegion),
-		Credentials: credentials.NewStaticCredentials(config.AttachmentsAccessKey, config.AttachmentsSecretAccessKey, ""),
+	var credentialsProvider aws.CredentialsProviderFunc = func(ctx context.Context) (aws.Credentials, error) {
+		return aws.Credentials{
+			AccessKeyID:     config.AttachmentsAccessKey,
+			SecretAccessKey: config.AttachmentsSecretAccessKey,
+		}, nil
 	}
 
-	// if a custom endpoint was set, then most likely,
-	// we are in local development mode and should force
-	// path style instead of S3 virtual path styles.
-	if config.AWSEndpoint != "" {
-		awsConfig.S3ForcePathStyle = aws.Bool(true)
-		awsConfig.Endpoint = aws.String(config.AWSEndpoint)
+	awsConfig := &aws.Config{
+		Region:      config.AttachmentsBucketRegion,
+		Credentials: credentialsProvider,
 	}
 
+	client := s3.NewFromConfig(*awsConfig, func(o *s3.Options) {
+		endpoint := config.AWSEndpoint
+		if endpoint != "" {
+			o.BaseEndpoint = aws.String(endpoint)
+			o.UsePathStyle = *aws.Bool(true)
+		}
+	})
+
 	// set mime type from extension
 	ext := filepath.Ext(a.Key)
 	contentType := "application/octet-stream"
 	if ext != "" {
 		contentType = mime.TypeByExtension(ext)
 	}
 
-	awsSession := session.Must(session.NewSession(awsConfig))
-	uploader := s3manager.NewUploader(awsSession)
-	output, err = uploader.Upload(&s3manager.UploadInput{
+	putObjectInput := &s3.PutObjectInput{
 		Bucket: aws.String(config.AttachmentsBucket),
 		Key:    aws.String(a.Key),
 		Body:   a.Reader,
-		Metadata: map[string]*string{
-			"original_file_name": aws.String(a.Name),
+		Metadata: map[string]string{
+			"original_file_name": a.Name,
 		},
 		ContentType: aws.String(contentType),
-	})
+	}
+
+	// for now, we construct the location manually
+	// implement a better solution later using
+	// EndpointResolverV2 with custom resolvers
+	// for non-AWS clouds like GCS
+	if config.AWSEndpoint != "" {
+		location = fmt.Sprintf("%s/%s/%s", config.AWSEndpoint, config.AttachmentsBucket, a.Key)
+	} else {
+		location = fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", config.AttachmentsBucket, config.AttachmentsBucketRegion, a.Key)
+	}
+
+	// ignore the putObjectOutput, don't need
+	// it for now
+	_, err = client.PutObject(ctx, putObjectInput)
+	if err != nil {
+		return
+	}
 
 	return
 }
 
 // PreSignURL generates a S3-compatible
 // pre-signed URL for the attachment.
-func (a *Attachment) PreSignURL() (err error) {
+func (a *Attachment) PreSignURL(ctx context.Context) (err error) {
 	config := server.Server.Config
-	awsConfig := &aws.Config{
-		Region:      aws.String(config.AttachmentsBucketRegion),
-		Credentials: credentials.NewStaticCredentials(config.AttachmentsAccessKey, config.AttachmentsSecretAccessKey, ""),
-	}
-
 	shouldProxy := true
-
 	if config.AttachmentOrigin != "" {
 		shouldProxy = false
 	}
 
-	// if a custom endpoint was set, then most likely,
-	// external object store is not native S3 like,
-	// hence should force path style instead of S3 virtual
-	// path styles.
-	if config.AWSEndpoint != "" {
-		awsConfig.S3ForcePathStyle = aws.Bool(true)
+	var credentialsProvider aws.CredentialsProviderFunc = func(ctx context.Context) (aws.Credentials, error) {
+		return aws.Credentials{
+			AccessKeyID:     config.AttachmentsAccessKey,
+			SecretAccessKey: config.AttachmentsSecretAccessKey,
+		}, nil
+	}
 
-		if shouldProxy {
-			awsConfig.Endpoint = aws.String(config.AWSEndpoint)
-		} else {
-			awsConfig.Endpoint = aws.String(config.AttachmentOrigin)
-		}
+	awsConfig := &aws.Config{
+		Region:      config.AttachmentsBucketRegion,
+		Credentials: credentialsProvider,
 	}
 
-	awsSession := session.Must(session.NewSession(awsConfig))
+	client := s3.NewFromConfig(*awsConfig, func(o *s3.Options) {
+		endpoint := config.AWSEndpoint
+		if endpoint != "" {
+			o.BaseEndpoint = aws.String(endpoint)
+			o.UsePathStyle = *aws.Bool(true)
+		}
+	})
+
+	presignClient := s3.NewPresignClient(client, func(o *s3.PresignOptions) {
+		o.Expires = 48 * time.Hour
+	})
 
-	svc := s3.New(awsSession)
-	req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
+	getObjectInput := &s3.GetObjectInput{
 		Bucket: aws.String(config.AttachmentsBucket),
 		Key:    aws.String(a.Key),
-	})
+	}
 
-	urlStr, err := req.Presign(48 * time.Hour)
+	req, err := presignClient.PresignGetObject(ctx, getObjectInput)
 	if err != nil {
-		return err
+		return
 	}
 
+	urlStr := req.URL
+
 	if shouldProxy {
 		endpoint, err := url.JoinPath(config.APIOrigin, "attachments")
 		if err != nil {

backend/api/go.mod

@@ -4,7 +4,8 @@ go 1.22
 
 require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.25.0
-	github.com/aws/aws-sdk-go v1.50.19
+	github.com/aws/aws-sdk-go-v2 v1.32.7
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1
 	github.com/gin-contrib/cors v1.7.2
 	github.com/gin-gonic/gin v1.10.0
 	github.com/golang-jwt/jwt/v5 v5.2.0
@@ -29,6 +30,15 @@ require (
 	cloud.google.com/go/compute/metadata v0.5.0 // indirect
 	github.com/ClickHouse/ch-go v0.61.5 // indirect
 	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect
+	github.com/aws/smithy-go v1.22.1 // indirect
 	github.com/bytedance/sonic v1.12.3 // indirect
 	github.com/bytedance/sonic/loader v0.2.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -53,7 +63,6 @@ require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
@@ -84,6 +93,5 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240709173604-40e1e62336c5 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

backend/api/go.sum

@@ -12,8 +12,28 @@ github.com/ClickHouse/clickhouse-go/v2 v2.25.0 h1:rKscwqgQHzWBTZySZDcHKxgs0Ad+xF
 github.com/ClickHouse/clickhouse-go/v2 v2.25.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/aws/aws-sdk-go v1.50.19 h1:YSIDKRSkh/TW0RPWoocdLqtC/T5W6IGBVhFs6P7Qcac=
-github.com/aws/aws-sdk-go v1.50.19/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw=
+github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q=
+github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
+github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU=
 github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
@@ -115,10 +135,6 @@ github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
 github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
-github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -329,9 +345,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

backend/api/measure/app.go

@@ -2933,7 +2933,7 @@ func GetCrashDetailCrashes(c *gin.Context) {
 	for i := range eventExceptions {
 		if len(eventExceptions[i].Attachments) > 0 {
 			for j := range eventExceptions[i].Attachments {
-				if err := eventExceptions[i].Attachments[j].PreSignURL(); err != nil {
+				if err := eventExceptions[i].Attachments[j].PreSignURL(ctx); err != nil {
 					msg := `failed to generate URLs for attachment`
 					fmt.Println(msg, err)
 					c.JSON(http.StatusInternalServerError, gin.H{
@@ -3859,7 +3859,7 @@ func GetANRDetailANRs(c *gin.Context) {
 	for i := range eventANRs {
 		if len(eventANRs[i].Attachments) > 0 {
 			for j := range eventANRs[i].Attachments {
-				if err := eventANRs[i].Attachments[j].PreSignURL(); err != nil {
+				if err := eventANRs[i].Attachments[j].PreSignURL(ctx); err != nil {
 					msg := `failed to generate URLs for attachment`
 					fmt.Println(msg, err)
 					c.JSON(http.StatusInternalServerError, gin.H{
@@ -4777,7 +4777,7 @@ func GetSession(c *gin.Context) {
 			continue
 		}
 		for j := range session.Events[i].Attachments {
-			if err := session.Events[i].Attachments[j].PreSignURL(); err != nil {
+			if err := session.Events[i].Attachments[j].PreSignURL(ctx); err != nil {
 				msg := `failed to generate URLs for attachment`
 				fmt.Println(msg, err)
 				c.JSON(http.StatusInternalServerError, gin.H{

backend/api/measure/event.go

@@ -89,7 +89,7 @@ func (s status) String() string {
 }
 
 // uploadAttachments prepares and uploads each attachment.
-func (e *eventreq) uploadAttachments() error {
+func (e *eventreq) uploadAttachments(ctx context.Context) error {
 	for id, attachment := range e.attachments {
 		ext := filepath.Ext(attachment.header.Filename)
 		key := attachment.id.String() + ext
@@ -107,14 +107,14 @@ func (e *eventreq) uploadAttachments() error {
 
 		eventAttachment.Reader = file
 
-		output, err := eventAttachment.Upload()
+		location, err := eventAttachment.Upload(ctx)
 		if err != nil {
 			return err
 		}
 
 		attachment.uploaded = true
 		attachment.key = key
-		attachment.location = output.Location
+		attachment.location = location
 	}
 
 	return nil
@@ -2157,7 +2157,7 @@ func PutEvents(c *gin.Context) {
 
 		defer uploadAttachmentSpan.End()
 
-		if err := eventReq.uploadAttachments(); err != nil {
+		if err := eventReq.uploadAttachments(ctx); err != nil {
 			msg := `failed to upload attachments`
 			fmt.Println(msg, err)
 			c.JSON(http.StatusInternalServerError, gin.H{