feat(minio): pass UserUID as header and delegate logging to MinIO (#36)

Because

- Audit logs need to be emitted by MinIO in order to include actions
from any client. Still, we need to identify who performed an action.

This commit

- Remove (most of) the logging and pass the user UID as metadata to the
MinIO actions.

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/gojuno/minimock/v3 v3.4.0
 	github.com/google/uuid v1.6.0
 	github.com/iancoleman/strcase v0.2.0
-	github.com/minio/minio-go/v7 v7.0.76
+	github.com/minio/minio-go/v7 v7.0.85
 	github.com/stretchr/testify v1.9.0
 	go.temporal.io/sdk v1.13.1
 	go.uber.org/zap v1.21.0
@@ -22,16 +22,16 @@ require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/goccy/go-json v0.10.4 // indirect
 	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/gogo/status v1.1.0 // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect

go.sum

@@ -39,8 +39,8 @@ github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
-github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM=
+github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
 github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@@ -93,11 +93,11 @@ github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHL
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
-github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
+github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -109,8 +109,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.76 h1:9nxHH2XDai61cT/EFhyIw/wW4vJfpPNvl7lSFpRt+Ng=
-github.com/minio/minio-go/v7 v7.0.76/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg=
+github.com/minio/minio-go/v7 v7.0.85 h1:9psTLS/NTvC3MWoyjhjXpwcKoNbkongaCSF3PNpSuXo=
+github.com/minio/minio-go/v7 v7.0.85/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
@@ -211,7 +211,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

minio/minio.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/http"
 	"sync"
 	"time"
 
@@ -19,6 +20,10 @@ import (
 	miniogo "github.com/minio/minio-go/v7"
 )
 
+// MinIOHeaderUserUID is sent as metadata as a MinIO header to indicate the
+// user that triggered the MinIO action.
+const MinIOHeaderUserUID = "x-amz-meta-instill-user-uid"
+
 // MinioI defines the methods to interact with MinIO.
 type MinioI interface {
 	WithLogger(*zap.Logger) MinioI
@@ -171,23 +176,29 @@ func (m *minio) UploadFile(ctx context.Context, param *UploadFileParam) (url str
 }
 
 func (m *minio) UploadFileBytes(ctx context.Context, param *UploadFileBytesParam) (url string, objectInfo *miniogo.ObjectInfo, err error) {
-	m.logAction(param.UserUID, "uploadFile", param.FilePath)
 	reader := bytes.NewReader(param.FileBytes)
 
 	// Create the file path with folder structure
-	_, err = m.client.PutObject(ctx, m.bucket, param.FilePath, reader, int64(len(param.FileBytes)), miniogo.PutObjectOptions{
-		ContentType: param.FileMimeType,
-		UserTags:    map[string]string{ExpiryTag: param.ExpiryRuleTag},
-	})
+	_, err = m.client.PutObject(ctx,
+		m.bucket,
+		param.FilePath,
+		reader,
+		int64(len(param.FileBytes)),
+		miniogo.PutObjectOptions{
+			ContentType:  param.FileMimeType,
+			UserTags:     map[string]string{ExpiryTag: param.ExpiryRuleTag},
+			UserMetadata: map[string]string{MinIOHeaderUserUID: param.UserUID.String()},
+		},
+	)
 	if err != nil {
-		return "", nil, err
+		return "", nil, fmt.Errorf("putting object in MinIO: %w", err)
 	}
 
 	// Get the object stat (metadata)
-	m.logAction(param.UserUID, "statObject", param.FilePath)
-	stat, err := m.client.StatObject(ctx, m.bucket, param.FilePath, miniogo.StatObjectOptions{})
+	statOpts := miniogo.StatObjectOptions(m.getObjectOptions(param.UserUID))
+	stat, err := m.client.StatObject(ctx, m.bucket, param.FilePath, statOpts)
 	if err != nil {
-		return "", nil, err
+		return "", nil, fmt.Errorf("getting object stats: %w", err)
 	}
 
 	// Generate the presigned URL
@@ -197,42 +208,57 @@ func (m *minio) UploadFileBytes(ctx context.Context, param *UploadFileBytesParam
 	}
 	expiryDuration := time.Hour * 24 * time.Duration(expiryDays)
 
-	m.logAction(param.UserUID, "presignedGetObject", param.FilePath)
-	presignedURL, err := m.client.PresignedGetObject(ctx, m.bucket, param.FilePath, expiryDuration, nil)
+	// We're using PresignHeader in order to be able to pass the user UID in
+	// the request. If PresignedGetObject supports this at any point, we should
+	// use that method.
+	presignedURL, err := m.client.PresignHeader(
+		ctx,
+		http.MethodGet,
+		m.bucket,
+		param.FilePath,
+		expiryDuration,
+		nil,
+		statOpts.Header(),
+	)
 	if err != nil {
-		return "", nil, err
+		return "", nil, fmt.Errorf("getting presigned object URL: %w", err)
 	}
 
 	return presignedURL.String(), &stat, nil
 }
 
 // DeleteFile delete the file frotom minio
 func (m *minio) DeleteFile(ctx context.Context, userUID uuid.UUID, filePath string) (err error) {
-	m.logAction(userUID, "deleteObject", filePath)
+	// MinIO (and S3) API doesn't expose a way to pass headers to the deletion
+	// method. The client will be responsible of logging this information.
+	log := m.logger.With(
+		zap.String("path", filePath),
+		zap.String("userUID", userUID.String()),
+	)
+	log.Info("Object deletion in MinIO")
+
 	err = m.client.RemoveObject(ctx, m.bucket, filePath, miniogo.RemoveObjectOptions{})
 	if err != nil {
-		m.logger.Error("Failed to delete file from MinIO", zap.Error(err))
-		return err
+		log.Error("Failed to delete file from MinIO", zap.Error(err))
+		return fmt.Errorf("removing object in MinIO: %w", err)
 	}
+
 	return nil
 }
 
 // GetFile Get the object using the client
 func (m *minio) GetFile(ctx context.Context, userUID uuid.UUID, filePath string) ([]byte, error) {
-	m.logAction(userUID, "getObject", filePath)
-	object, err := m.client.GetObject(ctx, m.bucket, filePath, miniogo.GetObjectOptions{})
+	object, err := m.client.GetObject(ctx, m.bucket, filePath, m.getObjectOptions(userUID))
 	if err != nil {
-		m.logger.Error("Failed to get file from MinIO", zap.Error(err))
-		return nil, err
+		return nil, fmt.Errorf("getting object from MinIO: %w", err)
 	}
 	defer object.Close()
 
 	// Read the object's content
 	buf := new(bytes.Buffer)
 	_, err = buf.ReadFrom(object)
 	if err != nil {
-		m.logger.Error("Failed to read file from MinIO", zap.Error(err))
-		return nil, err
+		return nil, fmt.Errorf("reading MinIO object: %w", err)
 	}
 
 	return buf.Bytes(), nil
@@ -257,26 +283,15 @@ func (m *minio) GetFilesByPaths(ctx context.Context, userUID uuid.UUID, filePath
 		go func(filePath string) {
 			defer wg.Done()
 
-			m.logAction(userUID, "getObject", filePath)
-			obj, err := m.client.GetObject(ctx, m.bucket, filePath, miniogo.GetObjectOptions{})
+			obj, err := m.GetFile(ctx, userUID, filePath)
 			if err != nil {
-				m.logger.Error("Failed to get object from MinIO", zap.String("path", filePath), zap.Error(err))
-				errCh <- err
-				return
-			}
-			defer obj.Close()
-
-			var buffer bytes.Buffer
-			_, err = io.Copy(&buffer, obj)
-			if err != nil {
-				m.logger.Error("Failed to read object content", zap.String("path", filePath), zap.Error(err))
 				errCh <- err
 				return
 			}
 
 			fileContent := FileContent{
 				Name:    filePath,
-				Content: buffer.Bytes(),
+				Content: obj,
 			}
 			resultCh <- fileContent
 		}(path)
@@ -359,11 +374,9 @@ func (m *minioClientWrapper) GetFile(ctx context.Context, bucketName, objectPath
 
 	return data, info.ContentType, nil
 }
-func (m *minio) logAction(userUID uuid.UUID, action, filePath string) {
-	m.logger.Info(
-		"File action in MinIO",
-		zap.String("userUID", userUID.String()),
-		zap.String("path", filePath),
-		zap.String("action", action),
-	)
+
+func (m *minio) getObjectOptions(userUID uuid.UUID) miniogo.GetObjectOptions {
+	opts := miniogo.GetObjectOptions{}
+	opts.Set(MinIOHeaderUserUID, userUID.String())
+	return opts
 }