DRAFT: feat(encode special characters): comma for now

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
 	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
 	github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect
-	github.com/aws/aws-sdk-go v1.44.37
+	github.com/aws/aws-sdk-go v1.55.6
 	github.com/go-ole/go-ole v1.2.5 // indirect
 	github.com/gofrs/uuid v4.2.0+incompatible
 	github.com/google/uuid v1.3.0

go.sum

@@ -779,8 +779,8 @@ github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/aws/aws-sdk-go v1.44.37 h1:KvDxCX6dfJeEDC77U5GPGSP0ErecmNnhDHFxw+NIvlI=
-github.com/aws/aws-sdk-go v1.44.37/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk=
+github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=

internal/backend_s3.go

@@ -15,6 +15,9 @@
 package internal
 
 import (
+	"encoding/json"
+	"io"
+
 	. "github.com/StatCan/goofys/api/common"
 
 	"fmt"
@@ -219,7 +222,7 @@ func (s *S3Backend) detectBucketLocationByHEAD() (err error, isAws bool) {
 }
 
 func (s *S3Backend) testBucket(key string) (err error) {
-	_, err = s.HeadBlob(&HeadBlobInput{Key: key})
+	_, err = s.HeadBlob(&HeadBlobInput{Key: key}) // leave this out for now, just a test
 	if err != nil {
 		if err == fuse.ENOENT {
 			err = nil
@@ -291,6 +294,8 @@ func (s *S3Backend) ListObjectsV2(params *s3.ListObjectsV2Input) (*s3.ListObject
 	if s.aws {
 		req, resp := s.S3.ListObjectsV2Request(params)
 		err := req.Send()
+		s3Log.Debugf("MATHIS TEST list: req body %v, req data %v, req signed %v, req req %v, req params %v, req id %v, req resp %v, resp %v",
+			req.Body, req.Data, req.SignedHeaderVals, req.HTTPRequest, req.Params, req.RequestID, req.HTTPResponse, resp)
 		if err != nil {
 			return nil, "", err
 		}
@@ -311,14 +316,15 @@ func (s *S3Backend) ListObjectsV2(params *s3.ListObjectsV2Input) (*s3.ListObject
 		}
 
 		objs, err := s.S3.ListObjects(&v1)
+		s3Log.Debugf("MATHIS TEST: objs %v, err %v", objs, err)
 		if err != nil {
 			return nil, "", err
 		}
-		s3Log.Debugf("MATHIS TEST: objs %v, err %v", objs, err)
 		count := int64(len(objs.Contents))
+		// theres another one in MATHIS TEST: resp jose
 		v2Objs := s3.ListObjectsV2Output{
 			CommonPrefixes:        objs.CommonPrefixes,
-			Contents:              objs.Contents,
+			Contents:              objs.Contents, // do i need to make changes here? like before the return?
 			ContinuationToken:     objs.Marker,
 			Delimiter:             objs.Delimiter,
 			EncodingType:          objs.EncodingType,
@@ -357,8 +363,26 @@ func (s *S3Backend) getRequestId(r *request.Request) string {
 		r.HTTPResponse.Header.Get("x-amz-id-2")
 }
 
+// Jose, pointer stuff might get a bit hairy so test
+/*
+	encodedString := url.PathEscape(*i.Key)
+		items = append(items, BlobItemOutput{
+			//Key:          &escapedCommas,
+			//Key:          i.Key,
+			Key:          &encodedString,
+*/
+func encodeKey(key string) *string {
+	//encodedString := url.PathEscape(key)
+	// Attempt only encoding the commas, as the above results in also encoding the `/`'s
+	// leading to files getting stored at top level named; jose%2Finvalid instead of jose/invalid
+	encodedString := strings.ReplaceAll(key, ",", "%2C")
+	return &encodedString
+}
+
 func (s *S3Backend) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) {
+	//encodedKey := encodeKey(param.Key)
 	head := s3.HeadObjectInput{Bucket: &s.bucket,
+		//Key: encodedKey,
 		Key: &param.Key,
 	}
 	if s.config.SseC != "" {
@@ -374,6 +398,7 @@ func (s *S3Backend) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) {
 	}
 	return &HeadBlobOutput{
 		BlobItemOutput: BlobItemOutput{
+			//Key:          encodedKey,
 			Key:          &param.Key,
 			ETag:         resp.ETag,
 			LastModified: resp.LastModified,
@@ -382,7 +407,7 @@ func (s *S3Backend) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) {
 		},
 		ContentType: resp.ContentType,
 		Metadata:    metadataToLower(resp.Metadata),
-		IsDirBlob:   strings.HasSuffix(param.Key, "/"),
+		IsDirBlob:   strings.HasSuffix(param.Key, "/"), //i dont think i need to mess with this?
 		RequestId:   s.getRequestId(req),
 	}, nil
 }
@@ -414,15 +439,22 @@ func (s *S3Backend) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) {
 		prefixes = append(prefixes, BlobPrefixOutput{Prefix: p.Prefix})
 	}
 	for _, i := range resp.Contents {
+		// when is this called is this where i can escape the commma? This seems to happen after
+		// MATHIS TEST: objs but maybe this is where it matters
+		//escapedCommas := strings.ReplaceAll(*i.Key, ",", "\\,")
+		//encodedKey := encodeKey(*i.Key)
 		items = append(items, BlobItemOutput{
-			Key:          i.Key,
+			//Key:          &escapedCommas,
+			Key: i.Key,
+			//Key:          encodedKey,
 			ETag:         i.ETag,
 			LastModified: i.LastModified,
 			Size:         uint64(*i.Size),
 			StorageClass: i.StorageClass,
 		})
 	}
-	s3Log.Debugf("MATHIS TEST: prefixes %v, items %v", prefixes, items)
+	s3Log.Debugf("MATHIS TEST jose: prefixes %v, items %v", prefixes, items)
+	// `prefixes` references folders, `items` references files
 	isTruncatedFlag := false
 	if resp.IsTruncated != nil {
 		isTruncatedFlag = *resp.IsTruncated
@@ -441,7 +473,8 @@ func (s *S3Backend) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) {
 func (s *S3Backend) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) {
 	req, _ := s.DeleteObjectRequest(&s3.DeleteObjectInput{
 		Bucket: &s.bucket,
-		Key:    &param.Key,
+		//Key:    encodeKey(param.Key),
+		Key: &param.Key,
 	})
 	err := req.Send()
 	if err != nil {
@@ -457,6 +490,8 @@ func (s *S3Backend) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, er
 	var objs = make([]*s3.ObjectIdentifier, num_objs)
 
 	for i, _ := range param.Items {
+		// Jose: going to try the encode here though could get hairy
+		// will need to re-test deletes
 		objs[i] = &s3.ObjectIdentifier{Key: &param.Items[i]}
 	}
 
@@ -488,7 +523,7 @@ func (s *S3Backend) mpuCopyPart(from string, to string, mpuId string, bytes stri
 	// we are copying from the same object
 	params := &s3.UploadPartCopyInput{
 		Bucket:            &s.bucket,
-		Key:               &to,
+		Key:               &to, // no multipart support
 		CopySource:        aws.String(url.QueryEscape(from)),
 		UploadId:          &mpuId,
 		CopySourceRange:   &bytes,
@@ -572,7 +607,7 @@ func (s *S3Backend) copyObjectMultipart(size int64, from string, to string, mpuI
 	if mpuId == "" {
 		params := &s3.CreateMultipartUploadInput{ // should be unreachable, assuming this is the one that is used
 			Bucket:       &s.bucket,
-			Key:          &to,
+			Key:          &to, // no multipart support
 			StorageClass: storageClass,
 			ContentType:  s.flags.GetMimeType(to),
 			Metadata:     metadataToLower(metadata),
@@ -616,7 +651,7 @@ func (s *S3Backend) copyObjectMultipart(size int64, from string, to string, mpuI
 
 		params := &s3.CompleteMultipartUploadInput{
 			Bucket:   &s.bucket,
-			Key:      &to,
+			Key:      &to, // no multipart support
 			UploadId: &mpuId,
 			MultipartUpload: &s3.CompletedMultipartUpload{
 				Parts: parts,
@@ -645,11 +680,12 @@ func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) {
 	}
 
 	COPY_LIMIT := uint64(5 * 1024 * 1024 * 1024)
-
+	sourceParamKey := param.Source
+	//trings.ReplaceAll(key, ",", "%2C")
 	if param.Size == nil || param.ETag == nil || (*param.Size > COPY_LIMIT &&
 		(param.Metadata == nil || param.StorageClass == nil)) {
 
-		params := &HeadBlobInput{Key: param.Source}
+		params := &HeadBlobInput{Key: sourceParamKey} //moderately unsure here
 		resp, err := s.HeadBlob(params)
 		if err != nil {
 			return nil, err
@@ -671,7 +707,7 @@ func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) {
 		}
 	}
 
-	from := s.bucket + "/" + param.Source
+	from := s.bucket + "/" + sourceParamKey
 
 	/*
 		if !s.gcs && *param.Size > COPY_LIMIT {
@@ -730,10 +766,13 @@ func (s *S3Backend) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) {
 }
 
 func (s *S3Backend) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) {
+	encodedKey := encodeKey(param.Key)
 	get := s3.GetObjectInput{
 		Bucket: &s.bucket,
-		Key:    &param.Key,
+		//Key:    &param.Key,
+		Key: encodedKey,
 	}
+	// modify the getObjectInput?
 
 	if s.config.SseC != "" {
 		get.SSECustomerAlgorithm = PString("AES256")
@@ -753,25 +792,58 @@ func (s *S3Backend) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) {
 	// TODO handle IfMatch
 
 	req, resp := s.GetObjectRequest(&get)
+	// before we send the request we modify req.HttpRequest
 	err := req.Send()
+	s3Log.Debugf("MATHIS TEST list: req body %v, req data %v, req signed %v, req req %v, req params %v, req id %v, req resp %v, resp %v",
+		req.Body, req.Data, req.SignedHeaderVals, req.HTTPRequest, req.Params, req.RequestID, req.HTTPResponse, resp)
 	if err != nil {
 		return nil, mapAwsError(err)
 	}
+	//s3Log.Debugf("MATHIS TEST jose getblob: req %v, resp %v", req, resp)
+
+	// need to replicate this object not using the sdk
+	s3Log.Debugf("\nJose: DEBUG THE GETBLOB OUTPUT DATA STRUCTURE")
+	s3Log.Debugf("Etag: %v. LastModified: %v. Size: %v. StorageClass: %v. ContentType %v."+
+		"\nRequestId: %v",
+		resp.ETag, resp.LastModified, uint64(*resp.ContentLength), resp.StorageClass, resp.ContentType, s.getRequestId(req))
+	responseMetadata, _ := json.Marshal(resp.Metadata)
+	s3Log.Debugf("METADATA:\n%v", string(responseMetadata))
+
+	// for k, v := range responseMetadata {
+	// 	s3Log.Debugf("-Key:%v and value:%v\n", k, v)
+	// }
+	// s3Log.Debug("\nBODY:")
+	// buf := new(strings.Builder) // this causes it to hang
+	// _, errCopy := io.Copy(buf, resp.Body)
+	// if errCopy != nil {
+	// 	log.Fatal(err)
+	// }
+	// s3Log.Debug(buf.String()) // but in the body I do see the contents of the file and thats it
+	// "s3.DEBUG small file 1kb" for test.txt in test share
+
+	bytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+	s3Log.Debug("Printing:")
+	s3Log.Debug(string(bytes))
 
 	return &GetBlobOutput{
 		HeadBlobOutput: HeadBlobOutput{
 			BlobItemOutput: BlobItemOutput{
-				Key:          &param.Key,
+				Key:          encodedKey,
 				ETag:         resp.ETag,
 				LastModified: resp.LastModified,
 				Size:         uint64(*resp.ContentLength),
-				StorageClass: resp.StorageClass,
+				StorageClass: resp.StorageClass, // in testing doesnt exist / is nil (should be a header)
 			},
 			ContentType: resp.ContentType,
 			Metadata:    metadataToLower(resp.Metadata),
 		},
-		Body:      resp.Body,
-		RequestId: s.getRequestId(req),
+		Body:      resp.Body,           // gives me &{0x4c25a0 0xc00012b240 0x68d300}
+		RequestId: s.getRequestId(req), // this just looks like; 1577957049:
+		// does not have that extra "x-amz-id-2" since i guess it does not exist
+		// given that it works just fine we can keep it at whatever it actually gives
 	}, nil
 }
 
@@ -795,7 +867,8 @@ func (s *S3Backend) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) {
 	}
 
 	put := &s3.PutObjectInput{
-		Bucket:       &s.bucket,
+		Bucket: &s.bucket,
+		//Key:          encodeKey(param.Key),
 		Key:          &param.Key,
 		Metadata:     metadataToLower(param.Metadata),
 		Body:         param.Body,
@@ -835,6 +908,7 @@ func (s *S3Backend) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) {
 // reached from file.go
 func (s *S3Backend) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) {
 	// references API then
+	// no multipart upload
 	mpu := s3.CreateMultipartUploadInput{
 		Bucket:       &s.bucket,
 		Key:          &param.Key,
@@ -878,7 +952,7 @@ func (s *S3Backend) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBl
 
 	params := s3.UploadPartInput{
 		Bucket:     &s.bucket,
-		Key:        param.Commit.Key,
+		Key:        param.Commit.Key, // no multipart
 		PartNumber: aws.Int64(int64(param.PartNumber)),
 		UploadId:   param.Commit.UploadId,
 		Body:       param.Body,
@@ -912,7 +986,7 @@ func (s *S3Backend) MultipartBlobCommit(param *MultipartBlobCommitInput) (*Multi
 			PartNumber: aws.Int64(int64(i + 1)),
 		}
 	}
-
+	// no multipart
 	mpu := s3.CompleteMultipartUploadInput{
 		Bucket:   &s.bucket,
 		Key:      param.Key,
@@ -940,6 +1014,7 @@ func (s *S3Backend) MultipartBlobCommit(param *MultipartBlobCommitInput) (*Multi
 }
 
 func (s *S3Backend) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) {
+	// dont support multipart afaik
 	mpu := s3.AbortMultipartUploadInput{
 		Bucket:   &s.bucket,
 		Key:      param.Key,
@@ -967,6 +1042,7 @@ func (s *S3Backend) MultipartExpire(param *MultipartExpireInput) (*MultipartExpi
 		expireTime := upload.Initiated.Add(48 * time.Hour)
 
 		if !expireTime.After(now) {
+			// dont support multipart
 			params := &s3.AbortMultipartUploadInput{
 				Bucket:   &s.bucket,
 				Key:      upload.Key,