From 3a03e1c63c7f3cdccc2311abdd51727a705ab188 Mon Sep 17 00:00:00 2001 From: cappyzawa Date: Wed, 12 Dec 2018 12:47:39 +0900 Subject: [PATCH 1/2] fix based on reviews --- models.go | 1 + out/command.go | 1 + out/command_test.go | 23 +++++++++++++++++++++++ s3client.go | 23 +++++++++++++++-------- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/models.go b/models.go index 1a944f7e..c2f4adfc 100644 --- a/models.go +++ b/models.go @@ -21,6 +21,7 @@ type Source struct { InitialPath string `json:"initial_path"` InitialContentText string `json:"initial_content_text"` InitialContentBinary string `json:"initial_content_binary"` + DisableMultipart bool `json:"disable_multipart"` } func (source Source) IsValid() (bool, string) { diff --git a/out/command.go b/out/command.go index 765b50f0..f07ee415 100644 --- a/out/command.go +++ b/out/command.go @@ -64,6 +64,7 @@ func (command *Command) Run(sourceDir string, request Request) (Response, error) options.ContentType = request.Params.ContentType options.ServerSideEncryption = request.Source.ServerSideEncryption options.KmsKeyId = request.Source.SSEKMSKeyId + options.DisableMultipart = request.Source.DisableMultipart versionID, err := command.s3client.UploadFile( bucketName, diff --git a/out/command_test.go b/out/command_test.go index 0fdb86db..2652d5e6 100644 --- a/out/command_test.go +++ b/out/command_test.go @@ -386,5 +386,28 @@ var _ = Describe("Out Command", func() { Ω(options.ContentType).Should(Equal("")) }) }) + + Context("when specifying disable multipart upload", func() { + It("uploads to the parent directory without multipart upload", func() { + request.Params.File = "my/special-file.tgz" + request.Source.Regexp = "a-folder/some-file-(.*).tgz" + request.Source.DisableMultipart = true + createFile("my/special-file.tgz") + + response, err := command.Run(sourceDir, request) + Expect(err).ToNot(HaveOccurred()) + Ω(s3client.UploadFileCallCount()).Should(Equal(1)) + + bucketName, remotePath, localPath, options := s3client.UploadFileArgsForCall(0) + Expect(bucketName).To(Equal("bucket-name")) + Expect(remotePath).To(Equal("a-folder/special-file.tgz")) + Expect(localPath).To(Equal(filepath.Join(sourceDir, "my/special-file.tgz"))) + Ω(options).Should(Equal(s3resource.UploadFileOptions{Acl: "private", DisableMultipart: true})) + + Expect(response.Metadata[0].Name).To(Equal("filename")) + Expect(response.Metadata[0].Value).To(Equal("special-file.tgz")) + }) + }) + }) }) diff --git a/s3client.go b/s3client.go index a860e9ed..12e5d6af 100644 --- a/s3client.go +++ b/s3client.go @@ -50,6 +50,7 @@ type UploadFileOptions struct { ServerSideEncryption string KmsKeyId string ContentType string + DisableMultipart bool } func NewUploadFileOptions() UploadFileOptions { @@ -184,15 +185,21 @@ func (client *s3client) UploadFile(bucketName string, remotePath string, localPa } defer localFile.Close() - + // Automatically adjust partsize for larger files. fSize := stat.Size() - if fSize > int64(uploader.MaxUploadParts) * uploader.PartSize { - partSize := fSize / int64(uploader.MaxUploadParts) - if fSize % int64(uploader.MaxUploadParts) != 0 { - partSize++ + if !options.DisableMultipart { + if fSize > int64(uploader.MaxUploadParts)*uploader.PartSize { + partSize := fSize / int64(uploader.MaxUploadParts) + if fSize%int64(uploader.MaxUploadParts) != 0 { + partSize++ + } + uploader.PartSize = partSize } - uploader.PartSize = partSize + } else { + uploader.MaxUploadParts = 1 + uploader.Concurrency = 1 + uploader.PartSize = fSize + 1 } progress := client.newProgressBar(fSize) @@ -385,8 +392,8 @@ func (client *s3client) getVersionedBucketContents(bucketName string, prefix str for { params := &s3.ListObjectVersionsInput{ - Bucket: aws.String(bucketName), - Prefix: aws.String(prefix), + Bucket: aws.String(bucketName), + Prefix: aws.String(prefix), } if keyMarker != "" { From 7dff543f216adb325aeb5677ed4868744f01d606 Mon Sep 17 00:00:00 2001 From: cappyzawa Date: Wed, 12 Dec 2018 12:48:10 +0900 Subject: [PATCH 2/2] add description for disable multipart param --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 63485a13..f6fc7c5a 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ version numbers. * `use_v2_signing`: *Optional.* Use signature v2 signing, useful for S3 compatible providers that do not support v4. +* `disable_multipart`: *Optional.* Disable Multipart Upload. useful for S3 compatible providers that do not support multipart upload. + ### File Names One of the following two options must be specified: