From 9f2f40fa48e498926beed93bac41d9b5fd89fc52 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Wed, 6 Mar 2024 16:51:50 +0200 Subject: [PATCH 01/22] update deps --- http/multipart/multipart.go | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 http/multipart/multipart.go diff --git a/http/multipart/multipart.go b/http/multipart/multipart.go new file mode 100644 index 0000000..11ef92b --- /dev/null +++ b/http/multipart/multipart.go @@ -0,0 +1,48 @@ +package multipart + +import ( + "fmt" + fileutils "github.com/jfrog/gofrog/io" + "io" + "mime/multipart" + "os" +) + +type FileWriterFunc func(fileName string) (io.WriteCloser, error) + +func ReadFromStream(multipartReader *multipart.Reader, fileWriterFunc FileWriterFunc) error { + for { + // Read the next file streamed from client + fileReader, err := multipartReader.NextPart() + if err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("failed to read file: %w", err) + } + fileName := fileReader.FileName() + fileWriter, err := fileWriterFunc(fileName) + if err != nil { + return err + } + defer fileutils.Close(fileWriter, &err) + + // Stream file directly to disk + if _, err = io.Copy(fileWriter, fileReader); err != nil { + return fmt.Errorf("failed writing '%s' file: %w", fileName, err) + } + } + return nil +} + +func exampleFunc(filepath string) (fileWriter io.WriteCloser, err error) { + fileWriter, err = os.Create(filepath) + if err != nil { + return nil, fmt.Errorf("create file: %w", err) + } + return +} + +func myMain(filepath string) { + ReadFromStream(nil, exampleFunc) +} From 4b1d3737849f7181d6e26f4a3ba44190d903144d Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 7 Mar 2024 12:34:34 +0200 Subject: [PATCH 02/22] update deps --- go.mod | 1 + go.sum | 6 ++++ http/multipart/multipart.go | 61 ++++++++++++++++++++++++++++++++----- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index a60ea6b..396038f 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/jfrog/gofrog go 1.20 require ( + github.com/cespare/xxhash v1.1.0 github.com/jfrog/archiver/v3 v3.6.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index b9baeb6..f0eb7be 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= @@ -23,6 +27,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= diff --git a/http/multipart/multipart.go b/http/multipart/multipart.go index 11ef92b..4ba7b54 100644 --- a/http/multipart/multipart.go +++ b/http/multipart/multipart.go @@ -1,16 +1,19 @@ package multipart import ( + "errors" "fmt" - fileutils "github.com/jfrog/gofrog/io" + "github.com/cespare/xxhash" + ioutils "github.com/jfrog/gofrog/io" + "hash" "io" "mime/multipart" "os" ) -type FileWriterFunc func(fileName string) (io.WriteCloser, error) +type FileWriterFunc func(fileName string) (io.Writer, error) -func ReadFromStream(multipartReader *multipart.Reader, fileWriterFunc FileWriterFunc) error { +func ReadFilesFromStream(multipartReader *multipart.Reader, fileWriterFunc FileWriterFunc) error { for { // Read the next file streamed from client fileReader, err := multipartReader.NextPart() @@ -25,9 +28,7 @@ func ReadFromStream(multipartReader *multipart.Reader, fileWriterFunc FileWriter if err != nil { return err } - defer fileutils.Close(fileWriter, &err) - // Stream file directly to disk if _, err = io.Copy(fileWriter, fileReader); err != nil { return fmt.Errorf("failed writing '%s' file: %w", fileName, err) } @@ -35,14 +36,58 @@ func ReadFromStream(multipartReader *multipart.Reader, fileWriterFunc FileWriter return nil } -func exampleFunc(filepath string) (fileWriter io.WriteCloser, err error) { - fileWriter, err = os.Create(filepath) +func WriteFilesToStream(writer io.Writer, filesList []string) error { + multipartWriter := multipart.NewWriter(writer) + for _, filePath := range filesList { + fileReader, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + defer ioutils.Close(fileReader, &err) + fileWriter, err := multipartWriter.CreateFormFile("fieldname", filePath) + if err != nil { + return fmt.Errorf("failed to CreateFormFile: %w", err) + } + _, err = io.Copy(fileWriter, fileReader) + if err != nil { + return err + } + } + return nil +} + +func getFileWriter(fileName string) (fileWriter io.Writer, err error) { + lockfileMock := map[string]string{ + "SDFDSFSDFSDFDSF": "file1", + "XXSDFDSFSDFSDFDSF": "file2", + } + realFileName := lockfileMock[fileName] + fileWriter, err = os.Create(realFileName) if err != nil { return nil, fmt.Errorf("create file: %w", err) } + // Currently we are using the file name as the hash + fileHash := fileName + return io.MultiWriter(fileWriter, NewHashWrapper(fileHash)), nil +} + +type HashWrapper struct { + hash hash.Hash64 + actualChecksum string +} + +func NewHashWrapper(actualChecksum string) *HashWrapper { + return &HashWrapper{hash: xxhash.New(), actualChecksum: actualChecksum} +} + +func (hw *HashWrapper) Write(p []byte) (n int, err error) { + n, err = hw.hash.Write(p) + if fmt.Sprintf("%x", hw.hash.Sum(nil)) != hw.actualChecksum { + err = errors.New("checksum mismatch") + } return } func myMain(filepath string) { - ReadFromStream(nil, exampleFunc) + ReadFilesFromStream(nil, getFileWriter) } From 65f07f32aac7ae9f8ff95f4ecdaabb241c4b6152 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 7 Mar 2024 20:19:30 +0200 Subject: [PATCH 03/22] update deps --- http/multipart/multipart.go | 52 +++++-------------------------- http/multipart/multipart_test.go | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 44 deletions(-) create mode 100644 http/multipart/multipart_test.go diff --git a/http/multipart/multipart.go b/http/multipart/multipart.go index 4ba7b54..248419c 100644 --- a/http/multipart/multipart.go +++ b/http/multipart/multipart.go @@ -3,22 +3,20 @@ package multipart import ( "errors" "fmt" - "github.com/cespare/xxhash" ioutils "github.com/jfrog/gofrog/io" - "hash" "io" "mime/multipart" "os" ) -type FileWriterFunc func(fileName string) (io.Writer, error) +type FileWriterFunc func(fileName string) (io.WriteCloser, error) func ReadFilesFromStream(multipartReader *multipart.Reader, fileWriterFunc FileWriterFunc) error { for { // Read the next file streamed from client fileReader, err := multipartReader.NextPart() if err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { break } return fmt.Errorf("failed to read file: %w", err) @@ -32,19 +30,21 @@ func ReadFilesFromStream(multipartReader *multipart.Reader, fileWriterFunc FileW if _, err = io.Copy(fileWriter, fileReader); err != nil { return fmt.Errorf("failed writing '%s' file: %w", fileName, err) } + ioutils.Close(fileWriter, &err) } return nil } -func WriteFilesToStream(writer io.Writer, filesList []string) error { - multipartWriter := multipart.NewWriter(writer) - for _, filePath := range filesList { +func WriteFilesToStream(multipartWriter *multipart.Writer, filePaths []string) (err error) { + defer ioutils.Close(multipartWriter, &err) + + for _, filePath := range filePaths { fileReader, err := os.Open(filePath) if err != nil { return fmt.Errorf("failed to open file: %w", err) } defer ioutils.Close(fileReader, &err) - fileWriter, err := multipartWriter.CreateFormFile("fieldname", filePath) + fileWriter, err := multipartWriter.CreateFormFile("file", filePath) if err != nil { return fmt.Errorf("failed to CreateFormFile: %w", err) } @@ -55,39 +55,3 @@ func WriteFilesToStream(writer io.Writer, filesList []string) error { } return nil } - -func getFileWriter(fileName string) (fileWriter io.Writer, err error) { - lockfileMock := map[string]string{ - "SDFDSFSDFSDFDSF": "file1", - "XXSDFDSFSDFSDFDSF": "file2", - } - realFileName := lockfileMock[fileName] - fileWriter, err = os.Create(realFileName) - if err != nil { - return nil, fmt.Errorf("create file: %w", err) - } - // Currently we are using the file name as the hash - fileHash := fileName - return io.MultiWriter(fileWriter, NewHashWrapper(fileHash)), nil -} - -type HashWrapper struct { - hash hash.Hash64 - actualChecksum string -} - -func NewHashWrapper(actualChecksum string) *HashWrapper { - return &HashWrapper{hash: xxhash.New(), actualChecksum: actualChecksum} -} - -func (hw *HashWrapper) Write(p []byte) (n int, err error) { - n, err = hw.hash.Write(p) - if fmt.Sprintf("%x", hw.hash.Sum(nil)) != hw.actualChecksum { - err = errors.New("checksum mismatch") - } - return -} - -func myMain(filepath string) { - ReadFilesFromStream(nil, getFileWriter) -} diff --git a/http/multipart/multipart_test.go b/http/multipart/multipart_test.go new file mode 100644 index 0000000..287725a --- /dev/null +++ b/http/multipart/multipart_test.go @@ -0,0 +1,53 @@ +package multipart + +import ( + "bytes" + "github.com/stretchr/testify/assert" + "io" + "mime/multipart" + "os" + "path" + "path/filepath" + "testing" +) + +var targetDir string + +func TestReadFilesFromStream(t *testing.T) { + sourceDir := t.TempDir() + + // Create 2 file to be transferred via our multipart stream + file1 := path.Join(sourceDir, "test1.txt") + file2 := path.Join(sourceDir, "test2.txt") + file1Content := []byte("test content1") + file2Content := []byte("test content2") + assert.NoError(t, os.WriteFile(file1, file1Content, 0644)) + assert.NoError(t, os.WriteFile(file2, file2Content, 0644)) + + // Create the multipart writer that will stream our files + body := &bytes.Buffer{} + multipartWriter := multipart.NewWriter(body) + assert.NoError(t, WriteFilesToStream(multipartWriter, []string{file1, file2})) + + // Create local temp dir that will store our files + targetDir = t.TempDir() + + // Create the multipart reader that will read the files from the stream + multipartReader := multipart.NewReader(body, multipartWriter.Boundary()) + assert.NoError(t, ReadFilesFromStream(multipartReader, getFileWriter)) + + // Validate file 1 transferred successfully + content, err := os.ReadFile(filepath.Join(targetDir, "test1.txt")) + assert.NoError(t, err) + assert.Equal(t, file1Content, content) + + // Validate file 2 transferred successfully + content, err = os.ReadFile(filepath.Join(targetDir, "test2.txt")) + assert.NoError(t, err) + assert.Equal(t, file2Content, content) + +} + +func getFileWriter(fileName string) (fileWriter io.WriteCloser, err error) { + return os.Create(filepath.Join(targetDir, fileName)) +} From 2ef07249ea3c72887d71d36c110d0b506bb28e44 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Thu, 7 Mar 2024 20:33:11 +0200 Subject: [PATCH 04/22] update deps --- http/multipart/multipart_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http/multipart/multipart_test.go b/http/multipart/multipart_test.go index 287725a..c6dcf55 100644 --- a/http/multipart/multipart_test.go +++ b/http/multipart/multipart_test.go @@ -21,8 +21,8 @@ func TestReadFilesFromStream(t *testing.T) { file2 := path.Join(sourceDir, "test2.txt") file1Content := []byte("test content1") file2Content := []byte("test content2") - assert.NoError(t, os.WriteFile(file1, file1Content, 0644)) - assert.NoError(t, os.WriteFile(file2, file2Content, 0644)) + assert.NoError(t, os.WriteFile(file1, file1Content, 0600)) + assert.NoError(t, os.WriteFile(file2, file2Content, 0600)) // Create the multipart writer that will stream our files body := &bytes.Buffer{} From 82ca55f15df489d88503041dded8bd93880f7c14 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Mar 2024 14:17:25 +0200 Subject: [PATCH 05/22] update deps --- go.mod | 1 - go.sum | 6 ------ http/multipart/{multipart.go => filestream.go} | 2 +- http/multipart/{multipart_test.go => filestream_test.go} | 7 +++---- 4 files changed, 4 insertions(+), 12 deletions(-) rename http/multipart/{multipart.go => filestream.go} (98%) rename http/multipart/{multipart_test.go => filestream_test.go} (92%) diff --git a/go.mod b/go.mod index 396038f..a60ea6b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/jfrog/gofrog go 1.20 require ( - github.com/cespare/xxhash v1.1.0 github.com/jfrog/archiver/v3 v3.6.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index f0eb7be..b9baeb6 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,5 @@ -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= @@ -27,8 +23,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= diff --git a/http/multipart/multipart.go b/http/multipart/filestream.go similarity index 98% rename from http/multipart/multipart.go rename to http/multipart/filestream.go index 248419c..835c6f3 100644 --- a/http/multipart/multipart.go +++ b/http/multipart/filestream.go @@ -1,4 +1,4 @@ -package multipart +package filestream import ( "errors" diff --git a/http/multipart/multipart_test.go b/http/multipart/filestream_test.go similarity index 92% rename from http/multipart/multipart_test.go rename to http/multipart/filestream_test.go index c6dcf55..2ac4dec 100644 --- a/http/multipart/multipart_test.go +++ b/http/multipart/filestream_test.go @@ -1,4 +1,4 @@ -package multipart +package filestream import ( "bytes" @@ -6,7 +6,6 @@ import ( "io" "mime/multipart" "os" - "path" "path/filepath" "testing" ) @@ -17,8 +16,8 @@ func TestReadFilesFromStream(t *testing.T) { sourceDir := t.TempDir() // Create 2 file to be transferred via our multipart stream - file1 := path.Join(sourceDir, "test1.txt") - file2 := path.Join(sourceDir, "test2.txt") + file1 := filepath.Join(sourceDir, "test1.txt") + file2 := filepath.Join(sourceDir, "test2.txt") file1Content := []byte("test content1") file2Content := []byte("test content2") assert.NoError(t, os.WriteFile(file1, file1Content, 0600)) From ecb92b8dbe9fe84c72e8398a991aacfa4f356b23 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Mar 2024 14:24:56 +0200 Subject: [PATCH 06/22] update deps --- http/{multipart => filestream}/filestream.go | 0 http/{multipart => filestream}/filestream_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename http/{multipart => filestream}/filestream.go (100%) rename http/{multipart => filestream}/filestream_test.go (100%) diff --git a/http/multipart/filestream.go b/http/filestream/filestream.go similarity index 100% rename from http/multipart/filestream.go rename to http/filestream/filestream.go diff --git a/http/multipart/filestream_test.go b/http/filestream/filestream_test.go similarity index 100% rename from http/multipart/filestream_test.go rename to http/filestream/filestream_test.go From c25565957a4c8dea75fddfc2226c663d8edf9d9e Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Mar 2024 17:05:39 +0200 Subject: [PATCH 07/22] update deps --- go.mod | 2 ++ go.sum | 5 +++ http/filestream/filestream.go | 55 ++++++++++++++++++------------ http/filestream/filestream_test.go | 47 +++++++++++++++++++++---- 4 files changed, 81 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index a60ea6b..06f55ba 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/jfrog/archiver/v3 v3.6.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.4 + github.com/zeebo/xxh3 v1.0.2 ) require ( @@ -14,6 +15,7 @@ require ( github.com/dsnet/compress v0.0.1 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect diff --git a/go.sum b/go.sum index b9baeb6..8a65411 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= @@ -30,6 +32,9 @@ github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/http/filestream/filestream.go b/http/filestream/filestream.go index 835c6f3..65d9c1d 100644 --- a/http/filestream/filestream.go +++ b/http/filestream/filestream.go @@ -6,12 +6,19 @@ import ( ioutils "github.com/jfrog/gofrog/io" "io" "mime/multipart" + "net/http" "os" ) -type FileWriterFunc func(fileName string) (io.WriteCloser, error) +const ( + contentType = "Content-Type" + FileType = "file" +) + +// The expected type of function that should be provided to the ReadFilesFromStream func, that returns the writer that should handle each file +type FileHandlerFunc func(fileName string) (writer io.Writer, err error) -func ReadFilesFromStream(multipartReader *multipart.Reader, fileWriterFunc FileWriterFunc) error { +func ReadFilesFromStream(multipartReader *multipart.Reader, fileHandlerFunc FileHandlerFunc) error { for { // Read the next file streamed from client fileReader, err := multipartReader.NextPart() @@ -22,36 +29,42 @@ func ReadFilesFromStream(multipartReader *multipart.Reader, fileWriterFunc FileW return fmt.Errorf("failed to read file: %w", err) } fileName := fileReader.FileName() - fileWriter, err := fileWriterFunc(fileName) + fileWriter, err := fileHandlerFunc(fileName) if err != nil { return err } - if _, err = io.Copy(fileWriter, fileReader); err != nil { - return fmt.Errorf("failed writing '%s' file: %w", fileName, err) + err = fmt.Errorf("failed writing '%s' file: %w", fileName, err) } - ioutils.Close(fileWriter, &err) } return nil } -func WriteFilesToStream(multipartWriter *multipart.Writer, filePaths []string) (err error) { - defer ioutils.Close(multipartWriter, &err) +func WriteFilesToStream(responseWriter http.ResponseWriter, filePaths []string) (err error) { + multipartWriter := multipart.NewWriter(responseWriter) + responseWriter.Header().Set(contentType, multipartWriter.FormDataContentType()) for _, filePath := range filePaths { - fileReader, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open file: %w", err) - } - defer ioutils.Close(fileReader, &err) - fileWriter, err := multipartWriter.CreateFormFile("file", filePath) - if err != nil { - return fmt.Errorf("failed to CreateFormFile: %w", err) - } - _, err = io.Copy(fileWriter, fileReader) - if err != nil { - return err + if err = writeSingleFile(multipartWriter, filePath); err != nil { + return } } - return nil + + // Close finishes the multipart message and writes the trailing + // boundary end line to the output. + return multipartWriter.Close() +} + +func writeSingleFile(multipartWriter *multipart.Writer, filePath string) (err error) { + fileReader, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + defer ioutils.Close(fileReader, &err) + fileWriter, err := multipartWriter.CreateFormFile(FileType, filePath) + if err != nil { + return fmt.Errorf("failed to CreateFormFile: %w", err) + } + _, err = io.Copy(fileWriter, fileReader) + return err } diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index 2ac4dec..4541f61 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -1,12 +1,17 @@ package filestream import ( - "bytes" + "errors" + "fmt" "github.com/stretchr/testify/assert" + "github.com/zeebo/xxh3" + "hash" "io" "mime/multipart" + "net/http/httptest" "os" "path/filepath" + "strings" "testing" ) @@ -24,16 +29,17 @@ func TestReadFilesFromStream(t *testing.T) { assert.NoError(t, os.WriteFile(file2, file2Content, 0600)) // Create the multipart writer that will stream our files - body := &bytes.Buffer{} - multipartWriter := multipart.NewWriter(body) - assert.NoError(t, WriteFilesToStream(multipartWriter, []string{file1, file2})) + responseWriter := httptest.NewRecorder() + assert.NoError(t, WriteFilesToStream(responseWriter, []string{file1, file2})) // Create local temp dir that will store our files targetDir = t.TempDir() + // Get boundary hash from writer + boundary := strings.Split(responseWriter.Header().Get(contentType), "boundary=")[1] // Create the multipart reader that will read the files from the stream - multipartReader := multipart.NewReader(body, multipartWriter.Boundary()) - assert.NoError(t, ReadFilesFromStream(multipartReader, getFileWriter)) + multipartReader := multipart.NewReader(responseWriter.Body, boundary) + assert.NoError(t, ReadFilesFromStream(multipartReader, simpleFileHandler)) // Validate file 1 transferred successfully content, err := os.ReadFile(filepath.Join(targetDir, "test1.txt")) @@ -47,6 +53,33 @@ func TestReadFilesFromStream(t *testing.T) { } -func getFileWriter(fileName string) (fileWriter io.WriteCloser, err error) { +func simpleFileHandler(fileName string) (fileWriter io.Writer, err error) { return os.Create(filepath.Join(targetDir, fileName)) } + +func fileHandlerWithHash(fileName string) (fileWriter io.Writer, err error) { + fileWriter, err = simpleFileHandler(fileName) + if err != nil { + return + } + // GetExpectedHashFromLockFile(fileName) + expectedHash := "SDFDSFSDFSDFDSF" + return io.MultiWriter(fileWriter, NewHashWrapper(expectedHash)), nil +} + +type HashWrapper struct { + hash hash.Hash64 + actualChecksum string +} + +func NewHashWrapper(actualChecksum string) *HashWrapper { + return &HashWrapper{hash: xxh3.New(), actualChecksum: actualChecksum} +} + +func (hw *HashWrapper) Write(p []byte) (n int, err error) { + n, err = hw.hash.Write(p) + if fmt.Sprintf("%x", hw.hash.Sum(nil)) != hw.actualChecksum { + err = errors.New("checksum mismatch") + } + return +} From 7910788dda3c7758122a53f59ec682e12be29424 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Mar 2024 17:12:53 +0200 Subject: [PATCH 08/22] update deps --- http/filestream/filestream_test.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index 4541f61..42440cd 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -39,7 +39,7 @@ func TestReadFilesFromStream(t *testing.T) { boundary := strings.Split(responseWriter.Header().Get(contentType), "boundary=")[1] // Create the multipart reader that will read the files from the stream multipartReader := multipart.NewReader(responseWriter.Body, boundary) - assert.NoError(t, ReadFilesFromStream(multipartReader, simpleFileHandler)) + assert.NoError(t, ReadFilesFromStream(multipartReader, fileHandlerWithHashValidation)) // Validate file 1 transferred successfully content, err := os.ReadFile(filepath.Join(targetDir, "test1.txt")) @@ -57,28 +57,31 @@ func simpleFileHandler(fileName string) (fileWriter io.Writer, err error) { return os.Create(filepath.Join(targetDir, fileName)) } -func fileHandlerWithHash(fileName string) (fileWriter io.Writer, err error) { +func fileHandlerWithHashValidation(fileName string) (fileWriter io.Writer, err error) { fileWriter, err = simpleFileHandler(fileName) if err != nil { return } // GetExpectedHashFromLockFile(fileName) - expectedHash := "SDFDSFSDFSDFDSF" - return io.MultiWriter(fileWriter, NewHashWrapper(expectedHash)), nil + lockFileMock := map[string]string{ + "file1": "070afab2066d3b16", + "file2": "070afab2066d3b16", + } + return io.MultiWriter( + fileWriter, + &HashValidator{hash: xxh3.New(), actualChecksum: lockFileMock[fileName]}, + ), nil } -type HashWrapper struct { +type HashValidator struct { hash hash.Hash64 actualChecksum string } -func NewHashWrapper(actualChecksum string) *HashWrapper { - return &HashWrapper{hash: xxh3.New(), actualChecksum: actualChecksum} -} - -func (hw *HashWrapper) Write(p []byte) (n int, err error) { +func (hw *HashValidator) Write(p []byte) (n int, err error) { n, err = hw.hash.Write(p) - if fmt.Sprintf("%x", hw.hash.Sum(nil)) != hw.actualChecksum { + sd := fmt.Sprintf("%x", hw.hash.Sum(nil)) + if sd != hw.actualChecksum { err = errors.New("checksum mismatch") } return From b1899de046c6561903c10cea0f5f9ee97ed1f936 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Mar 2024 17:22:02 +0200 Subject: [PATCH 09/22] update deps --- http/filestream/filestream.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/filestream/filestream.go b/http/filestream/filestream.go index 65d9c1d..fd47b7e 100644 --- a/http/filestream/filestream.go +++ b/http/filestream/filestream.go @@ -34,7 +34,7 @@ func ReadFilesFromStream(multipartReader *multipart.Reader, fileHandlerFunc File return err } if _, err = io.Copy(fileWriter, fileReader); err != nil { - err = fmt.Errorf("failed writing '%s' file: %w", fileName, err) + return fmt.Errorf("failed writing '%s' file: %w", fileName, err) } } return nil From f2deb38e19f95fc6b14d8f3567c56f9b9b110885 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Mar 2024 17:48:11 +0200 Subject: [PATCH 10/22] update deps --- http/filestream/filestream_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index 42440cd..c02a0a8 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -17,9 +17,8 @@ import ( var targetDir string -func TestReadFilesFromStream(t *testing.T) { +func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { sourceDir := t.TempDir() - // Create 2 file to be transferred via our multipart stream file1 := filepath.Join(sourceDir, "test1.txt") file2 := filepath.Join(sourceDir, "test2.txt") @@ -42,19 +41,22 @@ func TestReadFilesFromStream(t *testing.T) { assert.NoError(t, ReadFilesFromStream(multipartReader, fileHandlerWithHashValidation)) // Validate file 1 transferred successfully - content, err := os.ReadFile(filepath.Join(targetDir, "test1.txt")) + file1 = filepath.Join(targetDir, "test1.txt") + assert.FileExists(t, file1) + content, err := os.ReadFile(file1) assert.NoError(t, err) assert.Equal(t, file1Content, content) // Validate file 2 transferred successfully - content, err = os.ReadFile(filepath.Join(targetDir, "test2.txt")) + file2 = filepath.Join(targetDir, "test2.txt") + assert.FileExists(t, file2) + content, err = os.ReadFile(file2) assert.NoError(t, err) assert.Equal(t, file2Content, content) - } func simpleFileHandler(fileName string) (fileWriter io.Writer, err error) { - return os.Create(filepath.Join(targetDir, fileName)) + return os.OpenFile(filepath.Join(targetDir, fileName), os.O_RDWR|os.O_CREATE, 0777) } func fileHandlerWithHashValidation(fileName string) (fileWriter io.Writer, err error) { From ec736de3e820c35db94a803c6e9353b5839da769 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Mar 2024 17:53:53 +0200 Subject: [PATCH 11/22] update deps --- http/filestream/filestream.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http/filestream/filestream.go b/http/filestream/filestream.go index fd47b7e..f80cad8 100644 --- a/http/filestream/filestream.go +++ b/http/filestream/filestream.go @@ -45,7 +45,7 @@ func WriteFilesToStream(responseWriter http.ResponseWriter, filePaths []string) responseWriter.Header().Set(contentType, multipartWriter.FormDataContentType()) for _, filePath := range filePaths { - if err = writeSingleFile(multipartWriter, filePath); err != nil { + if err = writeFile(multipartWriter, filePath); err != nil { return } } @@ -55,7 +55,7 @@ func WriteFilesToStream(responseWriter http.ResponseWriter, filePaths []string) return multipartWriter.Close() } -func writeSingleFile(multipartWriter *multipart.Writer, filePath string) (err error) { +func writeFile(multipartWriter *multipart.Writer, filePath string) (err error) { fileReader, err := os.Open(filePath) if err != nil { return fmt.Errorf("failed to open file: %w", err) From 3b757f18435fe8b0bbda00c447a1b66b411db109 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Mar 2024 18:10:53 +0200 Subject: [PATCH 12/22] update deps --- http/filestream/filestream_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index c02a0a8..6f6e285 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -66,8 +66,8 @@ func fileHandlerWithHashValidation(fileName string) (fileWriter io.Writer, err e } // GetExpectedHashFromLockFile(fileName) lockFileMock := map[string]string{ - "file1": "070afab2066d3b16", - "file2": "070afab2066d3b16", + "test1": "070afab2066d3b16", + "test2": "070afab2066d3b16", } return io.MultiWriter( fileWriter, From ccec754eef27ffb107d044c3f96c8f043ca0df89 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Mar 2024 18:30:40 +0200 Subject: [PATCH 13/22] update deps --- http/filestream/filestream_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index 6f6e285..8f19b2d 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -66,8 +66,8 @@ func fileHandlerWithHashValidation(fileName string) (fileWriter io.Writer, err e } // GetExpectedHashFromLockFile(fileName) lockFileMock := map[string]string{ - "test1": "070afab2066d3b16", - "test2": "070afab2066d3b16", + "test1.txt": "070afab2066d3b16", + "test2.txt": "48bc7295420af89d", } return io.MultiWriter( fileWriter, From 303dc5b102efe38f911d36a96f72f1583274e7d7 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Mar 2024 18:49:14 +0200 Subject: [PATCH 14/22] update deps --- http/filestream/filestream_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index 8f19b2d..fbaaf0b 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -46,6 +46,7 @@ func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { content, err := os.ReadFile(file1) assert.NoError(t, err) assert.Equal(t, file1Content, content) + assert.NoError(t, os.Remove(file1)) // Validate file 2 transferred successfully file2 = filepath.Join(targetDir, "test2.txt") @@ -53,6 +54,7 @@ func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { content, err = os.ReadFile(file2) assert.NoError(t, err) assert.Equal(t, file2Content, content) + assert.NoError(t, os.Remove(file2)) } func simpleFileHandler(fileName string) (fileWriter io.Writer, err error) { From 7712d46cea63d8940fad9c274ea51417d957ae81 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Sun, 10 Mar 2024 18:49:47 +0200 Subject: [PATCH 15/22] update deps --- http/filestream/filestream_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index fbaaf0b..905e125 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -58,6 +58,7 @@ func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { } func simpleFileHandler(fileName string) (fileWriter io.Writer, err error) { + // #nosec G302 return os.OpenFile(filepath.Join(targetDir, fileName), os.O_RDWR|os.O_CREATE, 0777) } From 579e7a5030630317b5b9e86cf2c865a8a330fb80 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Mon, 11 Mar 2024 10:52:23 +0200 Subject: [PATCH 16/22] update deps --- http/filestream/filestream.go | 6 ++++- http/filestream/filestream_test.go | 36 ++++++++++++++++-------------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/http/filestream/filestream.go b/http/filestream/filestream.go index f80cad8..50e9b90 100644 --- a/http/filestream/filestream.go +++ b/http/filestream/filestream.go @@ -16,7 +16,7 @@ const ( ) // The expected type of function that should be provided to the ReadFilesFromStream func, that returns the writer that should handle each file -type FileHandlerFunc func(fileName string) (writer io.Writer, err error) +type FileHandlerFunc func(fileName string) (writer io.WriteCloser, err error) func ReadFilesFromStream(multipartReader *multipart.Reader, fileHandlerFunc FileHandlerFunc) error { for { @@ -36,6 +36,10 @@ func ReadFilesFromStream(multipartReader *multipart.Reader, fileHandlerFunc File if _, err = io.Copy(fileWriter, fileReader); err != nil { return fmt.Errorf("failed writing '%s' file: %w", fileName, err) } + err = fileWriter.Close() + if err != nil { + return err + } } return nil } diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index 905e125..96941a8 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -57,37 +57,39 @@ func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { assert.NoError(t, os.Remove(file2)) } -func simpleFileHandler(fileName string) (fileWriter io.Writer, err error) { - // #nosec G302 - return os.OpenFile(filepath.Join(targetDir, fileName), os.O_RDWR|os.O_CREATE, 0777) -} - -func fileHandlerWithHashValidation(fileName string) (fileWriter io.Writer, err error) { - fileWriter, err = simpleFileHandler(fileName) +func fileHandlerWithHashValidation(fileName string) (io.WriteCloser, error) { + fd, err := os.Create(filepath.Join(targetDir, fileName)) if err != nil { - return + return nil, err } // GetExpectedHashFromLockFile(fileName) lockFileMock := map[string]string{ "test1.txt": "070afab2066d3b16", "test2.txt": "48bc7295420af89d", } - return io.MultiWriter( - fileWriter, - &HashValidator{hash: xxh3.New(), actualChecksum: lockFileMock[fileName]}, - ), nil + return &WriteWrapper{fd: fd, hash: xxh3.New(), actualChecksum: lockFileMock[fileName]}, nil } -type HashValidator struct { +type WriteWrapper struct { + fd *os.File hash hash.Hash64 actualChecksum string } -func (hw *HashValidator) Write(p []byte) (n int, err error) { - n, err = hw.hash.Write(p) - sd := fmt.Sprintf("%x", hw.hash.Sum(nil)) - if sd != hw.actualChecksum { +func (ww *WriteWrapper) Write(p []byte) (n int, err error) { + // Write file + n, err = ww.Write(p) + if err != nil { + return + } + // Get checksum + n, err = ww.hash.Write(p) + sd := fmt.Sprintf("%x", ww.hash.Sum(nil)) + if sd != ww.actualChecksum { err = errors.New("checksum mismatch") } return } +func (ww *WriteWrapper) Close() error { + return ww.fd.Close() +} From 46a363c2b94d30f02e8f3c5b5c18374fd75added Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Mon, 11 Mar 2024 10:56:24 +0200 Subject: [PATCH 17/22] update deps --- http/filestream/filestream.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/http/filestream/filestream.go b/http/filestream/filestream.go index 50e9b90..9f81879 100644 --- a/http/filestream/filestream.go +++ b/http/filestream/filestream.go @@ -28,22 +28,28 @@ func ReadFilesFromStream(multipartReader *multipart.Reader, fileHandlerFunc File } return fmt.Errorf("failed to read file: %w", err) } - fileName := fileReader.FileName() - fileWriter, err := fileHandlerFunc(fileName) - if err != nil { - return err - } - if _, err = io.Copy(fileWriter, fileReader); err != nil { - return fmt.Errorf("failed writing '%s' file: %w", fileName, err) - } - err = fileWriter.Close() + err = readFile(fileReader, fileHandlerFunc) if err != nil { return err } + } return nil } +func readFile(fileReader *multipart.Part, fileHandlerFunc FileHandlerFunc) (err error) { + fileName := fileReader.FileName() + fileWriter, err := fileHandlerFunc(fileName) + if err != nil { + return err + } + defer ioutils.Close(fileWriter, &err) + if _, err = io.Copy(fileWriter, fileReader); err != nil { + return fmt.Errorf("failed writing '%s' file: %w", fileName, err) + } + return err +} + func WriteFilesToStream(responseWriter http.ResponseWriter, filePaths []string) (err error) { multipartWriter := multipart.NewWriter(responseWriter) responseWriter.Header().Set(contentType, multipartWriter.FormDataContentType()) From d7cdb159c62b9cdf1d086ead6d59d2fb5ad3db3d Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Mon, 11 Mar 2024 11:14:40 +0200 Subject: [PATCH 18/22] update deps --- http/filestream/filestream_test.go | 43 +++--------------------------- 1 file changed, 3 insertions(+), 40 deletions(-) diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index 96941a8..f021314 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -1,11 +1,7 @@ package filestream import ( - "errors" - "fmt" "github.com/stretchr/testify/assert" - "github.com/zeebo/xxh3" - "hash" "io" "mime/multipart" "net/http/httptest" @@ -38,7 +34,7 @@ func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { boundary := strings.Split(responseWriter.Header().Get(contentType), "boundary=")[1] // Create the multipart reader that will read the files from the stream multipartReader := multipart.NewReader(responseWriter.Body, boundary) - assert.NoError(t, ReadFilesFromStream(multipartReader, fileHandlerWithHashValidation)) + assert.NoError(t, ReadFilesFromStream(multipartReader, simpleFileHandler)) // Validate file 1 transferred successfully file1 = filepath.Join(targetDir, "test1.txt") @@ -57,39 +53,6 @@ func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { assert.NoError(t, os.Remove(file2)) } -func fileHandlerWithHashValidation(fileName string) (io.WriteCloser, error) { - fd, err := os.Create(filepath.Join(targetDir, fileName)) - if err != nil { - return nil, err - } - // GetExpectedHashFromLockFile(fileName) - lockFileMock := map[string]string{ - "test1.txt": "070afab2066d3b16", - "test2.txt": "48bc7295420af89d", - } - return &WriteWrapper{fd: fd, hash: xxh3.New(), actualChecksum: lockFileMock[fileName]}, nil -} - -type WriteWrapper struct { - fd *os.File - hash hash.Hash64 - actualChecksum string -} - -func (ww *WriteWrapper) Write(p []byte) (n int, err error) { - // Write file - n, err = ww.Write(p) - if err != nil { - return - } - // Get checksum - n, err = ww.hash.Write(p) - sd := fmt.Sprintf("%x", ww.hash.Sum(nil)) - if sd != ww.actualChecksum { - err = errors.New("checksum mismatch") - } - return -} -func (ww *WriteWrapper) Close() error { - return ww.fd.Close() +func simpleFileHandler(fileName string) (fileWriter io.WriteCloser, err error) { + return os.Create(filepath.Join(targetDir, fileName)) } From e42884edb1ce39ecce86b427e5bc9135a4a30a7c Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Mon, 11 Mar 2024 12:02:13 +0200 Subject: [PATCH 19/22] update deps --- http/filestream/filestream.go | 9 ++------- http/filestream/filestream_test.go | 12 +++++------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/http/filestream/filestream.go b/http/filestream/filestream.go index 9f81879..311fdfa 100644 --- a/http/filestream/filestream.go +++ b/http/filestream/filestream.go @@ -6,13 +6,11 @@ import ( ioutils "github.com/jfrog/gofrog/io" "io" "mime/multipart" - "net/http" "os" ) const ( - contentType = "Content-Type" - FileType = "file" + FileType = "file" ) // The expected type of function that should be provided to the ReadFilesFromStream func, that returns the writer that should handle each file @@ -50,10 +48,7 @@ func readFile(fileReader *multipart.Part, fileHandlerFunc FileHandlerFunc) (err return err } -func WriteFilesToStream(responseWriter http.ResponseWriter, filePaths []string) (err error) { - multipartWriter := multipart.NewWriter(responseWriter) - responseWriter.Header().Set(contentType, multipartWriter.FormDataContentType()) - +func WriteFilesToStream(multipartWriter *multipart.Writer, filePaths []string) (err error) { for _, filePath := range filePaths { if err = writeFile(multipartWriter, filePath); err != nil { return diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index f021314..cb15e01 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -1,13 +1,12 @@ package filestream import ( + "bytes" "github.com/stretchr/testify/assert" "io" "mime/multipart" - "net/http/httptest" "os" "path/filepath" - "strings" "testing" ) @@ -24,16 +23,15 @@ func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { assert.NoError(t, os.WriteFile(file2, file2Content, 0600)) // Create the multipart writer that will stream our files - responseWriter := httptest.NewRecorder() - assert.NoError(t, WriteFilesToStream(responseWriter, []string{file1, file2})) + body := &bytes.Buffer{} + multipartWriter := multipart.NewWriter(body) + assert.NoError(t, WriteFilesToStream(multipartWriter, []string{file1, file2})) // Create local temp dir that will store our files targetDir = t.TempDir() - // Get boundary hash from writer - boundary := strings.Split(responseWriter.Header().Get(contentType), "boundary=")[1] // Create the multipart reader that will read the files from the stream - multipartReader := multipart.NewReader(responseWriter.Body, boundary) + multipartReader := multipart.NewReader(body, multipartWriter.Boundary()) assert.NoError(t, ReadFilesFromStream(multipartReader, simpleFileHandler)) // Validate file 1 transferred successfully From ba8fe3a94c75244c95f761e8ed01555fdd7bd9ac Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Mon, 11 Mar 2024 12:24:20 +0200 Subject: [PATCH 20/22] update deps --- http/filestream/filestream.go | 29 ++++++++++++++--------------- http/filestream/filestream_test.go | 10 +++++++--- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/http/filestream/filestream.go b/http/filestream/filestream.go index 311fdfa..1cbf9e0 100644 --- a/http/filestream/filestream.go +++ b/http/filestream/filestream.go @@ -6,7 +6,6 @@ import ( ioutils "github.com/jfrog/gofrog/io" "io" "mime/multipart" - "os" ) const ( @@ -14,9 +13,9 @@ const ( ) // The expected type of function that should be provided to the ReadFilesFromStream func, that returns the writer that should handle each file -type FileHandlerFunc func(fileName string) (writer io.WriteCloser, err error) +type FileWriterFunc func(fileName string) (writer io.WriteCloser, err error) -func ReadFilesFromStream(multipartReader *multipart.Reader, fileHandlerFunc FileHandlerFunc) error { +func ReadFilesFromStream(multipartReader *multipart.Reader, fileWriterFunc FileWriterFunc) error { for { // Read the next file streamed from client fileReader, err := multipartReader.NextPart() @@ -26,7 +25,7 @@ func ReadFilesFromStream(multipartReader *multipart.Reader, fileHandlerFunc File } return fmt.Errorf("failed to read file: %w", err) } - err = readFile(fileReader, fileHandlerFunc) + err = readFile(fileReader, fileWriterFunc) if err != nil { return err } @@ -35,9 +34,9 @@ func ReadFilesFromStream(multipartReader *multipart.Reader, fileHandlerFunc File return nil } -func readFile(fileReader *multipart.Part, fileHandlerFunc FileHandlerFunc) (err error) { +func readFile(fileReader *multipart.Part, fileWriterFunc FileWriterFunc) (err error) { fileName := fileReader.FileName() - fileWriter, err := fileHandlerFunc(fileName) + fileWriter, err := fileWriterFunc(fileName) if err != nil { return err } @@ -48,9 +47,12 @@ func readFile(fileReader *multipart.Part, fileHandlerFunc FileHandlerFunc) (err return err } -func WriteFilesToStream(multipartWriter *multipart.Writer, filePaths []string) (err error) { - for _, filePath := range filePaths { - if err = writeFile(multipartWriter, filePath); err != nil { +// The expected type of function that should be provided to the WriteFilesToStream func, that returns the reader that should handle each file +type FileReaderFunc func(fileName string) (writer io.ReadCloser, err error) + +func WriteFilesToStream(multipartWriter *multipart.Writer, checksumsList []string, fileReaderFunc FileReaderFunc) (err error) { + for _, fileChecksum := range checksumsList { + if err = writeFile(multipartWriter, fileChecksum, fileReaderFunc); err != nil { return } } @@ -60,13 +62,10 @@ func WriteFilesToStream(multipartWriter *multipart.Writer, filePaths []string) ( return multipartWriter.Close() } -func writeFile(multipartWriter *multipart.Writer, filePath string) (err error) { - fileReader, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open file: %w", err) - } +func writeFile(multipartWriter *multipart.Writer, fileChecksum string, fileReaderFunc FileReaderFunc) (err error) { + fileReader, err := fileReaderFunc(fileChecksum) defer ioutils.Close(fileReader, &err) - fileWriter, err := multipartWriter.CreateFormFile(FileType, filePath) + fileWriter, err := multipartWriter.CreateFormFile(FileType, fileChecksum) if err != nil { return fmt.Errorf("failed to CreateFormFile: %w", err) } diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index cb15e01..4ba2390 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -25,14 +25,14 @@ func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { // Create the multipart writer that will stream our files body := &bytes.Buffer{} multipartWriter := multipart.NewWriter(body) - assert.NoError(t, WriteFilesToStream(multipartWriter, []string{file1, file2})) + assert.NoError(t, WriteFilesToStream(multipartWriter, []string{file1, file2}, simpleFileReader)) // Create local temp dir that will store our files targetDir = t.TempDir() // Create the multipart reader that will read the files from the stream multipartReader := multipart.NewReader(body, multipartWriter.Boundary()) - assert.NoError(t, ReadFilesFromStream(multipartReader, simpleFileHandler)) + assert.NoError(t, ReadFilesFromStream(multipartReader, simpleFileWriter)) // Validate file 1 transferred successfully file1 = filepath.Join(targetDir, "test1.txt") @@ -51,6 +51,10 @@ func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { assert.NoError(t, os.Remove(file2)) } -func simpleFileHandler(fileName string) (fileWriter io.WriteCloser, err error) { +func simpleFileReader(fileName string) (fileWriter io.ReadCloser, err error) { + return os.Open(filepath.Join(targetDir, fileName)) +} + +func simpleFileWriter(fileName string) (fileWriter io.WriteCloser, err error) { return os.Create(filepath.Join(targetDir, fileName)) } From b970d58e64ac487af18ea318e5218177f98ed633 Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Mon, 11 Mar 2024 15:11:20 +0200 Subject: [PATCH 21/22] update deps --- http/filestream/filestream.go | 18 ++++++++++++------ http/filestream/filestream_test.go | 24 +++++++----------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/http/filestream/filestream.go b/http/filestream/filestream.go index 1cbf9e0..89b6078 100644 --- a/http/filestream/filestream.go +++ b/http/filestream/filestream.go @@ -6,6 +6,7 @@ import ( ioutils "github.com/jfrog/gofrog/io" "io" "mime/multipart" + "os" ) const ( @@ -50,9 +51,14 @@ func readFile(fileReader *multipart.Part, fileWriterFunc FileWriterFunc) (err er // The expected type of function that should be provided to the WriteFilesToStream func, that returns the reader that should handle each file type FileReaderFunc func(fileName string) (writer io.ReadCloser, err error) -func WriteFilesToStream(multipartWriter *multipart.Writer, checksumsList []string, fileReaderFunc FileReaderFunc) (err error) { - for _, fileChecksum := range checksumsList { - if err = writeFile(multipartWriter, fileChecksum, fileReaderFunc); err != nil { +type FileInfo struct { + Name string + Path string +} + +func WriteFilesToStream(multipartWriter *multipart.Writer, filesList []FileInfo) (err error) { + for _, file := range filesList { + if err = writeFile(multipartWriter, file); err != nil { return } } @@ -62,10 +68,10 @@ func WriteFilesToStream(multipartWriter *multipart.Writer, checksumsList []strin return multipartWriter.Close() } -func writeFile(multipartWriter *multipart.Writer, fileChecksum string, fileReaderFunc FileReaderFunc) (err error) { - fileReader, err := fileReaderFunc(fileChecksum) +func writeFile(multipartWriter *multipart.Writer, file FileInfo) (err error) { + fileReader, err := os.Open(file.Path) defer ioutils.Close(fileReader, &err) - fileWriter, err := multipartWriter.CreateFormFile(FileType, fileChecksum) + fileWriter, err := multipartWriter.CreateFormFile(FileType, file.Name) if err != nil { return fmt.Errorf("failed to CreateFormFile: %w", err) } diff --git a/http/filestream/filestream_test.go b/http/filestream/filestream_test.go index 4ba2390..3e10a7a 100644 --- a/http/filestream/filestream_test.go +++ b/http/filestream/filestream_test.go @@ -15,17 +15,17 @@ var targetDir string func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { sourceDir := t.TempDir() // Create 2 file to be transferred via our multipart stream - file1 := filepath.Join(sourceDir, "test1.txt") - file2 := filepath.Join(sourceDir, "test2.txt") + file1 := FileInfo{Name: "test1.txt", Path: filepath.Join(sourceDir, "test1.txt")} + file2 := FileInfo{Name: "test2.txt", Path: filepath.Join(sourceDir, "test2.txt")} file1Content := []byte("test content1") file2Content := []byte("test content2") - assert.NoError(t, os.WriteFile(file1, file1Content, 0600)) - assert.NoError(t, os.WriteFile(file2, file2Content, 0600)) + assert.NoError(t, os.WriteFile(file1.Path, file1Content, 0600)) + assert.NoError(t, os.WriteFile(file2.Path, file2Content, 0600)) // Create the multipart writer that will stream our files body := &bytes.Buffer{} multipartWriter := multipart.NewWriter(body) - assert.NoError(t, WriteFilesToStream(multipartWriter, []string{file1, file2}, simpleFileReader)) + assert.NoError(t, WriteFilesToStream(multipartWriter, []FileInfo{file1, file2})) // Create local temp dir that will store our files targetDir = t.TempDir() @@ -35,24 +35,14 @@ func TestWriteFilesToStreamAndReadFilesFromStream(t *testing.T) { assert.NoError(t, ReadFilesFromStream(multipartReader, simpleFileWriter)) // Validate file 1 transferred successfully - file1 = filepath.Join(targetDir, "test1.txt") - assert.FileExists(t, file1) - content, err := os.ReadFile(file1) + content, err := os.ReadFile(filepath.Join(targetDir, file1.Name)) assert.NoError(t, err) assert.Equal(t, file1Content, content) - assert.NoError(t, os.Remove(file1)) // Validate file 2 transferred successfully - file2 = filepath.Join(targetDir, "test2.txt") - assert.FileExists(t, file2) - content, err = os.ReadFile(file2) + content, err = os.ReadFile(filepath.Join(targetDir, file2.Name)) assert.NoError(t, err) assert.Equal(t, file2Content, content) - assert.NoError(t, os.Remove(file2)) -} - -func simpleFileReader(fileName string) (fileWriter io.ReadCloser, err error) { - return os.Open(filepath.Join(targetDir, fileName)) } func simpleFileWriter(fileName string) (fileWriter io.WriteCloser, err error) { From 8c8a0db4f4185291c5657238b8d3858c1b1b39cd Mon Sep 17 00:00:00 2001 From: Michael Sverdlov Date: Mon, 11 Mar 2024 16:02:25 +0200 Subject: [PATCH 22/22] update deps --- http/filestream/filestream.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/http/filestream/filestream.go b/http/filestream/filestream.go index 89b6078..dcaf579 100644 --- a/http/filestream/filestream.go +++ b/http/filestream/filestream.go @@ -48,9 +48,6 @@ func readFile(fileReader *multipart.Part, fileWriterFunc FileWriterFunc) (err er return err } -// The expected type of function that should be provided to the WriteFilesToStream func, that returns the reader that should handle each file -type FileReaderFunc func(fileName string) (writer io.ReadCloser, err error) - type FileInfo struct { Name string Path string