Generic Multipart reader/writer implementaion

go.mod

@@ -3,6 +3,7 @@ module github.com/jfrog/gofrog
 go 1.20
 
 require (
+	github.com/cespare/xxhash v1.1.0
 	github.com/jfrog/archiver/v3 v3.6.0
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.8.4

go.sum

@@ -1,5 +1,9 @@
+github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
@@ -23,6 +27,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=

http/multipart/multipart.go

@@ -0,0 +1,93 @@
+package multipart
+
+import (
+	"errors"
+	"fmt"
+	"github.com/cespare/xxhash"
+	ioutils "github.com/jfrog/gofrog/io"
+	"hash"
+	"io"
+	"mime/multipart"
+	"os"
+)
+
+type FileWriterFunc func(fileName string) (io.Writer, error)
+
+func ReadFilesFromStream(multipartReader *multipart.Reader, fileWriterFunc FileWriterFunc) error {
+	for {
+		// Read the next file streamed from client
+		fileReader, err := multipartReader.NextPart()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return fmt.Errorf("failed to read file: %w", err)
+		}
+		fileName := fileReader.FileName()
+		fileWriter, err := fileWriterFunc(fileName)
+		if err != nil {
+			return err
+		}
+
+		if _, err = io.Copy(fileWriter, fileReader); err != nil {
+			return fmt.Errorf("failed writing '%s' file: %w", fileName, err)
+		}
+	}
+	return nil
+}
+
+func WriteFilesToStream(writer io.Writer, filesList []string) error {
+	multipartWriter := multipart.NewWriter(writer)
+	for _, filePath := range filesList {
+		fileReader, err := os.Open(filePath)
+		if err != nil {
+			return fmt.Errorf("failed to open file: %w", err)
+		}
+		defer ioutils.Close(fileReader, &err)
+		fileWriter, err := multipartWriter.CreateFormFile("fieldname", filePath)
+		if err != nil {
+			return fmt.Errorf("failed to CreateFormFile: %w", err)
+		}
+		_, err = io.Copy(fileWriter, fileReader)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func getFileWriter(fileName string) (fileWriter io.Writer, err error) {
+	lockfileMock := map[string]string{
+		"SDFDSFSDFSDFDSF":   "file1",
+		"XXSDFDSFSDFSDFDSF": "file2",
+	}
+	realFileName := lockfileMock[fileName]
+	fileWriter, err = os.Create(realFileName)
+	if err != nil {
+		return nil, fmt.Errorf("create file: %w", err)
+	}
+	// Currently we are using the file name as the hash
+	fileHash := fileName
+	return io.MultiWriter(fileWriter, NewHashWrapper(fileHash)), nil
+}
+
+type HashWrapper struct {
+	hash           hash.Hash64
+	actualChecksum string
+}
+
+func NewHashWrapper(actualChecksum string) *HashWrapper {
+	return &HashWrapper{hash: xxhash.New(), actualChecksum: actualChecksum}
+}
+
+func (hw *HashWrapper) Write(p []byte) (n int, err error) {
+	n, err = hw.hash.Write(p)
+	if fmt.Sprintf("%x", hw.hash.Sum(nil)) != hw.actualChecksum {
+		err = errors.New("checksum mismatch")
+	}
+	return
+}
+
+func myMain(filepath string) {
+	ReadFilesFromStream(nil, getFileWriter)
+}