Merge pull request #2 from picosh/am/rsync-updates-2025

Started work on a number of rsync updates

go.mod

@@ -1,6 +1,12 @@
 module github.com/picosh/pobj
 
-go 1.22
+go 1.24
+
+toolchain go1.24.0
+
+replace github.com/picosh/send => ../send
+
+replace github.com/picosh/go-rsync-receiver => ../go-rsync-receiver
 
 require (
 	github.com/aws/aws-sdk-go-v2 v1.30.4
@@ -17,7 +23,6 @@ require (
 )
 
 require (
-	github.com/DavidGamba/go-getoptions v0.29.0 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect
@@ -41,7 +46,6 @@ require (
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.2 // indirect
@@ -61,7 +65,7 @@ require (
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.15.2 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
-	github.com/picosh/go-rsync-receiver v0.0.0-20240709135253-1daf4b12a9fc // indirect
+	github.com/picosh/go-rsync-receiver v0.0.0-20250213155332-9a9558ff1929 // indirect
 	github.com/pkg/sftp v1.13.6 // indirect
 	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
@@ -82,7 +86,7 @@ require (
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 // indirect
 	golang.org/x/net v0.18.0 // indirect
-	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/sync v0.11.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect

go.sum

@@ -1,5 +1,3 @@
-github.com/DavidGamba/go-getoptions v0.29.0 h1:cU8MjOyfAyPZke4hrgEuiGBJHS9PFYPAHve2fhDhdDk=
-github.com/DavidGamba/go-getoptions v0.29.0/go.mod h1:zE97E3PR9P3BI/HKyNYgdMlYxodcuiC6W68KIgeYT84=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8=
@@ -74,8 +72,6 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
-github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
 github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -124,12 +120,6 @@ github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo
 github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8=
 github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
 github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
-github.com/picosh/go-rsync-receiver v0.0.0-20240709135253-1daf4b12a9fc h1:bvcsoOvaNHPquFnRkdraEo7+8t6bW7nWEhlALnwZPdI=
-github.com/picosh/go-rsync-receiver v0.0.0-20240709135253-1daf4b12a9fc/go.mod h1:i0iR3W4GSm1PuvVxB9OH32E5jP+CYkVb2NQSe0JCtlo=
-github.com/picosh/send v0.0.0-20240709135413-81b1dd703a11 h1:ElTyYxJPPfAz+NLRL9qy+SQ9UYaSGlWXWl2yYrLPzCU=
-github.com/picosh/send v0.0.0-20240709135413-81b1dd703a11/go.mod h1:RAgLDK3LrDK6pNeXtU9tjo28obl5DxShcTUk2nm/KCM=
-github.com/picosh/send v0.0.0-20241008013240-6fdbff00f848 h1:VWbjNNOqpJ8AB3zdw+M5+XC/SINooWLGi6WCozKwt1o=
-github.com/picosh/send v0.0.0-20241008013240-6fdbff00f848/go.mod h1:RAgLDK3LrDK6pNeXtU9tjo28obl5DxShcTUk2nm/KCM=
 github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo=
 github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -203,8 +193,8 @@ golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

handler.go

@@ -80,7 +80,7 @@ func (h *UploadAssetHandler) Delete(s ssh.Session, entry *utils.FileEntry) error
 	return h.Cfg.Storage.DeleteObject(bucket, objectFileName)
 }
 
-func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) {
+func (h *UploadAssetHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReadAndReaderAtCloser, error) {
 	fileInfo := &utils.VirtualFile{
 		FName:    filepath.Base(entry.Filepath),
 		FIsDir:   false,
@@ -229,7 +229,7 @@ func (h *UploadAssetHandler) Write(s ssh.Session, entry *utils.FileEntry) (strin
 	return url, nil
 }
 
-func (h *UploadAssetHandler) validateAsset(data *FileData) (bool, error) {
+func (h *UploadAssetHandler) validateAsset(_ *FileData) (bool, error) {
 	return true, nil
 }
 
@@ -258,7 +258,7 @@ func (h *UploadAssetHandler) writeAsset(s ssh.Session, data *FileData) error {
 	_, _, err = h.Cfg.Storage.PutObject(
 		data.Bucket,
 		objectFileName,
-		utils.NopReaderAtCloser(reader),
+		utils.NopReadAndReaderAtCloser(reader),
 		data.FileEntry,
 	)
 	if err != nil {

reader.go

@@ -10,10 +10,10 @@ import (
 )
 
 type AllReaderAt struct {
-	Reader utils.ReaderAtCloser
+	Reader utils.ReadAndReaderAtCloser
 }
 
-func NewAllReaderAt(reader utils.ReaderAtCloser) *AllReaderAt {
+func NewAllReaderAt(reader utils.ReadAndReaderAtCloser) *AllReaderAt {
 	return &AllReaderAt{reader}
 }
 

storage/fs.go

@@ -93,7 +93,7 @@ func (s *StorageFS) DeleteBucket(bucket Bucket) error {
 	return os.RemoveAll(bucket.Path)
 }
 
-func (s *StorageFS) GetObject(bucket Bucket, fpath string) (utils.ReaderAtCloser, *ObjectInfo, error) {
+func (s *StorageFS) GetObject(bucket Bucket, fpath string) (utils.ReadAndReaderAtCloser, *ObjectInfo, error) {
 	objInfo := &ObjectInfo{
 		LastModified: time.Time{},
 		Metadata:     nil,

storage/memory.go

@@ -52,7 +52,7 @@ func (s *StorageMemory) GetBucketQuota(bucket Bucket) (uint64, error) {
 	objects := s.storage[bucket.Path]
 	size := 0
 	for _, val := range objects {
-		size += len(val)
+		size += len([]byte(val))
 	}
 	return uint64(size), nil
 }
@@ -62,7 +62,11 @@ func (s *StorageMemory) DeleteBucket(bucket Bucket) error {
 	return nil
 }
 
-func (s *StorageMemory) GetObject(bucket Bucket, fpath string) (utils.ReaderAtCloser, *ObjectInfo, error) {
+func (s *StorageMemory) GetObject(bucket Bucket, fpath string) (utils.ReadAndReaderAtCloser, *ObjectInfo, error) {
+	if !strings.HasPrefix(fpath, "/") {
+		fpath = "/" + fpath
+	}
+
 	objInfo := &ObjectInfo{
 		LastModified: time.Time{},
 		Metadata:     nil,
@@ -74,19 +78,19 @@ func (s *StorageMemory) GetObject(bucket Bucket, fpath string) (utils.ReaderAtCl
 		return nil, objInfo, fmt.Errorf("object does not exist: %s", fpath)
 	}
 
-	objInfo.Size = int64(len(dat))
-	reader := utils.NopReaderAtCloser(strings.NewReader(dat))
+	objInfo.Size = int64(len([]byte(dat)))
+	reader := utils.NopReadAndReaderAtCloser(strings.NewReader(dat))
 	return reader, objInfo, nil
 }
 
 func (s *StorageMemory) PutObject(bucket Bucket, fpath string, contents io.Reader, entry *utils.FileEntry) (string, int64, error) {
-	buf := new(strings.Builder)
-	size, err := io.Copy(buf, contents)
+	d, err := io.ReadAll(contents)
 	if err != nil {
 		return "", 0, err
 	}
-	s.storage[bucket.Path][fpath] = buf.String()
-	return fmt.Sprintf("%s%s", bucket.Path, fpath), size, nil
+
+	s.storage[bucket.Path][fpath] = string(d)
+	return fmt.Sprintf("%s%s", bucket.Path, fpath), int64(len(d)), nil
 }
 
 func (s *StorageMemory) DeleteObject(bucket Bucket, fpath string) error {
@@ -104,16 +108,21 @@ func (s *StorageMemory) ListBuckets() ([]string, error) {
 
 func (s *StorageMemory) ListObjects(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error) {
 	var fileList []os.FileInfo
+
 	resolved := dir
 
+	if !strings.HasPrefix(resolved, "/") {
+		resolved = "/" + resolved
+	}
+
 	objects := s.storage[bucket.Path]
 	// dir is actually an object
 	oval, ok := objects[resolved]
 	if ok {
 		fileList = append(fileList, &utils.VirtualFile{
 			FName:    filepath.Base(resolved),
 			FIsDir:   false,
-			FSize:    int64(len(oval)),
+			FSize:    int64(len([]byte(oval))),
 			FModTime: time.Time{},
 		})
 		return fileList, nil
@@ -153,14 +162,14 @@ func (s *StorageMemory) ListObjects(bucket Bucket, dir string, recursive bool) (
 			fileList = append(fileList, &utils.VirtualFile{
 				FName:    fname,
 				FIsDir:   false,
-				FSize:    int64(len(val)),
+				FSize:    int64(len([]byte(val))),
 				FModTime: time.Time{},
 			})
 		} else if resolved == dirKey || trimRes == dirKey {
 			fileList = append(fileList, &utils.VirtualFile{
 				FName:    fname,
 				FIsDir:   false,
-				FSize:    int64(len(val)),
+				FSize:    int64(len([]byte(val))),
 				FModTime: time.Time{},
 			})
 		}

storage/minio.go

@@ -152,7 +152,7 @@ func (s *StorageMinio) DeleteBucket(bucket Bucket) error {
 	return s.Client.RemoveBucket(context.TODO(), bucket.Name)
 }
 
-func (s *StorageMinio) GetObject(bucket Bucket, fpath string) (utils.ReaderAtCloser, *ObjectInfo, error) {
+func (s *StorageMinio) GetObject(bucket Bucket, fpath string) (utils.ReadAndReaderAtCloser, *ObjectInfo, error) {
 	objInfo := &ObjectInfo{
 		Size:         0,
 		LastModified: time.Time{},

storage/s3.go

@@ -207,7 +207,7 @@ func (s *StorageS3) DeleteBucket(bucket Bucket) error {
 	return err
 }
 
-func (s *StorageS3) GetObject(bucket Bucket, fpath string) (utils.ReaderAtCloser, *ObjectInfo, error) {
+func (s *StorageS3) GetObject(bucket Bucket, fpath string) (utils.ReadAndReaderAtCloser, *ObjectInfo, error) {
 	input := &s3.GetObjectInput{
 		Bucket: aws.String(bucket.Name),
 		Key:    aws.String(fpath),
@@ -239,7 +239,7 @@ func (s *StorageS3) GetObject(bucket Bucket, fpath string) (utils.ReaderAtCloser
 
 	// Create a bytes.Reader which implements io.ReaderAt
 	body := bytes.NewReader(data)
-	content := utils.NopReaderAtCloser(body)
+	content := utils.NopReadAndReaderAtCloser(body)
 
 	return content, objInfo, nil
 }

storage/storage.go

@@ -22,7 +22,7 @@ type ObjectStorage interface {
 	ListBuckets() ([]string, error)
 	DeleteBucket(bucket Bucket) error
 
-	GetObject(bucket Bucket, fpath string) (utils.ReaderAtCloser, *ObjectInfo, error)
+	GetObject(bucket Bucket, fpath string) (utils.ReadAndReaderAtCloser, *ObjectInfo, error)
 	PutObject(bucket Bucket, fpath string, contents io.Reader, entry *utils.FileEntry) (string, int64, error)
 	DeleteObject(bucket Bucket, fpath string) error
 	ListObjects(bucket Bucket, dir string, recursive bool) ([]os.FileInfo, error)