From 92eaf88c36b8aaf77384f1ad8d38d96a8b28ed86 Mon Sep 17 00:00:00 2001
From: Son Roy Almerol <github@snry.me>
Date: Thu, 14 Nov 2024 20:41:01 -0500
Subject: [PATCH] try to trigger file list ahead of time

---
 internal/agent/sftp/filelister.go | 38 ++++++++++++++++++++++++++++---
 internal/agent/sftp/filter.go     |  2 +-
 internal/backend/mount/mount.go   |  5 ++++
 3 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/internal/agent/sftp/filelister.go b/internal/agent/sftp/filelister.go
index c950f60e..1e134df2 100644
--- a/internal/agent/sftp/filelister.go
+++ b/internal/agent/sftp/filelister.go
@@ -9,17 +9,45 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/pkg/sftp"
 )
 
+var cacheMutex sync.RWMutex
+var sizeCache map[string]map[string]int64
+var cacheOnce sync.Once
+
 // CustomFileInfo wraps an os.FileInfo and overrides the Size method to return -1.
 type CustomFileInfo struct {
 	os.FileInfo
-	filePath string
+	filePath   string
+	snapshotId string
+}
+
+func initializeSizeCache() {
+	cacheMutex.Lock()
+	defer cacheMutex.Unlock()
+	if sizeCache == nil {
+		sizeCache = make(map[string]map[string]int64)
+	}
 }
 
 func (f *CustomFileInfo) Size() int64 {
+	cacheOnce.Do(initializeSizeCache)
+
+	cacheMutex.RLock()
+	if _, ok := sizeCache[f.filePath]; ok {
+		if cachedSize, ok := sizeCache[f.filePath][f.snapshotId]; ok {
+			return cachedSize
+		}
+	}
+	cacheMutex.RUnlock()
+
+	cacheMutex.Lock()
+	sizeCache[f.filePath] = make(map[string]int64)
+	cacheMutex.Unlock()
+
 	file, err := os.Open(f.filePath)
 	if err != nil {
 		return 0
@@ -31,6 +59,10 @@ func (f *CustomFileInfo) Size() int64 {
 		return 0
 	}
 
+	cacheMutex.Lock()
+	sizeCache[f.filePath][f.snapshotId] = byteCount
+	cacheMutex.Unlock()
+
 	return byteCount
 }
 
@@ -72,7 +104,7 @@ func (h *SftpHandler) FileLister(dirPath string) (*FileLister, error) {
 				continue
 			}
 			// Wrap the original os.FileInfo to override its Size method.
-			fileInfos = append(fileInfos, &CustomFileInfo{FileInfo: info, filePath: fullPath})
+			fileInfos = append(fileInfos, &CustomFileInfo{FileInfo: info, filePath: fullPath, snapshotId: h.Snapshot.Id})
 		}
 	}
 
@@ -97,7 +129,7 @@ func (h *SftpHandler) FileStat(filename string) (*FileLister, error) {
 	}
 
 	// Wrap the original os.FileInfo to override its Size method.
-	return &FileLister{files: []os.FileInfo{&CustomFileInfo{FileInfo: stat, filePath: filename}}}, nil
+	return &FileLister{files: []os.FileInfo{&CustomFileInfo{FileInfo: stat, filePath: filename, snapshotId: h.Snapshot.Id}}}, nil
 }
 
 func (h *SftpHandler) setFilePath(r *sftp.Request) {
diff --git a/internal/agent/sftp/filter.go b/internal/agent/sftp/filter.go
index 959897ac..465b0d96 100644
--- a/internal/agent/sftp/filter.go
+++ b/internal/agent/sftp/filter.go
@@ -86,7 +86,7 @@ func compileExcludedPaths() []*regexp.Regexp {
 		`AppData\Local\Microsoft\Windows\WebCache`,
 		`AppData\Local\Microsoft\Windows Store`,
 		`AppData\Local\Packages`,
-		`AppData\Local\Veritas\DLO`,
+		`AppData\Local\Veritas`,
 		`Application Data\Apple Computer\Mobile Sync`,
 		`Application Data\Application Data**`,
 		`Dropbox\Dropbox.exe.log`,
diff --git a/internal/backend/mount/mount.go b/internal/backend/mount/mount.go
index 69bde063..7474a3ee 100644
--- a/internal/backend/mount/mount.go
+++ b/internal/backend/mount/mount.go
@@ -80,6 +80,11 @@ func Mount(target *store.Target) (*AgentMount, error) {
 		return nil, fmt.Errorf("Mount: error starting rclone for sftp -> %w", err)
 	}
 
+	ls := exec.Command("bash", "-c", fmt.Sprintf("find %s -type f | xargs -P 8 -n 1 true", agentMount.Path))
+	ls.Env = os.Environ()
+
+	_ = ls.Start()
+
 	return agentMount, nil
 }