feat: comparedirs multithread (sander-skjulsvik#19)

Added compare dir with support for multithread. And refactored to mostly implement a strategy pattern

* fixed test status badge

* Selecting run method with passing runfunc around. This is  a start of a strategy pattern

* Implemented strategi pattern for  dupes (I think)

* Implemented strategi pattern for  compareDirs

* Moved func selection to cli.go

* runComparison now actually uses compFunc method, and we are adding paths to comparator struct

* Selecting the correct compare dirs functions for testing

* Added some error handling for cli unput

* moved is file to files, and added func AddBarDirSize

---------

Co-authored-by: sander skjulsvik <[email protected]>

MAKEFILE

@@ -4,6 +4,8 @@ actions:
 win-build:
 	go build -o bin\\ .\\...
 
+win-test:
+	go test .\\...
 
 win-deps:
 	choco install golangci-lint act-cli

README.md

@@ -3,5 +3,4 @@
 
 ## Statuses
 
-
-[!Test Status](https://github.com/sander-skjulsvik/tools/actions/workflows/test.yml/badge.svg)
+![Test Status](https://github.com/sander-skjulsvik/tools/actions/workflows/test.yml/badge.svg)

dupes/lib/common/main.go → dupes/lib/common/files.go

@@ -8,16 +8,8 @@ import (
 	"fmt"
 	"io"
 	"os"
-
-	"github.com/sander-skjulsvik/tools/libs/progressbar"
 )
 
-// Run is the main function to run for consumers of this lib.
-// First arg is the path to the folder,
-type Run func(string) *Dupes
-
-type RunWithProgressBar func(string, *progressbar.ProgressBar) *Dupes
-
 type File struct {
 	Path string
 	Hash string
@@ -27,13 +19,6 @@ func HashString(b []byte) string {
 	return hex.EncodeToString(b)
 }
 
-func IsFile(f os.FileInfo) bool {
-	if f == nil {
-		panic(fmt.Errorf("file info is nil"))
-	}
-	return f.Mode().IsRegular()
-}
-
 func HashFile(path string) (string, error) {
 	f, err := os.Open(path)
 	if err != nil {

dupes/lib/common/run.go

@@ -0,0 +1,24 @@
+package common
+
+import "github.com/sander-skjulsvik/tools/libs/progressbar"
+
+// Run is the main function to run for consumers of this lib.
+// First arg is the path to the folder,
+type Run func(string, progressbar.ProgressBar) *Dupes
+
+type Runner struct {
+	RunFunc     Run
+	ProgressBar progressbar.ProgressBar
+	OutputJson  bool
+}
+
+func NewRunner(runFunc Run, bar progressbar.ProgressBar) *Runner {
+	return &Runner{
+		RunFunc:     runFunc,
+		ProgressBar: bar,
+	}
+}
+
+func (r *Runner) Run(path string) *Dupes {
+	return r.RunFunc(path, r.ProgressBar)
+}

dupes/lib/producerConsumer/main.go

@@ -7,9 +7,10 @@ import (
 	"sync"
 
 	"github.com/sander-skjulsvik/tools/dupes/lib/common"
+	"github.com/sander-skjulsvik/tools/libs/progressbar"
 )
 
-// Works like a generator, yelding all regular files
+// Works like a generator, yielding all regular files
 func getFiles(root string, filePaths chan<- string) {
 	filepath.Walk(root, func(path string, info fs.FileInfo, err error) error {
 		if err != nil {
@@ -37,25 +38,24 @@ func appendFileTreadSafe(dupes *common.Dupes, path string, lock *sync.Mutex) {
 	// dupes.ProgressBar.Add1()
 }
 
-func ProcessFiles(filePaths <-chan string) *common.Dupes {
+func ProcessFiles(filePaths <-chan string, bar progressbar.ProgressBar) *common.Dupes {
 	dupes := common.NewDupes()
 	wg := sync.WaitGroup{}
 	dupesWl := sync.Mutex{}
-	// if chans.IsClosed(filePaths) {
-	// 	log.Fatalln("Chan closed before managed to access it 1")
-	// }
+
 	for filePath := range filePaths {
 		wg.Add(1)
 		go func(fp string) {
 			appendFileTreadSafe(&dupes, fp, &dupesWl)
+			bar.AddFileSize(filePath)
 			wg.Done()
 		}(filePath)
 	}
 	wg.Wait()
 	return &dupes
 }
 
-func ProcessFilesNCunsumers(filePaths <-chan string, numberOfConsumers int, doneWg *sync.WaitGroup) *common.Dupes {
+func ProcessFilesNConsumers(filePaths <-chan string, numberOfConsumers int, bar progressbar.ProgressBar) *common.Dupes {
 	dupes := common.NewDupes()
 	wg := sync.WaitGroup{}
 	dupesWl := sync.Mutex{}
@@ -64,20 +64,26 @@ func ProcessFilesNCunsumers(filePaths <-chan string, numberOfConsumers int, done
 		go func() {
 			for filePath := range filePaths {
 				appendFileTreadSafe(&dupes, filePath, &dupesWl)
+				bar.AddFileSize(filePath)
 			}
 			wg.Done()
 		}()
 	}
 	wg.Wait()
-	doneWg.Done()
 	return &dupes
 }
 
-func Run(path string) *common.Dupes {
+func Run(path string, bar progressbar.ProgressBar) *common.Dupes {
 	filePaths := make(chan string)
 	go getFiles(path, filePaths)
-	// sleep 10 seconds
-	dupes := ProcessFiles(filePaths)
-	// storer(files)
+	dupes := ProcessFiles(filePaths, bar)
 	return dupes
 }
+
+func GetRunNThreads(n int) common.Run {
+	return func(path string, bar progressbar.ProgressBar) *common.Dupes {
+		filePaths := make(chan string)
+		go getFiles(path, filePaths)
+		return ProcessFilesNConsumers(filePaths, n, bar)
+	}
+}

dupes/lib/producerConsumer/main_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/sander-skjulsvik/tools/dupes/lib/common"
 	"github.com/sander-skjulsvik/tools/dupes/lib/test"
+	"github.com/sander-skjulsvik/tools/libs/progressbar"
 	"gotest.tools/assert"
 )
 
@@ -275,7 +276,7 @@ func TestProcessFiles(t *testing.T) {
 		var d *common.Dupes
 		wg.Add(1)
 		go func() {
-			d = ProcessFiles(filePaths)
+			d = ProcessFiles(filePaths, progressbar.ProgressBarMoc{})
 			wg.Done()
 		}()
 		filePaths <- filepath.Clean(path)
@@ -331,7 +332,7 @@ func TestProcessFiles(t *testing.T) {
 		var d *common.Dupes
 		wg.Add(1)
 		go func() {
-			d = ProcessFiles(filePaths)
+			d = ProcessFiles(filePaths, progressbar.ProgressBarMoc{})
 			wg.Done()
 		}()
 		wgAdd := sync.WaitGroup{}
@@ -392,7 +393,7 @@ func TestProcessFilesNConsumers(t *testing.T) {
 		doneWg := sync.WaitGroup{}
 		doneWg.Add(1)
 		go func() {
-			d = ProcessFilesNCunsumers(filePaths, 3, &doneWg)
+			d = ProcessFilesNConsumers(filePaths, 3, progressbar.ProgressBarMoc{})
 			wg.Done()
 		}()
 		filePaths <- filepath.Clean(path)
@@ -450,7 +451,7 @@ func TestProcessFilesNConsumers(t *testing.T) {
 		doneWg := sync.WaitGroup{}
 		doneWg.Add(1)
 		go func() {
-			d = ProcessFilesNCunsumers(filePaths, 3, &doneWg)
+			d = ProcessFilesNConsumers(filePaths, 3, progressbar.ProgressBarMoc{})
 			wg.Done()
 		}()
 		wgAdd := sync.WaitGroup{}

dupes/lib/singleThread/main.go

@@ -6,42 +6,17 @@ import (
 	"path/filepath"
 
 	"github.com/sander-skjulsvik/tools/dupes/lib/common"
+	"github.com/sander-skjulsvik/tools/libs/files"
 	"github.com/sander-skjulsvik/tools/libs/progressbar"
 )
 
-func Run(src string) *common.Dupes {
+func Run(src string, bar progressbar.ProgressBar) *common.Dupes {
 	dupes := &common.Dupes{
 		D: map[string]*common.Dupe{},
-		// ProgressBar: common.NewSchollzProgressbar(),
 	}
 
 	err := filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
-		isFile := common.IsFile(info)
-		if !isFile {
-			return nil
-		}
-
-		dupes, err = dupes.Append(path)
-		if err != nil {
-			return nil
-		}
-
-		return nil
-	})
-	if err != nil {
-		log.Fatalf("Failed to walk src: %s, with err: %s", src, err.Error())
-	}
-	return dupes
-}
-
-func RunWithProgressBar(src string, bar progressbar.ProgressBar) *common.Dupes {
-	dupes := &common.Dupes{
-		D: map[string]*common.Dupe{},
-		// ProgressBar: common.NewSchollzProgressbar(),
-	}
-
-	err := filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
-		isFile := common.IsFile(info)
+		isFile := files.IsFile(info)
 		if !isFile {
 			return nil
 		}

dupes/lib/test/run.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/sander-skjulsvik/tools/dupes/lib/common"
 	"github.com/sander-skjulsvik/tools/libs/collections"
+	"github.com/sander-skjulsvik/tools/libs/progressbar"
 )
 
 func TestRun(path string, run common.Run, t *testing.T) {
@@ -17,7 +18,7 @@ func TestRun(path string, run common.Run, t *testing.T) {
 	// Setup the expected dupes
 	SetupExpectedDupes(path)
 	// Run the run function to find the dupes
-	calculatedDupes := run(path)
+	calculatedDupes := run(path, progressbar.ProgressBarMoc{})
 	// Check if the expected dupes are found
 	CheckExpectedDupes(GetExpectedDupes(path), *calculatedDupes, t)
 }
@@ -65,7 +66,7 @@ func TestRunManyFiles(path string, run common.Run, t *testing.T) {
 	GenerateNestedStructure(baseDir, numLevels, numFoldersPerLevel, numFilesPerFolder, content)
 
 	fmt.Println("Nested folder structure generated successfully.")
-	run(baseDir)
+	run(baseDir, progressbar.ProgressBarMoc{})
 	fmt.Printf("Done running! \n")
 }
 

dupes/main.go

@@ -9,18 +9,25 @@ import (
 	"github.com/sander-skjulsvik/tools/dupes/lib/common"
 	producerConsumer "github.com/sander-skjulsvik/tools/dupes/lib/producerConsumer"
 	singleThread "github.com/sander-skjulsvik/tools/dupes/lib/singleThread"
+	"github.com/sander-skjulsvik/tools/libs/progressbar"
 )
 
 func main() {
 	var (
 		method           string
 		path             string
 		presentOnlyDupes bool
+		useProgressBar   bool
+		presentJson      bool
+		nThreads         int
 	)
 
-	flag.StringVar(&method, "method", "single", "Method (single or producerConsumer)")
+	flag.StringVar(&method, "method", "single", "Method (single, producerConsumer or nThreads)")
+	flag.IntVar(&nThreads, "nThreads", 0, "Number of threads to use, ignored unless nThreads method is chosen")
 	flag.StringVar(&path, "path", ".", "File path")
 	flag.BoolVar(&presentOnlyDupes, "onlyDupes", true, "Only present dupes")
+	flag.BoolVar(&presentJson, "json", false, "present json")
+	flag.BoolVar(&useProgressBar, "progressBar", false, "Present a progress bar?")
 
 	// Parse the command-line arguments
 	flag.Parse()
@@ -39,16 +46,45 @@ func main() {
 	fmt.Printf("Path: %s\n", path)
 	fmt.Printf("PresentOnlyDupes: %t\n", presentOnlyDupes)
 
-	Run(path, method, presentOnlyDupes)
-}
-
-func Run(path, method string, presentOnlyDupes bool) {
-	var dupes *common.Dupes
+	var runFunc common.Run
 	switch {
 	case method == "single":
-		dupes = singleThread.Run(path)
-	case method == "producerconsumer":
-		dupes = producerConsumer.Run(path)
+		runFunc = singleThread.Run
+	case method == "producerConsumer":
+		runFunc = producerConsumer.Run
+	case method == "nThreads":
+		runFunc = producerConsumer.GetRunNThreads(nThreads)
+	}
+
+	var bar progressbar.ProgressBar
+	switch useProgressBar {
+	case true:
+		bar = progressbar.UiProgressBar{}
+	}
+
+	dupes := NewRunner(runFunc, bar).Run(path)
+	switch presentOnlyDupes {
+	case true:
+		dupes.GetOnlyDupes().Present(presentJson)
+	case false:
+		dupes.Present(presentJson)
+	}
+
+}
+
+type Runner struct {
+	RunFunc     common.Run
+	ProgressBar progressbar.ProgressBar
+	OutputJson  bool
+}
+
+func NewRunner(runFunc common.Run, bar progressbar.ProgressBar) *Runner {
+	return &Runner{
+		RunFunc:     runFunc,
+		ProgressBar: bar,
 	}
-	dupes.Present(presentOnlyDupes)
+}
+
+func (r *Runner) Run(path string) *common.Dupes {
+	return r.RunFunc(path, r.ProgressBar)
 }