Add archive with filter func

archive/archive.go

@@ -0,0 +1,90 @@
+package archive
+
+import (
+	"fmt"
+	"github.com/jfrog/archiver/v3"
+	"github.com/jfrog/gofrog/io"
+	"os"
+	"path/filepath"
+)
+
+// FilterFunc is a function that determines whether a file should be excluded from an archive.
+type FilterFunc func(path string, info os.FileInfo) bool
+
+// ArchiveWithFilterFunc creates an archive from the specified source directory, excluding files based on the filter function.
+// The archive is saved to the specified destination file.
+// The rootDir is used to calculate the relative path of the files in the archive.
+// The srcDir is the directory to be archived.
+// The destination is the path to the archive file to be created. The file extension determines the archive format (e.g., .zip, .tar).
+// The filterFunc is a function that determines whether a file should be excluded from an archive.
+func ArchiveWithFilterFunc(rootDir, srcDir string, destination string, filterFunc FilterFunc) (err error) {
+	// Identify the archive format based on the file extension (e.g., .zip, .tar)
+	archive, err := archiver.ByExtension(destination)
+	if err != nil {
+		return err
+	}
+
+	// Ensure the identified format supports creating an archive
+	archiveWriter, ok := archive.(archiver.Writer)
+	if !ok {
+		return fmt.Errorf("unsupported archive format for writing: %s", destination)
+	}
+
+	// Open the output file for the archive
+	out, err := os.Create(destination)
+	if err != nil {
+		return fmt.Errorf("failed while creating destination file %s: %v", destination, err)
+	}
+	defer io.Close(out, &err)
+
+	// Initialize the archive (ZIP, TAR, etc.)
+	err = archiveWriter.Create(out)
+	if err != nil {
+		return fmt.Errorf("failed while creating archive: %v", err)
+	}
+	defer io.Close(archiveWriter, &err)
+
+	// Walk through each source file and directory
+	return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) (walkErr error) {
+		if err != nil {
+			return err
+		}
+		// Filter out files that should not be included in the archive
+		excluded := filterFunc(path, info)
+		if excluded {
+			if info.IsDir() {
+				// Return SkipDir error to avoid walking into the directory
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		// Skip directories and symlinks
+		if info.IsDir() || io.IsFileSymlink(info) {
+			return nil
+		}
+
+		// Calculate the relative path to preserve the folder structure
+		relativePath, walkErr := filepath.Rel(filepath.Dir(rootDir), path)
+		if walkErr != nil {
+			return walkErr
+		}
+
+		// Open the file
+		file, walkErr := os.Open(path)
+		if walkErr != nil {
+			return walkErr
+		}
+		defer io.Close(archiveWriter, &walkErr)
+
+		// Add the file to the archive, using the relative path
+		return archiveWriter.Write(archiver.File{
+			FileInfo: archiver.FileInfo{
+				FileInfo: info,
+				// Preserve the relative path in the archive
+				CustomName: relativePath,
+			},
+			ReadCloser: file,
+		})
+	})
+}

unarchive/archive.go → archive/unarchive.go

@@ -1,4 +1,4 @@
-package unarchive
+package archive
 
 import (
 	"encoding/json"

unarchive/archive_test.go → archive/unarchive_test.go

@@ -1,4 +1,4 @@
-package unarchive
+package archive
 
 import (
 	"os"

go.mod

@@ -4,27 +4,27 @@ go 1.22
 
 require (
 	github.com/gookit/color v1.5.4
-	github.com/jfrog/archiver/v3 v3.6.0
+	github.com/jfrog/archiver/v3 v3.6.1
 	github.com/minio/sha256-simd v1.0.1
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.9.0
+	golang.org/x/sync v0.8.0
 )
 
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/klauspost/compress v1.17.4 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/ulikunitz/xz v0.5.11 // indirect
+	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
-	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.10.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -9,11 +9,11 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
 github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
-github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F1w=
-github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI=
+github.com/jfrog/archiver/v3 v3.6.1 h1:LOxnkw9pOn45DzCbZNFV6K0+6dCsQ0L8mR3ZcujO5eI=
+github.com/jfrog/archiver/v3 v3.6.1/go.mod h1:VgR+3WZS4N+i9FaDwLZbq+jeU4B4zctXL+gL4EMzfLw=
 github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
-github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
 github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
@@ -32,17 +32,17 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
-github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
-github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
+github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
 github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
+golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

parallel/runner.go

@@ -58,8 +58,6 @@ type runner struct {
 	threadCount atomic.Uint32
 	// The number of open threads.
 	openThreads atomic.Uint32
-	// A lock on openThreads.
-	openThreadsLock sync.Mutex
 	// The number of threads currently running tasks.
 	activeThreads atomic.Uint32
 	// The number of tasks in the queue.
@@ -250,9 +248,7 @@ func (r *runner) addThread() {
 	nextThreadId := r.threadCount.Add(1) - 1
 	go func(threadId int) {
 		defer r.threadsWaitGroup.Done()
-		r.openThreadsLock.Lock()
 		r.openThreads.Add(1)
-		r.openThreadsLock.Unlock()
 
 		// Keep on taking tasks from the queue.
 		for t := range r.tasks {
@@ -290,14 +286,11 @@ func (r *runner) addThread() {
 				}
 			}
 
-			r.openThreadsLock.Lock()
 			// If the total of open threads is larger than the maximum (maxParallel), then this thread should be closed.
 			if int(r.openThreads.Load()) > r.maxParallel {
 				r.openThreads.Add(^uint32(0))
-				r.openThreadsLock.Unlock()
 				break
 			}
-			r.openThreadsLock.Unlock()
 		}
 	}(int(nextThreadId))
 }