WIP: A better glob?

agent/artifact_uploader.go

@@ -4,16 +4,17 @@ import (
 	"context"
 	"crypto/sha1"
 	"crypto/sha256"
-	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/DrJosh9000/zzglob"
 	"github.com/buildkite/agent/v3/api"
 	"github.com/buildkite/agent/v3/internal/artifact"
 	"github.com/buildkite/agent/v3/internal/experiments"
@@ -22,7 +23,6 @@ import (
 	"github.com/buildkite/agent/v3/pool"
 	"github.com/buildkite/roko"
 	"github.com/dustin/go-humanize"
-	zglob "github.com/mattn/go-zglob"
 )
 
 const (
@@ -125,19 +125,26 @@ func (a *ArtifactUploader) Collect(ctx context.Context) (artifacts []*api.Artifa
 
 		a.logger.Debug("Searching for %s", globPath)
 
-		// Resolve the globs (with * and ** in them), if it's a non-globbed path and doesn't exists
-		// then we will get the ErrNotExist that is handled below
-		globfunc := zglob.Glob
-		if a.conf.GlobResolveFollowSymlinks {
-			// Follow symbolic links for files & directories while expanding globs
-			globfunc = zglob.GlobFollowSymlinks
+		// Resolve the globs (with * and ** in them)
+		pattern, err := zzglob.Parse(globPath)
+		if err != nil {
+			return nil, fmt.Errorf("invalid glob pattern: %w", err)
 		}
-		files, err := globfunc(globPath)
-		if errors.Is(err, os.ErrNotExist) {
-			a.logger.Info("File not found: %s", globPath)
-			continue
-		} else if err != nil {
-			return nil, fmt.Errorf("resolving glob: %w", err)
+		var files []string
+		err = pattern.Glob(func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				a.logger.Warn("Couldn't walk path %s", path)
+				return nil
+			}
+			if d != nil && d.IsDir() {
+				a.logger.Warn("Glob pattern %s matched a directory %s", globPath, path)
+				return nil
+			}
+			files = append(files, path)
+			return nil
+		}, a.conf.GlobResolveFollowSymlinks)
+		if err != nil {
+			return nil, fmt.Errorf("globbing pattern: %w", err)
 		}
 
 		// Process each glob match into an api.Artifact

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
 	github.com/DataDog/datadog-go/v5 v5.3.0
+	github.com/DrJosh9000/zzglob v0.0.4
 	github.com/aws/aws-sdk-go v1.44.334
 	github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
 	github.com/buildkite/bintest/v3 v3.1.1
@@ -22,7 +23,6 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-querystring v1.1.0
 	github.com/lestrrat-go/jwx/v2 v2.0.12
-	github.com/mattn/go-zglob v0.0.4
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/oleiade/reflections v1.0.1
 	github.com/opentracing/opentracing-go v1.2.0

go.sum

@@ -33,6 +33,10 @@ github.com/DataDog/go-tuf v1.0.1-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOA
 github.com/DataDog/gostackparse v0.5.0 h1:jb72P6GFHPHz2W0onsN51cS3FkaMDcjb0QzgxxA4gDk=
 github.com/DataDog/sketches-go v1.2.1 h1:qTBzWLnZ3kM2kw39ymh6rMcnN+5VULwFs++lEYUUsro=
 github.com/DataDog/sketches-go v1.2.1/go.mod h1:1xYmPLY1So10AwxV6MJV0J53XVH+WL9Ad1KetxVivVI=
+github.com/DrJosh9000/zzglob v0.0.3 h1:zzhWB7ZABLJJNJfr4Ad/mFoT2iBBbd2Ql20ZWRKIwkc=
+github.com/DrJosh9000/zzglob v0.0.3/go.mod h1:+iLI/qvROFsS1A/jl+B8MVYNu/0cjveYOJqU37O8fcs=
+github.com/DrJosh9000/zzglob v0.0.4 h1:VZmBZmzIS89475ekcJpwqRsNdLQ6pQK4pOhcepb3Phw=
+github.com/DrJosh9000/zzglob v0.0.4/go.mod h1:+iLI/qvROFsS1A/jl+B8MVYNu/0cjveYOJqU37O8fcs=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
@@ -182,8 +186,6 @@ github.com/lestrrat-go/jwx/v2 v2.0.12/go.mod h1:Mq4KN1mM7bp+5z/W5HS8aCNs5RKZ911G
 github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
 github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
 github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
-github.com/mattn/go-zglob v0.0.4 h1:LQi2iOm0/fGgu80AioIJ/1j9w9Oh+9DZ39J4VAGzHQM=
-github.com/mattn/go-zglob v0.0.4/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/oleiade/reflections v1.0.1 h1:D1XO3LVEYroYskEsoSiGItp9RUxG6jWnCVvrqH0HHQM=
@@ -226,6 +228,7 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=