Skip to content

Commit

Permalink
fs: split out include/exclude into new FilterFS
Browse files Browse the repository at this point in the history 
This patch creates a new FilterFS to pair with a new FilterOpt, which
contains the old WalkOpt parameters.

Essentially, this splits the functionality of the old FS into a new FS
and the FilterFS. The new FS implemenation simply performs operations
over a specified local filesystem, while the FilterFS wraps an existing
FS implementation to apply filtering patterns.

This allows the same logic for filtering to apply to *any* underlying
filesystem, for example, the StaticFS and MergeFS implementations in
BuildKit.

To do this, we also make some reasonably substantial changes to the
developer-facing API. We need to use fs.WalkDirFunc instead of the old
filepath.WalkFunc, which is required to preserve the lazy stat()
semantics from b9e22fc.

Existing implementations and callers of FS should fairly easily be able
to update to support the new API, which just requires updating to
support a new "path" parameter, and modify the signature to be
fs.WalkDirFunc (which may actually improve performance depending on the
exact usage).

Signed-off-by: Justin Chadwell <[email protected]>
  • Loading branch information
@jedevc
jedevc committed Aug 3, 2023
1 parent 36ef4d8 commit c871a2b
Show file tree
Hide file tree
Showing 11 changed files with 677 additions and 169 deletions.
6 changes: 5 additions & 1 deletion cmd/send/send.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,11 @@ func main() {
ctx := context.Background()
s := util.NewProtoStream(ctx, os.Stdin, os.Stdout)

if err := fsutil.Send(ctx, s, fsutil.NewFS(flag.Args()[0], nil), nil); err != nil {
fs, err := fsutil.NewFS(flag.Args()[0])
if err != nil {
panic(err)
}
if err := fsutil.Send(ctx, s, fs, nil); err != nil {
panic(err)
}
}
2 changes: 1 addition & 1 deletion cmd/walk/walk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func main() {
excludes = strings.Split(string(dt), "\n")
}

if err := fsutil.Walk(context.Background(), flag.Args()[0], &fsutil.WalkOpt{
if err := fsutil.Walk(context.Background(), flag.Args()[0], &fsutil.FilterOpt{
ExcludePatterns: excludes,
}, func(path string, fi os.FileInfo, err error) error {
if err != nil {
Expand Down
185 changes: 101 additions & 84 deletions walker.go → filter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,19 @@ package fsutil

import (
"context"
"io"
gofs "io/fs"
"os"
"path/filepath"
"strings"
"syscall"
"time"

"github.com/moby/patternmatcher"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil/types"
)

type WalkOpt struct {
type FilterOpt struct {
IncludePatterns []string
ExcludePatterns []string
// FollowPaths contains symlinks that are resolved into include patterns
Expand Down Expand Up @@ -43,31 +43,66 @@ const (
MapResultSkipDir
)

func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error {
root, err := filepath.EvalSymlinks(p)
if err != nil {
return errors.WithStack(&os.PathError{Op: "resolve", Path: root, Err: err})
}
rootFI, err := os.Stat(root)
if err != nil {
return errors.WithStack(err)
type filterFS struct {
fs FS
opt *FilterOpt
}

func NewFilterFS(fs FS, opt *FilterOpt) FS {
if opt == nil {
return fs
}
if !rootFI.IsDir() {
return errors.WithStack(&os.PathError{Op: "walk", Path: root, Err: syscall.ENOTDIR})

if fs, ok := fs.(*filterFS); ok {
includes := append([]string{}, fs.opt.IncludePatterns...)
includes = append(includes, opt.IncludePatterns...)
excludes := append([]string{}, fs.opt.ExcludePatterns...)
excludes = append(excludes, opt.ExcludePatterns...)
follows := append([]string{}, fs.opt.FollowPaths...)
follows = append(follows, opt.FollowPaths...)
var mapFn MapFunc
if fs.opt.Map != nil && opt.Map != nil {
mapFn = func(path string, stat *types.Stat) MapResult {
result := fs.opt.Map(path, stat)
if result != MapResultKeep {
return result
}
return opt.Map(path, stat)
}
} else if fs.opt.Map != nil {
mapFn = fs.opt.Map
} else if opt.Map != nil {
mapFn = opt.Map
}
return &filterFS{fs, &FilterOpt{
IncludePatterns: includes,
ExcludePatterns: excludes,
FollowPaths: follows,
Map: mapFn,
}}
}

return &filterFS{fs, opt}
}

func (fs *filterFS) Open(p string) (io.ReadCloser, error) {
return fs.fs.Open(p)
}

func (fs *filterFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error {
var (
includePatterns []string
includeMatcher *patternmatcher.PatternMatcher
excludeMatcher *patternmatcher.PatternMatcher
err error
)

if opt != nil && opt.IncludePatterns != nil {
includePatterns = make([]string, len(opt.IncludePatterns))
copy(includePatterns, opt.IncludePatterns)
if fs.opt != nil && fs.opt.IncludePatterns != nil {
includePatterns = make([]string, len(fs.opt.IncludePatterns))
copy(includePatterns, fs.opt.IncludePatterns)
}
if opt != nil && opt.FollowPaths != nil {
targets, err := FollowLinks(p, opt.FollowPaths)
if fs.opt != nil && fs.opt.FollowPaths != nil {
targets, err := FollowLinks(fs.fs, fs.opt.FollowPaths)
if err != nil {
return err
}
Expand All @@ -86,7 +121,7 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
if len(includePatterns) != 0 {
includeMatcher, err = patternmatcher.New(includePatterns)
if err != nil {
return errors.Wrapf(err, "invalid includepatterns: %s", opt.IncludePatterns)
return errors.Wrapf(err, "invalid includepatterns: %s", fs.opt.IncludePatterns)
}

for _, p := range includeMatcher.Patterns() {
Expand All @@ -99,10 +134,10 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}

onlyPrefixExcludeExceptions := true
if opt != nil && opt.ExcludePatterns != nil {
excludeMatcher, err = patternmatcher.New(opt.ExcludePatterns)
if fs.opt != nil && fs.opt.ExcludePatterns != nil {
excludeMatcher, err = patternmatcher.New(fs.opt.ExcludePatterns)
if err != nil {
return errors.Wrapf(err, "invalid excludepatterns: %s", opt.ExcludePatterns)
return errors.Wrapf(err, "invalid excludepatterns: %s", fs.opt.ExcludePatterns)
}

for _, p := range excludeMatcher.Patterns() {
Expand All @@ -114,9 +149,7 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}

type visitedDir struct {
fi os.FileInfo
path string
origpath string
entry gofs.DirEntry
pathWithSep string
includeMatchInfo patternmatcher.MatchInfo
excludeMatchInfo patternmatcher.MatchInfo
Expand All @@ -126,28 +159,16 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
// used only for include/exclude handling
var parentDirs []visitedDir

seenFiles := make(map[uint64]string)
return filepath.WalkDir(root, func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) {
return fs.fs.Walk(ctx, target, func(path string, dirEntry gofs.DirEntry, walkErr error) (retErr error) {
defer func() {
if retErr != nil && isNotExist(retErr) {
retErr = filepath.SkipDir
}
}()

origpath := path
path, err = filepath.Rel(root, path)
if err != nil {
return err
}
// Skip root
if path == "." {
return nil
}

var (
dir visitedDir
isDir bool
fi gofs.FileInfo
)
if dirEntry != nil {
isDir = dirEntry.IsDir()
Expand All @@ -163,15 +184,8 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
}

if isDir {
fi, err = dirEntry.Info()
if err != nil {
return err
}

dir = visitedDir{
fi: fi,
path: path,
origpath: origpath,
entry: dirEntry,
pathWithSep: path + string(filepath.Separator),
}
}
Expand Down Expand Up @@ -275,25 +289,21 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err

dir.calledFn = true

// The FileInfo might have already been read further up.
if fi == nil {
fi, err = dirEntry.Info()
if err != nil {
return err
}
}

stat, err := mkstat(origpath, path, fi, seenFiles)
fi, err := dirEntry.Info()
if err != nil {
return err
}
stat, ok := fi.Sys().(*types.Stat)
if !ok {
return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
}

select {
case <-ctx.Done():
return ctx.Err()
default:
if opt != nil && opt.Map != nil {
result := opt.Map(stat.Path, stat)
if fs.opt != nil && fs.opt.Map != nil {
result := fs.opt.Map(stat.Path, stat)
if result == MapResultSkipDir {
return filepath.SkipDir
} else if result == MapResultExclude {
Expand All @@ -304,36 +314,66 @@ func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) err
if parentDir.calledFn {
continue
}
parentStat, err := mkstat(parentDir.origpath, parentDir.path, parentDir.fi, seenFiles)
parentFi, err := parentDir.entry.Info()
if err != nil {
return err
}
parentStat, ok := parentFi.Sys().(*types.Stat)
if !ok {
return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"})
}

select {
case <-ctx.Done():
return ctx.Err()
default:
}
if opt != nil && opt.Map != nil {
result := opt.Map(parentStat.Path, parentStat)
if fs.opt != nil && fs.opt.Map != nil {
result := fs.opt.Map(parentStat.Path, parentStat)
if result == MapResultSkipDir || result == MapResultExclude {
continue
}
}

if err := fn(parentStat.Path, &StatInfo{parentStat}, nil); err != nil {
if err := fn(parentStat.Path, &DirEntryInfo{Stat: parentStat}, nil); err != nil {
return err
}
parentDirs[i].calledFn = true
}
if err := fn(stat.Path, &StatInfo{stat}, nil); err != nil {
if err := fn(stat.Path, &DirEntryInfo{Stat: stat}, nil); err != nil {
return err
}
}
return nil
})
}

func Walk(ctx context.Context, p string, opt *FilterOpt, fn filepath.WalkFunc) error {
f, err := NewFS(p)
if err != nil {
return err
}
return NewFilterFS(f, opt).Walk(ctx, "/", func(path string, d gofs.DirEntry, err error) error {
var info gofs.FileInfo
if d != nil {
var err2 error
info, err2 = d.Info()
if err == nil {
err = err2
}
}
return fn(path, info, err)
})
}

func WalkDir(ctx context.Context, p string, opt *FilterOpt, fn gofs.WalkDirFunc) error {
f, err := NewFS(p)
if err != nil {
return err
}
return NewFilterFS(f, opt).Walk(ctx, "/", fn)
}

func patternWithoutTrailingGlob(p *patternmatcher.Pattern) string {
patStr := p.String()
// We use filepath.Separator here because patternmatcher.Pattern patterns
Expand All @@ -344,29 +384,6 @@ func patternWithoutTrailingGlob(p *patternmatcher.Pattern) string {
return patStr
}

type StatInfo struct {
*types.Stat
}

func (s *StatInfo) Name() string {
return filepath.Base(s.Stat.Path)
}
func (s *StatInfo) Size() int64 {
return s.Stat.Size_
}
func (s *StatInfo) Mode() os.FileMode {
return os.FileMode(s.Stat.Mode)
}
func (s *StatInfo) ModTime() time.Time {
return time.Unix(s.Stat.ModTime/1e9, s.Stat.ModTime%1e9)
}
func (s *StatInfo) IsDir() bool {
return s.Mode().IsDir()
}
func (s *StatInfo) Sys() interface{} {
return s.Stat
}

func isNotExist(err error) bool {
return errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTDIR)
}
Loading

0 comments on commit c871a2b

Please sign in to comment.