diff --git a/go.mod b/go.mod index 5c3b57980..5ea2d512f 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/gdamore/encoding v1.0.0 // indirect + github.com/iceber/iouring-go v0.0.0-20220609112130-b1dc8dd9fbfd // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect diff --git a/go.sum b/go.sum index 3b94c37de..ef9b3c1d0 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.5.3 h1:b9XQrT6QGbgI7JvZOJXFNczOQeIYbo8BfeSMzt2sAV0= github.com/gdamore/tcell/v2 v2.5.3/go.mod h1:wSkrPaXoiIWZqW/g7Px4xc79di6FTcpB8tvaKJ6uGBo= +github.com/iceber/iouring-go v0.0.0-20220609112130-b1dc8dd9fbfd h1:UdLfG7nAV9de/1kkx6l9OJD5GdJTzl4HrIa5hfpAnmE= +github.com/iceber/iouring-go v0.0.0-20220609112130-b1dc8dd9fbfd/go.mod h1:LEzdaZarZ5aqROlLIwJ4P7h3+4o71008fSy6wpaEB+s= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -58,6 +60,7 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220318055525-2edf467146b5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/analyze/dir.go b/pkg/analyze/dir.go index 08c0b435d..96c38c916 100644 --- a/pkg/analyze/dir.go +++ b/pkg/analyze/dir.go @@ -44,7 +44,7 @@ func (a *ParallelAnalyzer) GetProgressChan() chan common.CurrentProgress { return a.progressOutChan } -// GetDoneChan returns channel for checking when analysis is done +// GetDone returns channel for checking when analysis is done func (a *ParallelAnalyzer) GetDone() common.SignalGroup { return a.doneChan } diff --git a/pkg/analyze/uring.go b/pkg/analyze/uring.go new file mode 100644 index 000000000..5a38f3487 --- /dev/null +++ b/pkg/analyze/uring.go @@ -0,0 +1,220 @@ +package analyze + +import ( + "os" + "path/filepath" + "runtime/debug" + "syscall" + "time" + + "golang.org/x/sys/unix" + + "github.com/dundee/gdu/v5/internal/common" + "github.com/dundee/gdu/v5/pkg/fs" + "github.com/iceber/iouring-go" + log "github.com/sirupsen/logrus" +) + +var statxFlags uint32 = unix.AT_SYMLINK_NOFOLLOW | unix.AT_STATX_DONT_SYNC +var statxMask = unix.STATX_TYPE | unix.STATX_SIZE + +// UringAnalyzer is a parallel analyzer with io_uring usage +type UringAnalyzer struct { + doneChan common.SignalGroup + progressChan chan common.CurrentProgress + ignoreDir common.ShouldDirBeIgnored + iour *iouring.IOURing + wait *WaitGroup +} + +// CreateUringAnalyzer returns analyzer which uses io_uring +func CreateUringAnalyzer() *UringAnalyzer { + return &UringAnalyzer{ + doneChan: make(common.SignalGroup), + progressChan: make(chan common.CurrentProgress, 1), + wait: (&WaitGroup{}).Init(), + } +} + +// GetDone returns channel for checking when analysis is done +func (a *UringAnalyzer) GetDone() common.SignalGroup { + return a.doneChan +} + +// GetProgressChan returns channel for getting progress +func (a *UringAnalyzer) GetProgressChan() chan common.CurrentProgress { + return a.progressChan +} + +// ResetProgress returns progress +func (a *UringAnalyzer) ResetProgress() { + a.doneChan = make(common.SignalGroup) + a.wait = (&WaitGroup{}).Init() +} + +// AnalyzeDir analyzes given path +func (a *UringAnalyzer) AnalyzeDir( + path string, ignore common.ShouldDirBeIgnored, constGC bool, +) fs.Item { + if !constGC { + defer debug.SetGCPercent(debug.SetGCPercent(-1)) + go manageMemoryUsage(a.doneChan) + } + + iour, err := iouring.New(32768) + if err != nil { + log.Print(err.Error()) + } + defer iour.Close() + a.iour = iour + + a.ignoreDir = ignore + + dir := a.processDir(path) + + dir.BasePath = filepath.Dir(path) + a.wait.Wait() + + a.doneChan.Broadcast() + + return dir +} + +func (a *UringAnalyzer) processDir(path string) *Dir { + var ( + file *File + err error + totalSize int64 + subDirChan = make(chan *Dir) + dirCount int + ) + + a.wait.Add(1) + + files, err := os.ReadDir(path) + if err != nil { + log.Print(err.Error()) + } + + dir := &Dir{ + File: &File{ + Name: filepath.Base(path), + Flag: getDirFlag(err, len(files)), + }, + ItemCount: 1, + Files: make(fs.Files, 0, len(files)), + } + a.setDirPlatformSpecificAttrs(dir, path) + + stats := make(map[string]*unix.Statx_t) + var reqs []iouring.PrepRequest + + for _, f := range files { + name := f.Name() + entryPath := filepath.Join(path, name) + if f.IsDir() { + if a.ignoreDir(name, entryPath) { + continue + } + dirCount++ + + go func(entryPath string) { + concurrencyLimit <- struct{}{} + subdir := a.processDir(entryPath) + subdir.Parent = dir + + subDirChan <- subdir + <-concurrencyLimit + }(entryPath) + } else { + stat := &unix.Statx_t{} + req, err := iouring.Statx( + unix.AT_FDCWD, entryPath, statxFlags, statxMask, stat, + ) + if err != nil { + log.Print(err.Error()) + continue + } + + reqs = append(reqs, req) + stats[name] = stat + } + } + + if len(reqs) > 0 { + res, err := a.iour.SubmitRequests(reqs, nil) + if err != nil { + log.Print(err.Error()) + return nil + } + <-res.Done() + + for name, stat := range stats { + file = &File{ + Name: name, + Flag: getFlagStatx(stat), + Size: int64(stat.Size), + Parent: dir, + } + a.setPlatformSpecificAttrs(file, stat) + + totalSize += int64(stat.Size) + + dir.AddFile(file) + } + } + + go func() { + var sub *Dir + + for i := 0; i < dirCount; i++ { + sub = <-subDirChan + dir.AddFile(sub) + } + + a.wait.Done() + }() + + return dir +} + +func (a *UringAnalyzer) setPlatformSpecificAttrs(file *File, stat *unix.Statx_t) { + file.Usage = int64(stat.Blocks * devBSize) + file.Mtime = time.Unix(int64(stat.Mtime.Sec), int64(stat.Mtime.Nsec)) + + if stat.Nlink > 1 { + file.Mli = stat.Ino + } +} + +func (a *UringAnalyzer) setDirPlatformSpecificAttrs(dir *Dir, path string) { + stat := &unix.Statx_t{} + req, err := iouring.Statx( + unix.AT_FDCWD, path, statxFlags, statxMask, stat, + ) + if err != nil { + log.Print(err.Error()) + return + } + res, err := a.iour.SubmitRequest(req, nil) + if err != nil { + log.Print(err.Error()) + return + } + <-res.Done() + + dir.Mtime = time.Unix(int64(stat.Mtime.Sec), int64(stat.Mtime.Nsec)) +} + +func getFlagStatx(f *unix.Statx_t) rune { + fType := f.Mode & syscall.S_IFMT + switch fType { + case syscall.S_IFLNK: + fallthrough + case syscall.S_IFSOCK: + return '@' + default: + return ' ' + + } +} diff --git a/stdout/stdout.go b/stdout/stdout.go index df47029d1..02953b162 100644 --- a/stdout/stdout.go +++ b/stdout/stdout.go @@ -48,7 +48,7 @@ func CreateStdoutUI( ShowProgress: showProgress, ShowApparentSize: showApparentSize, ShowRelativeSize: showRelativeSize, - Analyzer: analyze.CreateAnalyzer(), + Analyzer: analyze.CreateUringAnalyzer(), ConstGC: constGC, UseSIPrefix: useSIPrefix, },