Skip to content

Commit

Permalink
display spinner during indexing
Browse files Browse the repository at this point in the history
  • Loading branch information
creativeprojects committed Jun 8, 2024
1 parent acb152c commit ee1abc2
Show file tree
Hide file tree
Showing 13 changed files with 209 additions and 77 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go_version: ['1.21']
go_version: ['1.22']
os: [ubuntu-latest, windows-latest, macos-latest]

steps:
Expand Down
5 changes: 4 additions & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"os"

"github.com/creativeprojects/catalogue/constants"
"github.com/pterm/pterm"
"github.com/spf13/cobra"
)

Expand All @@ -19,7 +20,9 @@ var (
Short: constants.Description,
Long: `An offline file catalogue with fast search`,
Run: func(cmd *cobra.Command, args []string) {
// set log level here
if rootFlags.Verbose {
pterm.EnableDebugMessages()
}
},
}

Expand Down
28 changes: 25 additions & 3 deletions cmd/volume_add.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"fmt"
"os"
"strings"
"sync"
"time"

"github.com/creativeprojects/catalogue/index"
"github.com/creativeprojects/catalogue/volume"
Expand Down Expand Up @@ -40,10 +42,30 @@ var volumeAddCmd = &cobra.Command{
return
}
volume.PrintVolume(vol)
fmt.Println("")

progresser := index.NewProgress(&pterm.DefaultSpinner)
indexer := index.NewIndexer(volumePath)
indexer.Run(progresser)
wg := new(sync.WaitGroup)
fileIndexedChannel := make(chan index.FileIndexed, 1000)
wg.Add(1)
go func() {
defer wg.Done()
start := time.Now()
progresser := index.NewProgress()
progresser.Start()

for fileIndexed := range fileIndexedChannel {
if fileIndexed.Error != nil {
progresser.Error(fileIndexed.Path, fileIndexed.Error)
continue
}
progresser.Increment(fileIndexed.Path, fileIndexed.Info)
}
progresser.Success("Indexing complete in " + time.Since(start).String())
}()
indexer := index.NewIndexer(volumePath, fileIndexedChannel)
indexer.Run()
close(fileIndexedChannel)
wg.Wait()
},
}

Expand Down
14 changes: 7 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
module github.com/creativeprojects/catalogue

go 1.21
go 1.22

require (
github.com/google/uuid v1.4.0
github.com/pterm/pterm v0.12.72
github.com/google/uuid v1.6.0
github.com/pterm/pterm v0.12.79
github.com/spf13/afero v1.11.0
github.com/spf13/cobra v1.8.0
github.com/stretchr/testify v1.8.4
go.etcd.io/bbolt v1.3.8
golang.org/x/sys v0.15.0
github.com/stretchr/testify v1.9.0
go.etcd.io/bbolt v1.3.10
golang.org/x/sys v0.21.0
howett.net/plist v1.0.1
)

Expand All @@ -28,7 +28,7 @@ require (
github.com/rivo/uniseg v0.4.4 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/term v0.16.0 // indirect
golang.org/x/text v0.14.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
26 changes: 14 additions & 12 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo=
github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
Expand Down Expand Up @@ -56,8 +56,8 @@ github.com/pterm/pterm v0.12.31/go.mod h1:32ZAWZVXD7ZfG0s8qqHXePte42kdz8ECtRyEej
github.com/pterm/pterm v0.12.33/go.mod h1:x+h2uL+n7CP/rel9+bImHD5lF3nM9vJj80k9ybiiTTE=
github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5bUw8T8=
github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s=
github.com/pterm/pterm v0.12.72 h1:1W7Oqi5yjEvrI0QroBqUefqXKJW4aD8/wAuqqd5qj0Q=
github.com/pterm/pterm v0.12.72/go.mod h1:+M33aZWQVpmLmLbvjykyGZ4gAfeebznRo8JMbabaxQU=
github.com/pterm/pterm v0.12.79 h1:lH3yrYMhdpeqX9y5Ep1u7DejyHy7NSQg9qrBjF9dFT4=
github.com/pterm/pterm v0.12.79/go.mod h1:1v/gzOF1N0FsjbgTHZ1wVycRkKiatFvJSJC4IGaQAAo=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
Expand All @@ -74,14 +74,14 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E=
Expand All @@ -95,6 +95,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand All @@ -105,15 +107,15 @@ golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
Expand Down
39 changes: 22 additions & 17 deletions index/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,55 +4,60 @@ import (
"os"
"path/filepath"

"github.com/pterm/pterm"
"github.com/spf13/afero"
)

type FileIndexed struct {
Path string
Info os.FileInfo
Error error
}

type Indexer struct {
fs afero.Fs
rootPath string
progressBar *pterm.ProgressbarPrinter
fs afero.Fs
rootPath string
fileIndexedChannel chan<- FileIndexed
}

func NewIndexer(rootPath string) *Indexer {
return NewFsIndexer(rootPath, afero.NewOsFs())
func NewIndexer(rootPath string, fileIndexedChannel chan<- FileIndexed) *Indexer {
return NewFsIndexer(rootPath, fileIndexedChannel, afero.NewOsFs())
}

func NewFsIndexer(rootPath string, fs afero.Fs) *Indexer {
pbar, _ := pterm.DefaultProgressbar.WithShowElapsedTime().WithShowCount().WithShowTitle().Start()
func NewFsIndexer(rootPath string, fileIndexedChannel chan<- FileIndexed, fs afero.Fs) *Indexer {
return &Indexer{
fs: fs,
rootPath: rootPath,
progressBar: pbar,
fs: fs,
rootPath: rootPath,
fileIndexedChannel: fileIndexedChannel,
}
}

func (i *Indexer) Run(progresser Progresser) {
// Run starts the indexing process. It will walk the filesystem and send the results to the fileIndexedChannel.
// The Run method will return after all files have been indexed.
func (i *Indexer) Run() {
_ = i.walk(i.rootPath)
}

func (i *Indexer) walk(path string) error {
file, err := i.fs.Open(path)
if err != nil {
pterm.Error.Printf("%s: %s\n", path, err)
i.fileIndexedChannel <- FileIndexed{Path: path, Error: err}
return err
}
names, err := file.Readdirnames(-1)
file.Close()
if err != nil {
pterm.Error.Printf("%s: %s\n", path, err)
i.fileIndexedChannel <- FileIndexed{Path: path, Error: err}
return err
}
i.progressBar.Total += len(names)

for _, name := range names {
i.progressBar.Increment()
filename := filepath.Join(path, name)
fileInfo, err := lstatIfPossible(i.fs, filename)
if err != nil {
pterm.Error.Printf("%s: %s\n", filename, err)
i.fileIndexedChannel <- FileIndexed{Path: filename, Error: err}
continue
}
i.fileIndexedChannel <- FileIndexed{Path: filename, Info: fileInfo}
if fileInfo.IsDir() {
_ = i.walk(filename)
}
Expand Down
52 changes: 30 additions & 22 deletions index/progress.go
Original file line number Diff line number Diff line change
@@ -1,50 +1,58 @@
package index

import (
"fmt"
"os"
"strings"

"github.com/creativeprojects/catalogue/ui"
"github.com/pterm/pterm"
)

type Progresser interface {
Increment(path string, info os.FileInfo, fileCount, dirCount int)
Start()
Increment(path string, info os.FileInfo)
Error(path string, err error)
Success(message string)
}

type Progress struct {
spinner *pterm.SpinnerPrinter
spinner *ui.SpinnerPrinter
fileCount int
dirCount int
errorCount int
}

func NewProgress(spinner *pterm.SpinnerPrinter) *Progress {
if spinner == nil {
spinner = &pterm.DefaultSpinner
}
spinner.Start()
func NewProgress() *Progress {
return &Progress{
spinner: spinner,
spinner: ui.DefaultSpinner,
}
}

func (p *Progress) Increment(path string, info os.FileInfo, fileCount, dirCount int) {
maxWidth := pterm.GetTerminalWidth() - 5
display := path
if len(display) > maxWidth {
display = display[len(display)-maxWidth:]
}
if len(display) < maxWidth { // pad with spaces
display = display + strings.Repeat(" ", maxWidth-len(display))
func (p *Progress) Start() {
p.spinner.Start()
}

func (p *Progress) Increment(path string, info os.FileInfo) {
if info.IsDir() {
p.dirCount++
} else {
p.fileCount++
}
p.spinner.Text = display
p.spinner.WithText(display)
p.update()
}

func (p *Progress) Error(path string, err error) {
p.spinner.FailPrinter.Printf("%s: %s\n", path, err)
pterm.Error.Println("\r", err)
p.errorCount++
p.update()
}

func (p *Progress) Success(message string) {
p.spinner.Success(message)
_ = p.spinner.Stop()
p.spinner.Stop()
pterm.Success.Println(message)
}

func (p *Progress) update() {
text := fmt.Sprintf("Files: %d, Directories: %d, Errors: %d", p.fileCount, p.dirCount, p.errorCount)
p.spinner.UpdateText(text)
}
12 changes: 6 additions & 6 deletions store/bucket_test.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
package store

import (
"math/rand"
"testing"
"time"

"github.com/stretchr/testify/assert"
)
Expand All @@ -13,10 +11,6 @@ type testBucketData struct {
bucket Bucket
}

func init() {
rand.Seed(time.Now().UnixNano())
}

func getBuckets() []testBucketData {
data := make([]testBucketData, 1)

Expand All @@ -25,6 +19,8 @@ func getBuckets() []testBucketData {
}

func TestLoadingUnknownKeyFromBucket(t *testing.T) {
t.Parallel()

for _, data := range getBuckets() {
t.Run(data.name, func(t *testing.T) {
_, err := data.bucket.GetKey("some-key")
Expand All @@ -34,8 +30,12 @@ func TestLoadingUnknownKeyFromBucket(t *testing.T) {
}

func TestSetKeyAndGetKeyFromBucket(t *testing.T) {
t.Parallel()

for _, data := range getBuckets() {
t.Run(data.name, func(t *testing.T) {
t.Parallel()

var err error
var value1, value2 []byte

Expand Down
Loading

0 comments on commit ee1abc2

Please sign in to comment.