Skip to content

Commit

Permalink
Add functionality (disabled) for all .gitignores in directory
Browse files Browse the repository at this point in the history
I added support for finding all .gitignore files within a directory and adding those to the ignore list, but it's costly in large directories, so it's disabled
  • Loading branch information
caitlinelfring committed Sep 3, 2020
1 parent de427db commit 51416cf
Show file tree
Hide file tree
Showing 8 changed files with 155 additions and 41 deletions.
9 changes: 5 additions & 4 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,17 +79,18 @@ Provide a list file globs for files you'd like to check.`,
Msg("woke completed")
}()

if len(args) == 0 {
args = parser.DefaultPath
}

cfg, err := config.NewConfig(ruleConfig)
if err != nil {
return err
}

var ignorer *ignore.Ignore
if !noIgnore {
ignorer, err = ignore.NewIgnore(cfg.IgnoreFiles...)
if err != nil {
return err
}
ignorer = ignore.NewIgnore(cfg.IgnoreFiles, args)
}

p := parser.NewParser(cfg.Rules, ignorer)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ go 1.14

require (
github.com/fatih/color v1.9.0
github.com/get-woke/go-gitignore v1.1.0
github.com/rs/zerolog v1.19.0
github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94
github.com/spf13/cobra v1.0.0
github.com/stretchr/testify v1.6.1
gopkg.in/yaml.v2 v2.3.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/get-woke/go-gitignore v1.1.0 h1:KByIyNTe9eSz5vnfX3Q8HdHhEsEy77j5HMCW80oOIoc=
github.com/get-woke/go-gitignore v1.1.0/go.mod h1:qbLKpA7cfqI6TqvUDGqE4hsYke8nuiKvmepVBFBwVh4=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
Expand Down Expand Up @@ -85,8 +87,6 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.19.0 h1:hYz4ZVdUgjXTBUmrkrw55j1nHx68LfOKIQk5IYtyScg=
github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94 h1:G04eS0JkAIVZfaJLjla9dNxkJCPiKIGZlw9AfOhzOD0=
github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94/go.mod h1:b18R55ulyQ/h3RaWyloPyER7fWQVZvimKKhnI5OfrJQ=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
Expand Down
3 changes: 2 additions & 1 deletion pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,9 @@ func loadConfig(filename string) (*Config, error) {

func loadDefaultConfigFiles() (cfg *Config) {
for _, file := range defaultConfigFilenames {
log.Debug().Str("cfg", file).Msg("trying default config file")

if _, err := os.Stat(file); os.IsNotExist(err) {
log.Debug().Str("cfg", file).Err(err).Msg("tried default config file")
continue
}
var err error
Expand Down
71 changes: 57 additions & 14 deletions pkg/ignore/ignore.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@ import (
"errors"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"

gitignore "github.com/get-woke/go-gitignore"
"github.com/rs/zerolog/log"
gitignore "github.com/sabhiram/go-gitignore"
)

// DefaultIgnores is the default list of file globs that will be ignored
Expand All @@ -18,37 +19,67 @@ var DefaultIgnores = []string{

// Ignore is a gitignore-style object to ignore files/directories
type Ignore struct {
compiled *gitignore.GitIgnore
matcher *gitignore.GitIgnore
}

// NewIgnore produces an Ignore object, with compiled lines from .gitignore and DefaultIgnores
// which you can match files against
func NewIgnore(lines ...string) (*Ignore, error) {
func NewIgnore(lines []string, pathsForGitIgnores []string) *Ignore {
start := time.Now()
defer func() {
log.Debug().
Dur("durationMS", time.Since(start)).
Msg("finished compiling ignores")
}()

compiled, err := compileIgnoreLines(lines)
if err != nil {
return nil, err
lines = append(lines, DefaultIgnores...)
lines = append(lines, readIgnoreFile(".gitignore")...)
lines = append(lines, readIgnoreFile(".wokeignore")...)

ignorer := Ignore{
matcher: gitignore.CompileIgnoreLines(lines...),
}
return &Ignore{compiled: compiled}, nil

// FIXME: This is very costly with large directories with a lot of files, disabled for now
// ignorer.AddIgnoreFiles(".gitignore", pathsForGitIgnores)
// ignorer.AddIgnoreFiles(".wokeignore", pathsForGitIgnores)

return &ignorer
}

// Match returns true if the provided file matches any of the defined ignores
func (i *Ignore) Match(f string) bool {
return i.compiled.MatchesPath(f)
return i.matcher.MatchesPath(f)
}

func compileIgnoreLines(lines []string) (*gitignore.GitIgnore, error) {
lines = append(lines, DefaultIgnores...)
lines = append(lines, readIgnoreFile(".gitignore")...)
lines = append(lines, readIgnoreFile(".wokeignore")...)
// AddIgnoreFiles walks each path provided in search of any files that match ignoreName
// and add the contents of those files to the gitignore matcher
// NOTE: this is very costly in large directories and should be used with caution
func (i *Ignore) AddIgnoreFiles(ignoreName string, paths []string) {
lines := addRecursiveGitIgnores(ignoreName, paths)
i.matcher.AddPatternsFromLines(lines...)
}

// addRecursiveGitIgnores uses filepath.Walk to walk each path, search for a file that matches
// ignoreName and reads each file's lines
// NOTE: this is very costly in large directories and should be used with caution
func addRecursiveGitIgnores(ignoreName string, paths []string) (lines []string) {
for _, path := range paths {
_ = filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}

return gitignore.CompileIgnoreLines(lines...)
if info.Mode().IsRegular() && info.Name() == ignoreName {
newLines := append(readIgnoreFile(p), p)
lines = append(lines, newLines...)
}

return nil
})
}

return
}

func readIgnoreFile(file string) []string {
Expand All @@ -59,7 +90,19 @@ func readIgnoreFile(file string) []string {
_event = log.Debug()
}
_event.Err(err).Str("file", file).Msg("skipping ignorefile")
return []string{}
}

log.Debug().Str("file", file).Msg("adding ignorefile")
rawLines := strings.Split(strings.TrimSpace(string(buffer)), "\n")

// Pre-allocate the slice
lines := make([]string, len(rawLines))
// Here, we are prefixing each line with the base of the ignore file
// to suppose ignore files in subdirectories
for i := range rawLines {
lines[i] = filepath.Join(filepath.Dir(file), rawLines[i])
}

return strings.Split(strings.TrimSpace(string(buffer)), "\n")
return lines
}
82 changes: 77 additions & 5 deletions pkg/ignore/ignore_test.go
Original file line number Diff line number Diff line change
@@ -1,23 +1,95 @@
package ignore

import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strconv"
"testing"

"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
)

func init() {
zerolog.SetGlobalLevel(zerolog.NoLevel)
}

func TestIgnoreMatch(t *testing.T) {
i, err := NewIgnore("my/files/*")
assert.NoError(t, err)
i := NewIgnore([]string{"my/files/*"}, []string{})
assert.NotNil(t, i)

assert.False(t, i.Match("not/foo"))
assert.True(t, i.Match("my/files/file1"))
assert.False(t, i.Match("my/files"))
}

func TestReadIgnoreFIle(t *testing.T) {
lines := readIgnoreFile("testdata/.gitignore")
func TestIgnore_AddIgnoreFiles(t *testing.T) {
i := NewIgnore([]string{"my/files/*"}, []string{"."})
i.AddIgnoreFiles(".gitignore", []string{"testdata"})

assert.True(t, i.Match("testdata/.gitignore"))
assert.True(t, i.Match("testdata/.DS_Store"))
assert.False(t, i.Match(".DS_Store"))
assert.True(t, i.Match("my/files/file.txt"))
assert.False(t, i.Match("my/files"))
}

func TestAddRecursiveGitIgnores(t *testing.T) {
dir, err := ioutil.TempDir(os.TempDir(), "")
assert.NoError(t, err)
assert.DirExists(t, dir)
expected := make([]string, 0)
lastDir := dir
for i := 0; i < 5; i++ {
lastDir = filepath.Join(lastDir, strconv.Itoa(i))
assert.NoError(t, os.MkdirAll(lastDir, 0777))

ignoreFilename := filepath.Join(lastDir, ".gitignore")
file, err := os.Create(ignoreFilename)
assert.NoError(t, err)

content := fmt.Sprintf("%d.txt", i)
_, _ = file.WriteString(content)
assert.NoError(t, file.Close())

expected = append(expected, filepath.Join(lastDir, content))
expected = append(expected, ignoreFilename)
}
defer os.RemoveAll(dir)
lines := addRecursiveGitIgnores(".gitignore", []string{dir})

assert.EqualValues(t, expected, lines)
}

func BenchmarkIgnoreAddIgnoreFiles(b *testing.B) {
dir, err := ioutil.TempDir(os.TempDir(), "")
assert.NoError(b, err)
assert.DirExists(b, dir)
expected := []string{}

for i := 0; i < b.N; i++ {
newDir := filepath.Join(dir, strconv.Itoa(i))
assert.NoError(b, os.MkdirAll(newDir, 0777))

ignoreFilename := filepath.Join(newDir, ".gitignore")
file, err := os.Create(ignoreFilename)
assert.NoError(b, err)

content := fmt.Sprintf("%d.txt", i)
_, _ = file.WriteString(content)
assert.NoError(b, file.Close())

expected = append(expected, ignoreFilename)
expected = append(expected, filepath.Join(newDir, content))
}

defer os.RemoveAll(dir)
lines := addRecursiveGitIgnores(".gitignore", []string{dir})
sort.Strings(lines)
sort.Strings(expected)

assert.Equal(t, []string{"*.DS_Store"}, lines)
assert.EqualValues(b, expected, lines)
}
4 changes: 3 additions & 1 deletion pkg/parser/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (
"github.com/get-woke/woke/pkg/util"
)

var DefaultPath = []string{"."}

// Parser parses files and finds lines that break rules
type Parser struct {
Rules []*rule.Rule
Expand All @@ -36,7 +38,7 @@ func (p *Parser) ParsePaths(paths ...string) (results []result.FileResults, err
}

if len(paths) == 0 {
paths = []string{"."}
paths = DefaultPath
}

return p.processViolations(paths)
Expand Down
21 changes: 8 additions & 13 deletions pkg/parser/parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,7 @@ import (
)

func TestParser_ParsePaths(t *testing.T) {
i, err := ignore.NewIgnore()
assert.NoError(t, err)
p := NewParser(rule.DefaultRules, i)
p := NewParser(rule.DefaultRules, ignore.NewIgnore([]string{}, []string{}))

f1, err := newFile("i have a whitelist\n")
assert.NoError(t, err)
Expand Down Expand Up @@ -74,9 +72,8 @@ func TestParser_ParsePaths(t *testing.T) {
f4, err := newFile("i have a whitelist violation, but am ignored\n")
assert.NoError(t, err)
defer os.Remove(f4.Name())
i2, err := ignore.NewIgnore(f4.Name())
assert.NoError(t, err)
p.Ignorer = i2

p.Ignorer = ignore.NewIgnore([]string{f4.Name()}, []string{})
fr4, err := p.ParsePaths(f4.Name())
assert.NoError(t, err)
assert.Len(t, fr4, 0)
Expand Down Expand Up @@ -183,9 +180,8 @@ func BenchmarkParsePaths(b *testing.B) {
tmpFile.Close()

for i := 0; i < b.N; i++ {
i, err := ignore.NewIgnore()
assert.NoError(b, err)
p := NewParser(rule.DefaultRules, i)
ignorer := ignore.NewIgnore([]string{}, []string{})
p := NewParser(rule.DefaultRules, ignorer)
_, err = p.ParsePaths(tmpFile.Name())
assert.NoError(b, err)
}
Expand All @@ -195,10 +191,9 @@ func BenchmarkParsePathsRoot(b *testing.B) {
zerolog.SetGlobalLevel(zerolog.NoLevel)

for i := 0; i < b.N; i++ {
i, err := ignore.NewIgnore()
assert.NoError(b, err)
p := NewParser(rule.DefaultRules, i)
_, err = p.ParsePaths("../..")
Ignorer := ignore.NewIgnore([]string{}, []string{})
p := NewParser(rule.DefaultRules, Ignorer)
_, err := p.ParsePaths("../..")
assert.NoError(b, err)
}
}

0 comments on commit 51416cf

Please sign in to comment.