From d7fba91bc3894a252e17fef5d92e486dd66f7b00 Mon Sep 17 00:00:00 2001 From: David Lynch Date: Thu, 12 Nov 2015 15:41:13 -0800 Subject: [PATCH 1/7] Use scandal to build the project file list Atom core has scandal, which is _all about_ building a list of files in a fairly efficient manner, and thinking about VCS issues and symbolic links and whatnot. Instead of reimplementing pieces of that, just use it. This has the nice side-effect of fixing issue #111, because scandal is better at deciding which files should be ignored. --- lib/load-paths-handler.coffee | 133 ++++++++++------------------------ lib/path-loader.coffee | 14 ++-- package.json | 1 + 3 files changed, 46 insertions(+), 102 deletions(-) diff --git a/lib/load-paths-handler.coffee b/lib/load-paths-handler.coffee index 9d29a2a3..ca3c8552 100644 --- a/lib/load-paths-handler.coffee +++ b/lib/load-paths-handler.coffee @@ -1,107 +1,48 @@ async = require 'async' -fs = require 'fs' path = require 'path' _ = require 'underscore-plus' -{GitRepository} = require 'atom' -{Minimatch} = require 'minimatch' +{PathSearcher, PathScanner, search} = require 'scandal' -PathsChunkSize = 100 +module.exports = (rootPaths, options={}) -> + pathsSearched = 0 + PATHS_COUNTER_SEARCHED_CHUNK = 50 -emittedPaths = new Set - -class PathLoader - constructor: (@rootPath, ignoreVcsIgnores, @traverseSymlinkDirectories, @ignoredNames) -> - @paths = [] - @realPathCache = {} - @repo = null - if ignoreVcsIgnores - repo = GitRepository.open(@rootPath, refreshOnWindowFocus: false) - @repo = repo if repo?.relativize(path.join(@rootPath, 'test')) is 'test' - - load: (done) -> - @loadPath @rootPath, => - @flushPaths() - @repo?.destroy() - done() - - isIgnored: (loadedPath) -> - relativePath = path.relative(@rootPath, loadedPath) - if @repo?.isPathIgnored(relativePath) - true - else - for ignoredName in @ignoredNames - return true if ignoredName.match(relativePath) - - pathLoaded: (loadedPath, done) -> - unless @isIgnored(loadedPath) or emittedPaths.has(loadedPath) - @paths.push(loadedPath) - emittedPaths.add(loadedPath) - - if @paths.length is PathsChunkSize - @flushPaths() - done() - - flushPaths: -> - emit('load-paths:paths-found', @paths) - @paths = [] - - loadPath: (pathToLoad, done) -> - return done() if @isIgnored(pathToLoad) - fs.lstat pathToLoad, (error, stats) => - return done() if error? - if stats.isSymbolicLink() - @isInternalSymlink pathToLoad, (isInternal) => - return done() if isInternal - fs.stat pathToLoad, (error, stats) => - return done() if error? - if stats.isFile() - @pathLoaded(pathToLoad, done) - else if stats.isDirectory() - if @traverseSymlinkDirectories - @loadFolder(pathToLoad, done) - else - done() - else - done() - else if stats.isDirectory() - @loadFolder(pathToLoad, done) - else if stats.isFile() - @pathLoaded(pathToLoad, done) - else - done() + async.each( + rootPaths, + (rootPath, next) -> + options2 = _.extend {}, options, + inclusions: processPaths(rootPath, options.inclusions) + globalExclusions: processPaths(rootPath, options.globalExclusions) - loadFolder: (folderPath, done) -> - fs.readdir folderPath, (error, children=[]) => - async.each( - children, - (childName, next) => - @loadPath(path.join(folderPath, childName), next) - done - ) + scanner = new PathScanner(rootPath, options2) - isInternalSymlink: (pathToLoad, done) -> - fs.realpath pathToLoad, @realPathCache, (err, realPath) => - if err - done(false) - else - done(realPath.search(@rootPath) is 0) + paths = [] -module.exports = (rootPaths, followSymlinks, ignoreVcsIgnores, ignores=[]) -> - ignoredNames = [] - for ignore in ignores when ignore - try - ignoredNames.push(new Minimatch(ignore, matchBase: true, dot: true)) - catch error - console.warn "Error parsing ignore pattern (#{ignore}): #{error.message}" + scanner.on 'path-found', (path) -> + paths.push path + pathsSearched++ + if pathsSearched % PATHS_COUNTER_SEARCHED_CHUNK is 0 + emit('load-paths:paths-found', paths) + paths = [] + scanner.on 'finished-scanning', -> + emit('load-paths:paths-found', paths) + next() + scanner.scan() - async.each( - rootPaths, - (rootPath, next) -> - new PathLoader( - rootPath, - ignoreVcsIgnores, - followSymlinks, - ignoredNames - ).load(next) @async() ) + +processPaths = (rootPath, paths) -> + return paths unless paths?.length > 0 + rootPathBase = path.basename(rootPath) + results = [] + for givenPath in paths + segments = givenPath.split(path.sep) + firstSegment = segments.shift() + results.push(givenPath) + if firstSegment is rootPathBase + if segments.length is 0 + results.push(path.join("**", "*")) + else + results.push(path.join(segments...)) + results diff --git a/lib/path-loader.coffee b/lib/path-loader.coffee index 972c11bd..4a61d9c2 100644 --- a/lib/path-loader.coffee +++ b/lib/path-loader.coffee @@ -4,18 +4,20 @@ module.exports = startTask: (callback) -> projectPaths = [] taskPath = require.resolve('./load-paths-handler') - followSymlinks = atom.config.get 'core.followSymlinks' + ignoredNames = atom.config.get('fuzzy-finder.ignoredNames') ? [] ignoredNames = ignoredNames.concat(atom.config.get('core.ignoredNames') ? []) - ignoreVcsIgnores = atom.config.get('core.excludeVcsIgnoredPaths') + + options = + excludeVcsIgnores: atom.config.get 'core.excludeVcsIgnoredPaths', + exclusions: ignoredNames, + follow: atom.config.get 'core.followSymlinks', task = Task.once( taskPath, atom.project.getPaths(), - followSymlinks, - ignoreVcsIgnores, - ignoredNames, -> - callback(projectPaths) + options, + -> callback(projectPaths) ) task.on 'load-paths:paths-found', (paths) -> diff --git a/package.json b/package.json index 9225e149..7ead070e 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "fuzzaldrin-plus": "^0.1", "humanize-plus": "~1.4.2", "minimatch": "~0.3.0", + "scandal": "^2.2", "temp": "~0.8.1", "underscore-plus": "^1.0.0", "wrench": "^1.5" From 0653fa7374e56deb1aed3c83b8e83fe782f21d12 Mon Sep 17 00:00:00 2001 From: David Lynch Date: Thu, 12 Nov 2015 16:27:07 -0800 Subject: [PATCH 2/7] Restore double-display protection --- lib/load-paths-handler.coffee | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/load-paths-handler.coffee b/lib/load-paths-handler.coffee index ca3c8552..b7bc9e84 100644 --- a/lib/load-paths-handler.coffee +++ b/lib/load-paths-handler.coffee @@ -5,7 +5,8 @@ _ = require 'underscore-plus' module.exports = (rootPaths, options={}) -> pathsSearched = 0 - PATHS_COUNTER_SEARCHED_CHUNK = 50 + PATHS_COUNTER_SEARCHED_CHUNK = 100 + emittedPaths = new Set async.each( rootPaths, @@ -19,8 +20,10 @@ module.exports = (rootPaths, options={}) -> paths = [] scanner.on 'path-found', (path) -> - paths.push path - pathsSearched++ + unless emittedPaths.has(path) + paths.push path + emittedPaths.add(path) + pathsSearched++ if pathsSearched % PATHS_COUNTER_SEARCHED_CHUNK is 0 emit('load-paths:paths-found', paths) paths = [] From a434fa08c2d673a5b0f925b6f657510e74b33068 Mon Sep 17 00:00:00 2001 From: David Lynch Date: Thu, 12 Nov 2015 16:27:27 -0800 Subject: [PATCH 3/7] Coffeescript object syntax mistake --- lib/path-loader.coffee | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/path-loader.coffee b/lib/path-loader.coffee index 4a61d9c2..960a143a 100644 --- a/lib/path-loader.coffee +++ b/lib/path-loader.coffee @@ -9,9 +9,9 @@ module.exports = ignoredNames = ignoredNames.concat(atom.config.get('core.ignoredNames') ? []) options = - excludeVcsIgnores: atom.config.get 'core.excludeVcsIgnoredPaths', - exclusions: ignoredNames, - follow: atom.config.get 'core.followSymlinks', + excludeVcsIgnores: atom.config.get 'core.excludeVcsIgnoredPaths' + exclusions: ignoredNames + follow: atom.config.get 'core.followSymlinks' task = Task.once( taskPath, From 2ebb00116aeaa4b1f40eb352db2d638f8cb1adff Mon Sep 17 00:00:00 2001 From: David Lynch Date: Thu, 12 Nov 2015 17:13:27 -0800 Subject: [PATCH 4/7] Override Vcs exclusion if we're in a repo subdirectory It's arguable which way this should go; tree-view and find-in-project both treat a parent gitignore as authorative, but fuzzy-finder current tests require that it be ignored. As such, I've implemented this in a way that'd make it easy to switch the behavior around. --- lib/load-paths-handler.coffee | 6 ++++++ lib/path-loader.coffee | 1 + package.json | 1 + 3 files changed, 8 insertions(+) diff --git a/lib/load-paths-handler.coffee b/lib/load-paths-handler.coffee index b7bc9e84..0f670d9a 100644 --- a/lib/load-paths-handler.coffee +++ b/lib/load-paths-handler.coffee @@ -1,6 +1,7 @@ async = require 'async' path = require 'path' _ = require 'underscore-plus' +GitUtils = require 'git-utils' {PathSearcher, PathScanner, search} = require 'scandal' module.exports = (rootPaths, options={}) -> @@ -15,6 +16,11 @@ module.exports = (rootPaths, options={}) -> inclusions: processPaths(rootPath, options.inclusions) globalExclusions: processPaths(rootPath, options.globalExclusions) + if options.ignoreProjectParentVcsIgnores + repo = GitUtils.open(rootPath) + if repo and '' != repo.relativize(rootPath) + options2.excludeVcsIgnores = false + scanner = new PathScanner(rootPath, options2) paths = [] diff --git a/lib/path-loader.coffee b/lib/path-loader.coffee index 960a143a..437322b7 100644 --- a/lib/path-loader.coffee +++ b/lib/path-loader.coffee @@ -12,6 +12,7 @@ module.exports = excludeVcsIgnores: atom.config.get 'core.excludeVcsIgnoredPaths' exclusions: ignoredNames follow: atom.config.get 'core.followSymlinks' + ignoreProjectParentVcsIgnores: true task = Task.once( taskPath, diff --git a/package.json b/package.json index 7ead070e..5ca5ae74 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "fs-plus": "^2.0.0", "fuzzaldrin": "^2.0", "fuzzaldrin-plus": "^0.1", + "git-utils": "^4.0", "humanize-plus": "~1.4.2", "minimatch": "~0.3.0", "scandal": "^2.2", From e144abf7bec448a11c80e20117e81a998b409c4e Mon Sep 17 00:00:00 2001 From: David Lynch Date: Thu, 12 Nov 2015 17:17:08 -0800 Subject: [PATCH 5/7] Hidden files should be included --- lib/path-loader.coffee | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/path-loader.coffee b/lib/path-loader.coffee index 437322b7..5d665424 100644 --- a/lib/path-loader.coffee +++ b/lib/path-loader.coffee @@ -12,6 +12,7 @@ module.exports = excludeVcsIgnores: atom.config.get 'core.excludeVcsIgnoredPaths' exclusions: ignoredNames follow: atom.config.get 'core.followSymlinks' + includeHidden: true ignoreProjectParentVcsIgnores: true task = Task.once( From ad5e4ebe19042620e0e24367183b25e56951ae32 Mon Sep 17 00:00:00 2001 From: David Lynch Date: Thu, 12 Nov 2015 17:21:44 -0800 Subject: [PATCH 6/7] Style guide --- lib/load-paths-handler.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/load-paths-handler.coffee b/lib/load-paths-handler.coffee index 0f670d9a..eae536a7 100644 --- a/lib/load-paths-handler.coffee +++ b/lib/load-paths-handler.coffee @@ -18,7 +18,7 @@ module.exports = (rootPaths, options={}) -> if options.ignoreProjectParentVcsIgnores repo = GitUtils.open(rootPath) - if repo and '' != repo.relativize(rootPath) + if repo and '' isnt repo.relativize(rootPath) options2.excludeVcsIgnores = false scanner = new PathScanner(rootPath, options2) From db0e62cfc2ae094977a8fbc8a7cf29531a42615b Mon Sep 17 00:00:00 2001 From: David Lynch Date: Thu, 3 Dec 2015 14:28:01 -0600 Subject: [PATCH 7/7] Add a spec for #111: nested repos with the parent ignoring the child --- .../git/working-dir/another-repo/c.txt | 0 .../git/working-dir/another-repo/d.txt | 0 .../git/working-dir/another-repo/git.git/HEAD | 1 + .../working-dir/another-repo/git.git/config | 6 ++++ .../working-dir/another-repo/git.git/index | Bin 0 -> 190 bytes .../28/569d0a51e27dd112c0d4994c1e2914dd0db754 | Bin 0 -> 79 bytes .../65/a457425a679cbe9adf0d2741785d3ceabb44a7 | Bin 0 -> 50 bytes .../97/ff2919c02606bcad55588f3fa723b5a357470f | Bin 0 -> 164 bytes .../e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 | Bin 0 -> 15 bytes .../ec/5e386905ff2d36e291086a1207f2585aaa8920 | Bin 0 -> 50 bytes .../ef/046e9eecaa5255ea5e9817132d4001724d6ae1 | Bin 0 -> 132 bytes .../another-repo/git.git/refs/heads/master | 1 + spec/fuzzy-finder-spec.coffee | 30 +++++++++++++++++- 13 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/c.txt create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/d.txt create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/HEAD create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/config create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/index create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/objects/28/569d0a51e27dd112c0d4994c1e2914dd0db754 create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/objects/65/a457425a679cbe9adf0d2741785d3ceabb44a7 create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/objects/97/ff2919c02606bcad55588f3fa723b5a357470f create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/objects/ec/5e386905ff2d36e291086a1207f2585aaa8920 create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/objects/ef/046e9eecaa5255ea5e9817132d4001724d6ae1 create mode 100644 spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/refs/heads/master diff --git a/spec/fixtures/root-dir1/git/working-dir/another-repo/c.txt b/spec/fixtures/root-dir1/git/working-dir/another-repo/c.txt new file mode 100644 index 00000000..e69de29b diff --git a/spec/fixtures/root-dir1/git/working-dir/another-repo/d.txt b/spec/fixtures/root-dir1/git/working-dir/another-repo/d.txt new file mode 100644 index 00000000..e69de29b diff --git a/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/HEAD b/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/HEAD new file mode 100644 index 00000000..cb089cd8 --- /dev/null +++ b/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/config b/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/config new file mode 100644 index 00000000..af107929 --- /dev/null +++ b/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/config @@ -0,0 +1,6 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = false + logallrefupdates = true + ignorecase = true diff --git a/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/index b/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/index new file mode 100644 index 0000000000000000000000000000000000000000..67a98bd9f09cf53f4227ae393d1be5732e0147bd GIT binary patch literal 190 zcmZ?q402{*U|<5_pcvIxK$-zYGcqu+uxKsc!@$tEgn@zaD^N-VB>!ygquHA-UUT;D z)?8N}b>q;Jp5TcLtciLh6(t~PpnkPlWEx`5is?j|!gf0V^p=O;s>6W-v4`Ff%bxNYpE-C}DUu_tET47q2;ccWbUIkGgT_Nl)-Z l69XVnNXaZ>coS!l$@*W{?9oJyEFtz!5mBo;6#&w592%mGA720f literal 0 HcmV?d00001 diff --git a/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/objects/65/a457425a679cbe9adf0d2741785d3ceabb44a7 b/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/objects/65/a457425a679cbe9adf0d2741785d3ceabb44a7 new file mode 100644 index 0000000000000000000000000000000000000000..ba1f06fc0e5703e15be021b473eb64a7063688ae GIT binary patch literal 50 zcmbj3FtLHCl%f)r=TpQFIL;hKlZU=9 z6>Jk{byNi#eVYW?liDCcWHX9O$>m8Tav~^);GMbgsMiX5MAvFk&CK3UkJpr_XYQJ--hZ??^;|BCJoIazW@-eCF~J-IqM7XUS@r*d S*@}Raad}wOcQqf2L`$(5(@Y8g literal 0 HcmV?d00001 diff --git a/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 b/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 new file mode 100644 index 0000000000000000000000000000000000000000..711223894375fe1186ac5bfffdc48fb1fa1e65cc GIT binary patch literal 15 Wcmb2rPLQ6n?I4({XK>N literal 0 HcmV?d00001 diff --git a/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/refs/heads/master b/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/refs/heads/master new file mode 100644 index 00000000..99d72600 --- /dev/null +++ b/spec/fixtures/root-dir1/git/working-dir/another-repo/git.git/refs/heads/master @@ -0,0 +1 @@ +97ff2919c02606bcad55588f3fa723b5a357470f diff --git a/spec/fuzzy-finder-spec.coffee b/spec/fuzzy-finder-spec.coffee index 4e0464d2..feacdd96 100644 --- a/spec/fuzzy-finder-spec.coffee +++ b/spec/fuzzy-finder-spec.coffee @@ -595,7 +595,7 @@ describe 'FuzzyFinder', -> it "passes the indexed paths into the project view when it is created", -> {projectPaths} = fuzzyFinder - expect(projectPaths.length).toBe 18 + expect(projectPaths.length).toBe 30 projectView = fuzzyFinder.createProjectView() expect(projectView.paths).toBe projectPaths expect(projectView.reloadPaths).toBe false @@ -876,6 +876,7 @@ describe 'FuzzyFinder', -> beforeEach -> projectPath = atom.project.getDirectories()[0].resolve('git/working-dir') fs.moveSync(path.join(projectPath, 'git.git'), path.join(projectPath, '.git')) + fs.moveSync(path.join(projectPath, 'another-repo', 'git.git'), path.join(projectPath, 'another-repo', '.git')) atom.project.setPaths([rootDir2, projectPath]) gitDirectory = atom.project.getDirectories()[1] @@ -996,3 +997,30 @@ describe 'FuzzyFinder', -> runs -> expect(projectView.list.find("li:contains(file.txt)")).toExist() + + describe "when the project's path is a repository inside another repository", -> + beforeEach -> + ignoreFile = path.join(projectPath, '.gitignore') + fs.writeFileSync(ignoreFile, '*') + atom.project.setPaths([gitDirectory.resolve('another-repo')]) + + ignoreFile = path.join(gitDirectory.resolve('another-repo'), '.gitignore') + fs.writeFileSync(ignoreFile, 'd.txt') + + it "excludes paths that are git ignored in the child repository", -> + dispatchCommand('toggle-file-finder') + projectView.setMaxItems(Infinity) + + waitForPathsToDisplay(projectView) + + runs -> + expect(projectView.list.find("li:contains(d.txt)")).not.toExist() + + it "does not exclude the entire child repository if the parent repository is ignoring it", -> + dispatchCommand('toggle-file-finder') + projectView.setMaxItems(Infinity) + + waitForPathsToDisplay(projectView) + + runs -> + expect(projectView.list.find("li:contains(c.txt)")).toExist()