Add option to include untracked files in git and mercurial fetchers

It can be confusing (especially for new users) that the git and
mercurial fetchers do not include untracked files. This commit adds an
option for the fetchers to include untracked files. The option is
disabled by default.

src/libfetchers/fetch-settings.hh

@@ -70,6 +70,9 @@ struct FetchSettings : public Config
     Setting<bool> warnDirty{this, true, "warn-dirty",
         "Whether to warn about dirty Git/Mercurial trees."};
 
+    Setting<bool> includeUntrackedFiles{this, false, "include-untracked-files",
+        "Whether to include untracked files in Git/Mercurial trees."};
+
     Setting<std::string> flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry",
         R"(
           Path or URI of the global flake registry.

src/libfetchers/git.cc

@@ -246,16 +246,25 @@ WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
         if (hasHead) {
             // Using git diff is preferrable over lower-level operations here,
             // because its conceptually simpler and we only need the exit code anyways.
-            auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"});
+            Strings gitDiffOpts;
+            if (fetchSettings.includeUntrackedFiles) {
+                gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "status", "--short"});
+            } else {
+                gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"});
+            }
             if (!submodules) {
                 // Changes in submodules should only make the tree dirty
                 // when those submodules will be copied as well.
                 gitDiffOpts.emplace_back("--ignore-submodules");
             }
             gitDiffOpts.emplace_back("--");
-            runProgram("git", true, gitDiffOpts);
 
-            clean = true;
+            if (fetchSettings.includeUntrackedFiles) {
+                clean = (chomp(runProgram("git", true, gitDiffOpts)) == "");
+            } else {
+                runProgram("git", true, gitDiffOpts);
+                clean = true;
+            }
         }
     } catch (ExecError & e) {
         if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
@@ -276,6 +285,11 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co
         warn("Git tree '%s' is dirty", workdir);
 
     auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" });
+    if (fetchSettings.includeUntrackedFiles) {
+        gitOpts.emplace_back("--cached");
+        gitOpts.emplace_back("--others");
+        gitOpts.emplace_back("--exclude-standard");
+    }
     if (submodules)
         gitOpts.emplace_back("--recurse-submodules");
 

src/libfetchers/mercurial.cc

@@ -175,7 +175,10 @@ struct MercurialInputScheme : InputScheme
 
         if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
 
-            bool clean = runHg({ "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
+            Strings hgStatusOptions = { "status", "-R", actualUrl, "--modified", "--added", "--removed" };
+            if (fetchSettings.includeUntrackedFiles)
+                hgStatusOptions.push_back("--unknown");
+            bool clean = runHg(hgStatusOptions) == "";
 
             if (!clean) {
 
@@ -190,8 +193,11 @@ struct MercurialInputScheme : InputScheme
 
                 input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl })));
 
+                hgStatusOptions = { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" };
+                if (fetchSettings.includeUntrackedFiles)
+                    hgStatusOptions.push_back("--unknown");
                 auto files = tokenizeString<std::set<std::string>>(
-                    runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
+                    runHg(hgStatusOptions), "\0"s);
 
                 Path actualPath(absPath(actualUrl));
 

tests/functional/fetchGit.sh

@@ -106,6 +106,25 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
 [ ! -e $path2/.git ]
 [[ $(cat $path2/dir1/foo) = foo ]]
 
+# Using an unclean tree with --include-git-untracked-files option should yield the tracked and uncommitted changes
+path21=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchGit $repo).outPath")
+[ ! -e $path21/hello ]
+[ -e $path21/bar ]
+[ -e $path21/dir2/bar ]
+[ ! -e $path21/.git ]
+[[ $(cat $path21/dir1/foo) = foo ]]
+
+# Using an unclean tree with --include-git-untracked-files option should take into account .gitignore files
+echo "/bar" >> $repo/.gitignore
+echo "bar" >> $repo/dir2/.gitignore
+path22=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchGit $repo).outPath")
+[ ! -e $path22/hello ]
+[ ! -e $path22/bar ]
+[ ! -e $path22/dir2/bar ]
+[ ! -e $path22/.git ]
+git -C $repo checkout -- $repo/.gitignore
+git -C $repo clean -fd
+
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyRev") = "${rev2}-dirty" ]]
 [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyShortRev") = "${rev2:0:7}-dirty" ]]

tests/functional/fetchMercurial.sh

@@ -1,6 +1,6 @@
 source common.sh
 
-[[ $(type -p hq) ]] || skipTest "Mercurial not installed"
+[[ $(type -p hg) ]] || skipTest "Mercurial not installed"
 
 clearStore
 
@@ -88,6 +88,22 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).outPath"
 [ ! -e $path2/.hg ]
 [[ $(cat $path2/dir1/foo) = foo ]]
 
+path21=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchMercurial $repo).outPath")
+[ ! -e $path21/hello ]
+[ -e $path21/bar ]
+[ -e $path21/dir2/bar ]
+[ ! -e $path21/.hg ]
+[[ $(cat $path21/dir1/foo) = foo ]]
+
+echo "bar" >> $repo/.hgignore
+path22=$(nix eval --include-untracked-files --impure --raw --expr "(builtins.fetchMercurial $repo).outPath")
+[ ! -e $path22/hello ]
+[ ! -e $path22/bar ]
+[ ! -e $path22/dir2/bar ]
+[ ! -e $path22/.git ]
+sed -i '/bar/d' $repo/.hgignore
+hg status --cwd $repo
+
 [[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]]
 
 # ... unless we're using an explicit ref.

tests/functional/flakes/mercurial.sh

@@ -1,6 +1,6 @@
 source ./common.sh
 
-[[ $(type -p hq) ]] || skipTest "Mercurial not installed"
+[[ $(type -p hg) ]] || skipTest "Mercurial not installed"
 
 flake1Dir=$TEST_ROOT/flake-hg1
 mkdir -p $flake1Dir