Git hashing for Git Fetching

The git fetcher is now more tree-hash-oriented, and we will want to integrate this with git fetching eventually. This PR exposes `treeHash` inputs and outputs in a few ways for this purpose. Eventually, we should add something like `builtins.derivation`'s `outputHashMode` to `builtins.fetchTree`, in order to specify we should use git hashing, and then this and the store-layer git hashing should meet together, ensuring we have the same tree hash end-to-end. Part of RFC 133 Co-Authored-By: Matthew Bauer <[email protected]> Co-Authored-By: Carlo Nucera <[email protected]> Co-authored-by: Robert Hensing <[email protected]>
NixOS · Mar 25, 2024 · 33e945b · 33e945b
1 parent 6d90287
commit 33e945b
Show file tree

Hide file tree

Showing 8 changed files with 197 additions and 52 deletions.
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
@@ -31,9 +31,13 @@ void emitTreeAttrs(
 
     // FIXME: support arbitrary input attributes.
 
-    auto narHash = input.getNarHash();
-    assert(narHash);
-    attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
+    if (auto narHash = input.getNarHash()) {
+        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
+    } else if (auto treeHash = input.getTreeHash()) {
+        attrs.alloc("treeHash").mkString(treeHash->to_string(HashFormat::SRI, true));
+    } else
+        /* Must have either tree hash or NAR hash */
+        assert(false);
 
     if (input.getType() == "git")
         attrs.alloc("submodules").mkBool(
@@ -51,6 +55,10 @@ void emitTreeAttrs(
             attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
         }
 
+        if (auto treeHash = input.getTreeHash()) {
+            attrs.alloc("treeHash").mkString(treeHash->gitRev());
+        }
+
         if (auto revCount = input.getRevCount())
             attrs.alloc("revCount").mkInt(*revCount);
         else if (emptyRevFallback)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
@@ -301,14 +301,19 @@ std::string Input::getName() const
 
 StorePath Input::computeStorePath(Store & store) const
 {
-    auto narHash = getNarHash();
-    if (!narHash)
-        throw Error("cannot compute store path for unlocked input '%s'", to_string());
-    return store.makeFixedOutputPath(getName(), FixedOutputInfo {
-        .method = FileIngestionMethod::Recursive,
-        .hash = *narHash,
-        .references = {},
-    });
+    if (auto treeHash = getTreeHash())
+        return store.makeFixedOutputPath(getName(), FixedOutputInfo {
+            .method = FileIngestionMethod::Git,
+            .hash = *treeHash,
+            .references = {},
+        });
+    if (auto narHash = getNarHash())
+        return store.makeFixedOutputPath(getName(), FixedOutputInfo {
+            .method = FileIngestionMethod::Recursive,
+            .hash = *narHash,
+            .references = {},
+        });
+    throw Error("cannot compute store path for unlocked input '%s'", to_string());
 }
 
 std::string Input::getType() const
@@ -351,6 +356,15 @@ std::optional<Hash> Input::getRev() const
     return hash;
 }
 
+std::optional<Hash> Input::getTreeHash() const
+{
+    if (auto s = maybeGetStrAttr(attrs, "treeHash")) {
+        experimentalFeatureSettings.require(Xp::GitHashing);
+        return Hash::parseAny(*s, HashAlgorithm::SHA1);
+    }
+    return {};
+}
+
 std::optional<uint64_t> Input::getRevCount() const
 {
     if (auto n = maybeGetIntAttr(attrs, "revCount"))

diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
@@ -121,6 +121,7 @@ public:
     std::optional<Hash> getNarHash() const;
     std::optional<std::string> getRef() const;
     std::optional<Hash> getRev() const;
+    std::optional<Hash> getTreeHash() const;
     std::optional<uint64_t> getRevCount() const;
     std::optional<time_t> getLastModified() const;
 

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
@@ -57,7 +57,7 @@ bool operator == (const git_oid & oid1, const git_oid & oid2)
 
 namespace nix {
 
-struct GitInputAccessor;
+struct GitInputAccessorImpl;
 
 // Some wrapper types that ensure that the git_*_free functions get called.
 template<auto del>
@@ -334,9 +334,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     }
 
     /**
-     * A 'GitInputAccessor' with no regard for export-ignore or any other transformations.
+     * A 'GitInputAccessorImpl' with no regard for export-ignore or any other transformations.
      */
-    ref<GitInputAccessor> getRawAccessor(const Hash & rev);
+    ref<GitInputAccessorImpl> getRawAccessor(const Hash & rev);
+
+    ref<GitInputAccessor> getPlainAccessor(const Hash & rev) override;
 
     ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) override;
 
@@ -477,17 +479,24 @@ ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create,
 /**
  * Raw git tree input accessor.
  */
-struct GitInputAccessor : InputAccessor
+struct GitInputAccessorImpl : GitInputAccessor
 {
     ref<GitRepoImpl> repo;
     Tree root;
 
-    GitInputAccessor(ref<GitRepoImpl> repo_, const Hash & rev)
+    GitInputAccessorImpl(ref<GitRepoImpl> repo_, const Hash & rev)
         : repo(repo_)
         , root(peelObject<Tree>(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE))
     {
     }
 
+    Hash getTreeHash() override
+    {
+        auto * oid = git_tree_id(root.get());
+        assert(oid);
+        return toHash(*oid);
+    }
+
     std::string readBlob(const CanonPath & path, bool symlink)
     {
         auto blob = getBlob(path, symlink);
@@ -922,17 +931,22 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
     }
 };
 
-ref<GitInputAccessor> GitRepoImpl::getRawAccessor(const Hash & rev)
+ref<GitInputAccessorImpl> GitRepoImpl::getRawAccessor(const Hash & rev)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    return make_ref<GitInputAccessor>(self, rev);
+    return make_ref<GitInputAccessorImpl>(self, rev);
+}
+
+ref<GitInputAccessor> GitRepoImpl::getPlainAccessor(const Hash & rev)
+{
+    return getRawAccessor(rev);
 }
 
 ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
 {
-    auto self = ref<GitRepoImpl>(shared_from_this());
-    ref<GitInputAccessor> rawGitAccessor = getRawAccessor(rev);
+    ref<GitInputAccessorImpl> rawGitAccessor = getRawAccessor(rev);
     if (exportIgnore) {
+        auto self = ref<GitRepoImpl>(shared_from_this());
         return make_ref<GitExportIgnoreInputAccessor>(self, rawGitAccessor, rev);
     }
     else {

diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
@@ -16,6 +16,16 @@ struct GitFileSystemObjectSink : FileSystemObjectSink
     virtual Hash sync() = 0;
 };
 
+/**
+ * Git Input Accessor
+ *
+ * Created from `GitRepo`. Support some additional operations.
+ */
+struct GitInputAccessor : InputAccessor
+{
+    virtual Hash getTreeHash() = 0;
+};
+
 struct GitRepo
 {
     virtual ~GitRepo()
@@ -75,6 +85,8 @@ struct GitRepo
 
     virtual bool hasObject(const Hash & oid) = 0;
 
+    virtual ref<GitInputAccessor> getPlainAccessor(const Hash & rev) = 0;
+
     virtual ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) = 0;
 
     virtual ref<InputAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;