Git hashing for Git Fetching

Now that we have Git hashing of (the FSOs of) store objects, we should teach the git fetcher how to use it. It can make especially good use of it because then we can use a single hash for both identifying what to fetch, and validating the download. Part of RFC 133 Extracted from our old IPFS branches. Co-Authored-By: Matthew Bauer <[email protected]> Co-Authored-By: Carlo Nucera <[email protected]> Co-authored-by: Robert Hensing <[email protected]>
NixOS · Mar 25, 2024 · f0d1e1a · f0d1e1a
1 parent 6d90287
commit f0d1e1a
Show file tree

Hide file tree

Showing 8 changed files with 196 additions and 52 deletions.
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
@@ -31,9 +31,13 @@ void emitTreeAttrs(
 
     // FIXME: support arbitrary input attributes.
 
-    auto narHash = input.getNarHash();
-    assert(narHash);
-    attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
+    if (auto narHash = input.getNarHash()) {
+        attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
+    } else if (auto treeHash = input.getTreeHash()) {
+        attrs.alloc("treeHash").mkString(treeHash->to_string(HashFormat::SRI, true));
+    } else
+        /* Must have either tree hash or NAR hash */
+        assert(false);
 
     if (input.getType() == "git")
         attrs.alloc("submodules").mkBool(
@@ -51,6 +55,10 @@ void emitTreeAttrs(
             attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
         }
 
+        if (auto treeHash = input.getTreeHash()) {
+            attrs.alloc("treeHash").mkString(treeHash->gitRev());
+        }
+
         if (auto revCount = input.getRevCount())
             attrs.alloc("revCount").mkInt(*revCount);
         else if (emptyRevFallback)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
@@ -301,14 +301,19 @@ std::string Input::getName() const
 
 StorePath Input::computeStorePath(Store & store) const
 {
-    auto narHash = getNarHash();
-    if (!narHash)
-        throw Error("cannot compute store path for unlocked input '%s'", to_string());
-    return store.makeFixedOutputPath(getName(), FixedOutputInfo {
-        .method = FileIngestionMethod::Recursive,
-        .hash = *narHash,
-        .references = {},
-    });
+    if (auto treeHash = getTreeHash())
+        return store.makeFixedOutputPath(getName(), FixedOutputInfo {
+            .method = FileIngestionMethod::Git,
+            .hash = *treeHash,
+            .references = {},
+        });
+    if (auto narHash = getNarHash())
+        return store.makeFixedOutputPath(getName(), FixedOutputInfo {
+            .method = FileIngestionMethod::Recursive,
+            .hash = *narHash,
+            .references = {},
+        });
+    throw Error("cannot compute store path for unlocked input '%s'", to_string());
 }
 
 std::string Input::getType() const
@@ -351,6 +356,15 @@ std::optional<Hash> Input::getRev() const
     return hash;
 }
 
+std::optional<Hash> Input::getTreeHash() const
+{
+    if (auto s = maybeGetStrAttr(attrs, "treeHash")) {
+        experimentalFeatureSettings.require(Xp::GitHashing);
+        return Hash::parseAny(*s, HashAlgorithm::SHA1);
+    }
+    return {};
+}
+
 std::optional<uint64_t> Input::getRevCount() const
 {
     if (auto n = maybeGetIntAttr(attrs, "revCount"))

diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
@@ -121,6 +121,7 @@ public:
     std::optional<Hash> getNarHash() const;
     std::optional<std::string> getRef() const;
     std::optional<Hash> getRev() const;
+    std::optional<Hash> getTreeHash() const;
     std::optional<uint64_t> getRevCount() const;
     std::optional<time_t> getLastModified() const;
 

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
@@ -57,7 +57,7 @@ bool operator == (const git_oid & oid1, const git_oid & oid2)
 
 namespace nix {
 
-struct GitInputAccessor;
+struct GitInputAccessorImpl;
 
 // Some wrapper types that ensure that the git_*_free functions get called.
 template<auto del>
@@ -334,9 +334,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     }
 
     /**
-     * A 'GitInputAccessor' with no regard for export-ignore or any other transformations.
+     * A 'GitInputAccessorImpl' with no regard for export-ignore or any other transformations.
      */
-    ref<GitInputAccessor> getRawAccessor(const Hash & rev);
+    ref<GitInputAccessorImpl> getRawAccessor(const Hash & rev);
+
+    ref<GitInputAccessor> getPlainAccessor(const Hash & rev) override;
 
     ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) override;
 
@@ -477,17 +479,24 @@ ref<GitRepo> GitRepo::openRepo(const std::filesystem::path & path, bool create,
 /**
  * Raw git tree input accessor.
  */
-struct GitInputAccessor : InputAccessor
+struct GitInputAccessorImpl : GitInputAccessor
 {
     ref<GitRepoImpl> repo;
     Tree root;
 
-    GitInputAccessor(ref<GitRepoImpl> repo_, const Hash & rev)
+    GitInputAccessorImpl(ref<GitRepoImpl> repo_, const Hash & rev)
         : repo(repo_)
         , root(peelObject<Tree>(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE))
     {
     }
 
+    Hash getTreeHash() override
+    {
+        auto * oid = git_tree_id(root.get());
+        assert(oid);
+        return toHash(*oid);
+    }
+
     std::string readBlob(const CanonPath & path, bool symlink)
     {
         auto blob = getBlob(path, symlink);
@@ -922,17 +931,22 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
     }
 };
 
-ref<GitInputAccessor> GitRepoImpl::getRawAccessor(const Hash & rev)
+ref<GitInputAccessorImpl> GitRepoImpl::getRawAccessor(const Hash & rev)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
-    return make_ref<GitInputAccessor>(self, rev);
+    return make_ref<GitInputAccessorImpl>(self, rev);
+}
+
+ref<GitInputAccessor> GitRepoImpl::getPlainAccessor(const Hash & rev)
+{
+    return getRawAccessor(rev);
 }
 
 ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
 {
-    auto self = ref<GitRepoImpl>(shared_from_this());
-    ref<GitInputAccessor> rawGitAccessor = getRawAccessor(rev);
+    ref<GitInputAccessorImpl> rawGitAccessor = getRawAccessor(rev);
     if (exportIgnore) {
+        auto self = ref<GitRepoImpl>(shared_from_this());
         return make_ref<GitExportIgnoreInputAccessor>(self, rawGitAccessor, rev);
     }
     else {

diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
@@ -16,6 +16,16 @@ struct GitFileSystemObjectSink : FileSystemObjectSink
     virtual Hash sync() = 0;
 };
 
+/**
+ * Git Input Accessor
+ *
+ * Created from `GitRepo`. Support some additional operations.
+ */
+struct GitInputAccessor : InputAccessor
+{
+    virtual Hash getTreeHash() = 0;
+};
+
 struct GitRepo
 {
     virtual ~GitRepo()
@@ -75,6 +85,8 @@ struct GitRepo
 
     virtual bool hasObject(const Hash & oid) = 0;
 
+    virtual ref<GitInputAccessor> getPlainAccessor(const Hash & rev) = 0;
+
     virtual ref<InputAccessor> getAccessor(const Hash & rev, bool exportIgnore) = 0;
 
     virtual ref<InputAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;