Organize content addressing, use SourceAccessor with `Store::addToS…

…tore`

Co-authored-by: Robert Hensing <[email protected]>

src/libcmd/installable-value.cc

@@ -44,7 +44,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = v.path().fetchToStore(state->store);
+        auto storePath = v.path().fetchToStore(*state->store);
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),

src/libexpr/eval.cc

@@ -2318,7 +2318,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
     auto dstPath = i != srcToStore.end()
         ? i->second
         : [&]() {
-            auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
+            auto dstPath = path.fetchToStore(*store, path.baseName(), FileIngestionMethod::Recursive, defaultPathFilter, repair);
             allowPath(dstPath);
             srcToStore.insert_or_assign(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));

src/libexpr/primops.cc

@@ -2242,7 +2242,7 @@ static void addPath(
             });
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair);
+            auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(*state.store, name, method, filter, state.repair);
             if (expectedHash && expectedStorePath != dstPath)
                 state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
             state.allowAndSetStorePathString(dstPath, v);

src/libfetchers/git.cc

@@ -8,6 +8,7 @@
 #include "pathlocks.hh"
 #include "processes.hh"
 #include "git.hh"
+#include "posix-source-accessor.hh"
 
 #include "fetch-settings.hh"
 
@@ -299,7 +300,8 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co
         return files.count(file);
     };
 
-    auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
+    PosixSourceAccessor accessor;
+    auto storePath = store->addToStore(input.getName(), accessor, CanonPath { actualPath }, FileIngestionMethod::Recursive, htSHA256, {}, filter);
 
     // FIXME: maybe we should use the timestamp of the last
     // modified dirty file?
@@ -791,7 +793,8 @@ struct GitInputScheme : InputScheme
             unpackTarfile(*source, tmpDir);
         }
 
-        auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
+        PosixSourceAccessor accessor;
+        auto storePath = store->addToStore(name, accessor, CanonPath { tmpDir }, FileIngestionMethod::Recursive, htSHA256, {}, filter);
 
         auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
 

src/libfetchers/input-accessor.cc

@@ -3,31 +3,6 @@
 
 namespace nix {
 
-StorePath InputAccessor::fetchToStore(
-    ref<Store> store,
-    const CanonPath & path,
-    std::string_view name,
-    FileIngestionMethod method,
-    PathFilter * filter,
-    RepairFlag repair)
-{
-    Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
-
-    auto source = sinkToSource([&](Sink & sink) {
-        if (method == FileIngestionMethod::Recursive)
-            dumpPath(path, sink, filter ? *filter : defaultPathFilter);
-        else
-            readFile(path, sink);
-    });
-
-    auto storePath =
-        settings.readOnlyMode
-        ? store->computeStorePathFromDump(*source, name, method, htSHA256).first
-        : store->addToStoreFromDump(*source, name, method, htSHA256, repair);
-
-    return storePath;
-}
-
 SourcePath InputAccessor::root()
 {
     return {ref(shared_from_this()), CanonPath::root};
@@ -40,13 +15,18 @@ std::ostream & operator << (std::ostream & str, const SourcePath & path)
 }
 
 StorePath SourcePath::fetchToStore(
-    ref<Store> store,
+    Store & store,
     std::string_view name,
-    FileIngestionMethod method,
-    PathFilter * filter,
+    ContentAddressMethod method,
+    PathFilter & filter,
     RepairFlag repair) const
 {
-    return accessor->fetchToStore(store, path, name, method, filter, repair);
+    return
+        settings.readOnlyMode
+        ? store.computeStorePath(
+            name, *accessor, path, method, htSHA256, {}, filter).first
+        : store.addToStore(
+            name, *accessor, path, method, htSHA256, {}, filter, repair);
 }
 
 std::string_view SourcePath::baseName() const

src/libfetchers/input-accessor.hh

@@ -27,14 +27,6 @@ struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<Inpu
         return std::nullopt;
     }
 
-    StorePath fetchToStore(
-        ref<Store> store,
-        const CanonPath & path,
-        std::string_view name = "source",
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
-        PathFilter * filter = nullptr,
-        RepairFlag repair = NoRepair);
-
     SourcePath root();
 };
 
@@ -111,10 +103,10 @@ struct SourcePath
      * Copy this `SourcePath` to the Nix store.
      */
     StorePath fetchToStore(
-        ref<Store> store,
+        Store & store,
         std::string_view name = "source",
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
-        PathFilter * filter = nullptr,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
+        PathFilter & filter = defaultPathFilter,
         RepairFlag repair = NoRepair) const;
 
     /**

src/libfetchers/mercurial.cc

@@ -6,6 +6,7 @@
 #include "tarfile.hh"
 #include "store-api.hh"
 #include "url-parts.hh"
+#include "posix-source-accessor.hh"
 
 #include "fetch-settings.hh"
 
@@ -210,7 +211,12 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
+                PosixSourceAccessor accessor;
+                auto storePath = store->addToStore(
+                    input.getName(),
+                    accessor, CanonPath { actualPath },
+                    FileIngestionMethod::Recursive, htSHA256, {},
+                    filter);
 
                 return {std::move(storePath), input};
             }
@@ -315,7 +321,8 @@ struct MercurialInputScheme : InputScheme
 
         deletePath(tmpDir + "/.hg_archival.txt");
 
-        auto storePath = store->addToStore(name, tmpDir);
+        PosixSourceAccessor accessor;
+        auto storePath = store->addToStore(name, accessor, CanonPath { tmpDir });
 
         Attrs infoAttrs({
             {"rev", input.getRev()->gitRev()},

src/libfetchers/tarball.cc

@@ -8,6 +8,7 @@
 #include "tarfile.hh"
 #include "types.hh"
 #include "split.hh"
+#include "posix-source-accessor.hh"
 
 namespace nix::fetchers {
 
@@ -156,7 +157,8 @@ DownloadTarballResult downloadTarball(
             throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
         auto topDir = tmpDir + "/" + members.begin()->name;
         lastModified = lstat(topDir).st_mtime;
-        unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
+        PosixSourceAccessor accessor;
+        unpackedStorePath = store->addToStore(name, accessor, CanonPath { topDir }, FileIngestionMethod::Recursive, htSHA256, {}, defaultPathFilter, NoRepair);
     }
 
     Attrs infoAttrs({

src/libstore/binary-cache-store.cc

@@ -300,24 +300,28 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
     }});
 }
 
-StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name,
-    FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references)
+StorePath BinaryCacheStore::addToStoreFromDump(
+    Source & dump,
+    std::string_view name,
+    ContentAddressMethod method,
+    HashType hashAlgo,
+    const StorePathSet & references,
+    RepairFlag repair)
 {
     if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256)
         unsupported("addToStoreFromDump");
     return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             name,
-            FixedOutputInfo {
-                .method = method,
-                .hash = nar.first,
-                .references = {
+            ContentAddressWithReferences::fromParts(
+                method,
+                nar.first,
+                {
                     .others = references,
                     // caller is not capable of creating a self-reference, because this is content-addressed without modulus
                     .self = false,
-                },
-            },
+                }),
             nar.first,
         };
         info.narSize = nar.second;
@@ -400,41 +404,35 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath,
 
 StorePath BinaryCacheStore::addToStore(
     std::string_view name,
-    const Path & srcPath,
-    FileIngestionMethod method,
+    SourceAccessor & accessor,
+    const CanonPath & path,
+    ContentAddressMethod method,
     HashType hashAlgo,
+    const StorePathSet & references,
     PathFilter & filter,
-    RepairFlag repair,
-    const StorePathSet & references)
+    RepairFlag repair)
 {
     /* FIXME: Make BinaryCacheStore::addToStoreCommon support
        non-recursive+sha256 so we can just use the default
        implementation of this method in terms of addToStoreFromDump. */
 
-    HashSink sink { hashAlgo };
-    if (method == FileIngestionMethod::Recursive) {
-        dumpPath(srcPath, sink, filter);
-    } else {
-        readFile(srcPath, sink);
-    }
-    auto h = sink.finish().first;
+    auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first;
 
     auto source = sinkToSource([&](Sink & sink) {
-        dumpPath(srcPath, sink, filter);
+        accessor.dumpPath(path, sink, filter);
     });
     return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) {
         ValidPathInfo info {
             *this,
             name,
-            FixedOutputInfo {
-                .method = method,
-                .hash = h,
-                .references = {
+            ContentAddressWithReferences::fromParts(
+                method,
+                h,
+                {
                     .others = references,
                     // caller is not capable of creating a self-reference, because this is content-addressed without modulus
                     .self = false,
-                },
-            },
+                }),
             nar.first,
         };
         info.narSize = nar.second;

src/libstore/binary-cache-store.hh

@@ -123,17 +123,23 @@ public:
     void addToStore(const ValidPathInfo & info, Source & narSource,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
-    StorePath addToStoreFromDump(Source & dump, std::string_view name,
-        FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override;
+    StorePath addToStoreFromDump(
+        Source & dump,
+        std::string_view name,
+        ContentAddressMethod method,
+        HashType hashAlgo,
+        const StorePathSet & references,
+        RepairFlag repair) override;
 
     StorePath addToStore(
         std::string_view name,
-        const Path & srcPath,
-        FileIngestionMethod method,
+        SourceAccessor & accessor,
+        const CanonPath & srcPath,
+        ContentAddressMethod method,
         HashType hashAlgo,
+        const StorePathSet & references,
         PathFilter & filter,
-        RepairFlag repair,
-        const StorePathSet & references) override;
+        RepairFlag repair) override;
 
     StorePath addTextToStore(
         std::string_view name,