Merge pull request #9325 from NixOS/accessor-add-to-store

Content addressing and adding to store cleanup

perl/lib/Nix/Store.xs

@@ -13,6 +13,7 @@
 #include "globals.hh"
 #include "store-api.hh"
 #include "crypto.hh"
+#include "posix-source-accessor.hh"
 
 #include <sodium.h>
 #include <nlohmann/json.hpp>
@@ -205,7 +206,10 @@ void importPaths(int fd, int dontCheckSigs)
 SV * hashPath(char * algo, int base32, char * path)
     PPCODE:
         try {
-            Hash h = hashPath(parseHashAlgo(algo), path).first;
+            PosixSourceAccessor accessor;
+            Hash h = hashPath(
+                accessor, CanonPath::fromCwd(path),
+                FileIngestionMethod::Recursive, parseHashAlgo(algo)).first;
             auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
             XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
         } catch (Error & e) {
@@ -281,7 +285,11 @@ SV * addToStore(char * srcPath, int recursive, char * algo)
     PPCODE:
         try {
             auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo));
+            PosixSourceAccessor accessor;
+            auto path = store()->addToStore(
+                std::string(baseNameOf(srcPath)),
+                accessor, CanonPath::fromCwd(srcPath),
+                method, parseHashAlgo(algo));
             XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());

src/libcmd/installable-value.cc

@@ -44,7 +44,7 @@ ref<InstallableValue> InstallableValue::require(ref<Installable> installable)
 std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx)
 {
     if (v.type() == nPath) {
-        auto storePath = v.path().fetchToStore(state->store);
+        auto storePath = v.path().fetchToStore(*state->store);
         return {{
             .path = DerivedPath::Opaque {
                 .path = std::move(storePath),

src/libexpr/eval.cc

@@ -2317,7 +2317,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
     auto dstPath = i != srcToStore.end()
         ? i->second
         : [&]() {
-            auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
+            auto dstPath = path.fetchToStore(*store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
             allowPath(dstPath);
             srcToStore.insert_or_assign(path, dstPath);
             printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));

src/libexpr/primops.cc

@@ -2072,8 +2072,14 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val
     }
 
     auto storePath = settings.readOnlyMode
-        ? state.store->computeStorePathForText(name, contents, refs)
-        : state.store->addTextToStore(name, contents, refs, state.repair);
+        ? state.store->makeFixedOutputPathFromCA(name, TextInfo {
+            .hash = hashString(HashAlgorithm::SHA256, contents),
+            .references = std::move(refs),
+        })
+        : ({
+            StringSource s { contents };
+            state.store->addToStoreFromDump(s, name, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair);
+        });
 
     /* Note: we don't need to add `context' to the context of the
        result, since `storePath' itself has references to the paths
@@ -2229,7 +2235,7 @@ static void addPath(
             });
 
         if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
-            auto dstPath = path.fetchToStore(state.store, name, method, filter.get(), state.repair);
+            auto dstPath = path.fetchToStore(*state.store, name, method, filter.get(), state.repair);
             if (expectedHash && expectedStorePath != dstPath)
                 state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
             state.allowAndSetStorePathString(dstPath, v);

src/libfetchers/cache.cc

@@ -106,7 +106,7 @@ struct CacheImpl : Cache
     }
 
     void add(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs,
         const Attrs & infoAttrs,
         const StorePath & storePath,
@@ -115,13 +115,13 @@ struct CacheImpl : Cache
         _state.lock()->add.use()
             (attrsToJSON(inAttrs).dump())
             (attrsToJSON(infoAttrs).dump())
-            (store->printStorePath(storePath))
+            (store.printStorePath(storePath))
             (locked)
             (time(0)).exec();
     }
 
     std::optional<std::pair<Attrs, StorePath>> lookup(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) override
     {
         if (auto res = lookupExpired(store, inAttrs)) {
@@ -134,7 +134,7 @@ struct CacheImpl : Cache
     }
 
     std::optional<Result> lookupExpired(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) override
     {
         auto state(_state.lock());
@@ -148,19 +148,19 @@ struct CacheImpl : Cache
         }
 
         auto infoJSON = stmt.getStr(0);
-        auto storePath = store->parseStorePath(stmt.getStr(1));
+        auto storePath = store.parseStorePath(stmt.getStr(1));
         auto locked = stmt.getInt(2) != 0;
         auto timestamp = stmt.getInt(3);
 
-        store->addTempRoot(storePath);
-        if (!store->isValidPath(storePath)) {
+        store.addTempRoot(storePath);
+        if (!store.isValidPath(storePath)) {
             // FIXME: we could try to substitute 'storePath'.
             debug("ignoring disappeared cache entry '%s'", inAttrsJSON);
             return {};
         }
 
         debug("using cache entry '%s' -> '%s', '%s'",
-            inAttrsJSON, infoJSON, store->printStorePath(storePath));
+            inAttrsJSON, infoJSON, store.printStorePath(storePath));
 
         return Result {
             .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),

src/libfetchers/cache.hh

@@ -50,14 +50,14 @@ struct Cache
 
     /* Old cache for things that have a store path. */
     virtual void add(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs,
         const Attrs & infoAttrs,
         const StorePath & storePath,
         bool locked) = 0;
 
     virtual std::optional<std::pair<Attrs, StorePath>> lookup(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) = 0;
 
     struct Result
@@ -68,7 +68,7 @@ struct Cache
     };
 
     virtual std::optional<Result> lookupExpired(
-        ref<Store> store,
+        Store & store,
         const Attrs & inAttrs) = 0;
 };
 

src/libfetchers/fetchers.cc

@@ -374,7 +374,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
 std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
 {
     auto [accessor, input2] = getAccessor(store, input);
-    auto storePath = SourcePath(accessor).fetchToStore(store, input2.getName());
+    auto storePath = SourcePath(accessor).fetchToStore(*store, input2.getName());
     return {storePath, input2};
 }
 

src/libfetchers/git.cc

@@ -368,14 +368,14 @@ struct GitInputScheme : InputScheme
 
     RepoInfo getRepoInfo(const Input & input) const
     {
-        auto checkHashType = [&](const std::optional<Hash> & hash)
+        auto checkHashAlgorithm = [&](const std::optional<Hash> & hash)
         {
             if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256))
                 throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
         };
 
         if (auto rev = input.getRev())
-            checkHashType(rev);
+            checkHashAlgorithm(rev);
 
         RepoInfo repoInfo;
 

src/libfetchers/github.cc

@@ -201,7 +201,7 @@ struct GitArchiveInputScheme : InputScheme
             {"rev", rev->gitRev()},
         });
 
-        if (auto res = getCache()->lookup(store, lockedAttrs)) {
+        if (auto res = getCache()->lookup(*store, lockedAttrs)) {
             input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
             return {std::move(res->second), input};
         }
@@ -213,7 +213,7 @@ struct GitArchiveInputScheme : InputScheme
         input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
 
         getCache()->add(
-            store,
+            *store,
             lockedAttrs,
             {
                 {"rev", rev->gitRev()},

src/libfetchers/input-accessor.cc

@@ -5,10 +5,10 @@
 namespace nix {
 
 StorePath InputAccessor::fetchToStore(
-    ref<Store> store,
+    Store & store,
     const CanonPath & path,
     std::string_view name,
-    FileIngestionMethod method,
+    ContentAddressMethod method,
     PathFilter * filter,
     RepairFlag repair)
 {
@@ -20,10 +20,24 @@ StorePath InputAccessor::fetchToStore(
     if (!filter && fingerprint) {
         cacheKey = fetchers::Attrs{
             {"_what", "fetchToStore"},
-            {"store", store->storeDir},
+            {"store", store.storeDir},
             {"name", std::string(name)},
             {"fingerprint", *fingerprint},
-            {"method", (uint8_t) method},
+            {
+                "method",
+                std::visit(overloaded {
+                    [](const TextIngestionMethod &) {
+                        return "text";
+                    },
+                    [](const FileIngestionMethod & fim) {
+                        switch (fim) {
+                        case FileIngestionMethod::Flat: return "flat";
+                        case FileIngestionMethod::Recursive: return "nar";
+                        default: assert(false);
+                        }
+                    },
+                }, method.raw),
+            },
             {"path", path.abs()}
         };
         if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) {
@@ -35,17 +49,14 @@ StorePath InputAccessor::fetchToStore(
 
     Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
 
-    auto source = sinkToSource([&](Sink & sink) {
-        if (method == FileIngestionMethod::Recursive)
-            dumpPath(path, sink, filter ? *filter : defaultPathFilter);
-        else
-            readFile(path, sink);
-    });
+    auto filter2 = filter ? *filter : defaultPathFilter;
 
     auto storePath =
         settings.readOnlyMode
-        ? store->computeStorePathFromDump(*source, name, method, HashAlgorithm::SHA256).first
-        : store->addToStoreFromDump(*source, name, method, HashAlgorithm::SHA256, repair);
+        ? store.computeStorePath(
+            name, *this, path, method, HashAlgorithm::SHA256, {}, filter2).first
+        : store.addToStore(
+            name, *this, path, method, HashAlgorithm::SHA256, {}, filter2, repair);
 
     if (cacheKey)
         fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
@@ -60,9 +71,9 @@ std::ostream & operator << (std::ostream & str, const SourcePath & path)
 }
 
 StorePath SourcePath::fetchToStore(
-    ref<Store> store,
+    Store & store,
     std::string_view name,
-    FileIngestionMethod method,
+    ContentAddressMethod method,
     PathFilter * filter,
     RepairFlag repair) const
 {

src/libfetchers/input-accessor.hh

@@ -30,10 +30,10 @@ struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this<Inpu
     }
 
     StorePath fetchToStore(
-        ref<Store> store,
+        Store & store,
         const CanonPath & path,
         std::string_view name = "source",
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
         PathFilter * filter = nullptr,
         RepairFlag repair = NoRepair);
 };
@@ -116,9 +116,9 @@ struct SourcePath
      * Copy this `SourcePath` to the Nix store.
      */
     StorePath fetchToStore(
-        ref<Store> store,
+        Store & store,
         std::string_view name = "source",
-        FileIngestionMethod method = FileIngestionMethod::Recursive,
+        ContentAddressMethod method = FileIngestionMethod::Recursive,
         PathFilter * filter = nullptr,
         RepairFlag repair = NoRepair) const;
 

src/libfetchers/mercurial.cc

@@ -6,6 +6,7 @@
 #include "tarfile.hh"
 #include "store-api.hh"
 #include "url-parts.hh"
+#include "posix-source-accessor.hh"
 
 #include "fetch-settings.hh"
 
@@ -210,15 +211,20 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, filter);
+                PosixSourceAccessor accessor;
+                auto storePath = store->addToStore(
+                    input.getName(),
+                    accessor, CanonPath { actualPath },
+                    FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {},
+                    filter);
 
                 return {std::move(storePath), input};
             }
         }
 
         if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
 
-        auto checkHashType = [&](const std::optional<Hash> & hash)
+        auto checkHashAlgorithm = [&](const std::optional<Hash> & hash)
         {
             if (hash.has_value() && hash->algo != HashAlgorithm::SHA1)
                 throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true));
@@ -227,7 +233,7 @@ struct MercurialInputScheme : InputScheme
 
         auto getLockedAttrs = [&]()
         {
-            checkHashType(input.getRev());
+            checkHashAlgorithm(input.getRev());
 
             return Attrs({
                 {"type", "hg"},
@@ -246,7 +252,7 @@ struct MercurialInputScheme : InputScheme
         };
 
         if (input.getRev()) {
-            if (auto res = getCache()->lookup(store, getLockedAttrs()))
+            if (auto res = getCache()->lookup(*store, getLockedAttrs()))
                 return makeResult(res->first, std::move(res->second));
         }
 
@@ -259,7 +265,7 @@ struct MercurialInputScheme : InputScheme
             {"ref", *input.getRef()},
         });
 
-        if (auto res = getCache()->lookup(store, unlockedAttrs)) {
+        if (auto res = getCache()->lookup(*store, unlockedAttrs)) {
             auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1);
             if (!input.getRev() || input.getRev() == rev2) {
                 input.attrs.insert_or_assign("rev", rev2.gitRev());
@@ -305,7 +311,7 @@ struct MercurialInputScheme : InputScheme
         auto revCount = std::stoull(tokens[1]);
         input.attrs.insert_or_assign("ref", tokens[2]);
 
-        if (auto res = getCache()->lookup(store, getLockedAttrs()))
+        if (auto res = getCache()->lookup(*store, getLockedAttrs()))
             return makeResult(res->first, std::move(res->second));
 
         Path tmpDir = createTempDir();
@@ -315,7 +321,8 @@ struct MercurialInputScheme : InputScheme
 
         deletePath(tmpDir + "/.hg_archival.txt");
 
-        auto storePath = store->addToStore(name, tmpDir);
+        PosixSourceAccessor accessor;
+        auto storePath = store->addToStore(name, accessor, CanonPath { tmpDir });
 
         Attrs infoAttrs({
             {"rev", input.getRev()->gitRev()},
@@ -324,14 +331,14 @@ struct MercurialInputScheme : InputScheme
 
         if (!_input.getRev())
             getCache()->add(
-                store,
+                *store,
                 unlockedAttrs,
                 infoAttrs,
                 storePath,
                 false);
 
         getCache()->add(
-            store,
+            *store,
             getLockedAttrs(),
             infoAttrs,
             storePath,