From 1fe9512a42474bed9983c6dfc252961d89c3b75f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Jan 2023 23:55:15 -0500 Subject: [PATCH] Allow computed drvs in `inputDrvs` We use the same nested map representation we used for goals, in order to save space and be backwards compatible. --- src/build-remote/build-remote.cc | 2 +- src/libexpr/primops.cc | 12 +- src/libstore/build/derivation-goal.cc | 71 +++++++--- src/libstore/derivations.cc | 194 ++++++++++++++++++++------ src/libstore/derivations.hh | 7 +- src/libstore/derived-path-map.cc | 15 +- src/libstore/derived-path-map.hh | 18 ++- src/libstore/misc.cc | 69 ++++++--- src/libstore/tests/derivation.cc | 25 ++-- src/libutil/comparator.hh | 77 ++++++---- src/nix-build/nix-build.cc | 37 +++-- src/nix/app.cc | 40 +++++- tests/dyn-drv/dep-built-drv.sh | 4 +- 13 files changed, 423 insertions(+), 148 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index c1f03a8ef4a9..d69d3a0c263f 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -314,7 +314,7 @@ static int main_build_remote(int argc, char * * argv) // // 2. Changing the `inputSrcs` set changes the associated // output ids, which break CA derivations - if (!drv.inputDrvs.empty()) + if (!drv.inputDrvs.map.empty()) drv.inputSrcs = store->parseStorePathSet(inputs); optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); auto & result = *optResult; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index b8fba53c7c51..fd60b1158d83 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1248,15 +1248,15 @@ drvName, Bindings * attrs, Value & v) state.store->computeFSClosure(d.drvPath, refs); for (auto & j : refs) { drv.inputSrcs.insert(j); - if (j.isDerivation()) - drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); + if (j.isDerivation()) { + drv.inputDrvs.map[j] = DerivedPathMap::Node { + .value = state.store->readDerivation(j).outputNames(), + }; + } } }, [&](const NixStringContextElem::Built & b) { - if (auto * p = std::get_if(&*b.drvPath)) - drv.inputDrvs[p->path].insert(b.output); - else - throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported"); + drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output); }, [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 966b93b35cd1..a7d25d42e00c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -351,24 +351,37 @@ void DerivationGoal::gaveUpOnSubstitution() /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) { + { + std::function, const DerivedPathMap::Node &)> accumDerivedPath; + + accumDerivedPath = [&](ref inputDrv, const DerivedPathMap::Node & inputNode) { + if (!inputNode.value.empty()) + addWaitee(worker.makeGoal( + DerivedPath::Built { + .drvPath = inputDrv, + .outputs = inputNode.value, + }, + buildMode == bmRepair ? bmRepair : bmNormal)); + for (const auto & [outputName, childNode] : inputNode.childMap) + accumDerivedPath( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + + for (const auto & [inputDrvPath, inputNode] : dynamic_cast(drv.get())->inputDrvs.map) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(i.first); + auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); if (!inputDrv.type().isPure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", worker.store.printStorePath(drvPath), - worker.store.printStorePath(i.first)); + worker.store.printStorePath(inputDrvPath)); } - addWaitee(worker.makeGoal( - DerivedPath::Built { - .drvPath = makeConstantStorePathRef(i.first), - .outputs = i.second - }, - buildMode == bmRepair ? bmRepair : bmNormal)); + accumDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); } + } /* Copy the input sources from the eval store to the build store. */ @@ -501,7 +514,7 @@ void DerivationGoal::inputsRealised() return ia.deferred; }, [&](const DerivationType::ContentAddressed & ca) { - return !fullDrv.inputDrvs.empty() && ( + return !fullDrv.inputDrvs.map.empty() && ( ca.fixed /* Can optionally resolve if fixed, which is good for avoiding unnecessary rebuilds. */ @@ -515,7 +528,7 @@ void DerivationGoal::inputsRealised() } }, drvType.raw()); - if (resolveDrv && !fullDrv.inputDrvs.empty()) { + if (resolveDrv && !fullDrv.inputDrvs.map.empty()) { experimentalFeatureSettings.require(Xp::CaDerivations); /* We are be able to resolve this derivation based on the @@ -552,11 +565,13 @@ void DerivationGoal::inputsRealised() return; } - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { + std::function::Node &)> accumDerivedPath; + + accumDerivedPath = [&](const StorePath & depDrvPath, const DerivedPathMap::Node & inputNode) { /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - for (auto & j : wantedDepOutputs) { + auto getOutput = [&](const std::string & outputName) { /* TODO (impure derivations-induced tech debt): Tracking input derivation outputs statefully through the goals is error prone and has led to bugs. @@ -568,21 +583,30 @@ void DerivationGoal::inputsRealised() a representation in the store, which is a usability problem in itself. When implementing this logic entirely with lookups make sure that they're cached. */ - if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) { - worker.store.computeFSClosure(*outPath, inputPaths); + if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) { + return *outPath; } else { auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath); - auto outMapPath = outMap.find(j); + auto outMapPath = outMap.find(outputName); if (outMapPath == outMap.end()) { throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath)); } - worker.store.computeFSClosure(outMapPath->second, inputPaths); + return outMapPath->second; } - } - } + }; + + for (auto & outputName : inputNode.value) + worker.store.computeFSClosure(getOutput(outputName), inputPaths); + + for (auto & [outputName, childNode] : inputNode.childMap) + accumDerivedPath(getOutput(outputName), childNode); + }; + + for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map) + accumDerivedPath(depDrvPath, depNode); } /* Second, the input sources. */ @@ -1485,10 +1509,11 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) // By the time we get here, the outer goals should have. assert(dg); - auto outputs = fullDrv.inputDrvs.find(dg->drvPath); - if (outputs == fullDrv.inputDrvs.end()) return; + auto & outputs = fullDrv.inputDrvs.ensureSlot(*odg->drvReq).value; + // FIXME: need a `findSlot` + //if (outputs == fullDrv.inputDrvs.end()) return; - for (auto & outputName : outputs->second) { + for (auto & outputName : outputs) { auto buildResult = dg->getBuildResult(DerivedPath::Built { .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index ae1956d00441..a934bc7b557c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -136,7 +136,7 @@ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) { auto references = drv.inputSrcs; - for (auto & i : drv.inputDrvs) + for (auto & i : drv.inputDrvs.map) references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be @@ -267,6 +267,28 @@ static DerivationOutput parseDerivationOutput(const Store & store, std::istrings } +static DerivedPathMap::Node parseDerivedPathMapNode(const Store & store, std::istringstream & str) +{ + DerivedPathMap::Node node; + node.value = parseStrings(str, false); + char c = str.get(); + switch (c) { + case ')': + return node; + case ',': + expect(str, "["); + while (!endOfList(str)) { + expect(str, "("); + auto outputName = parseString(str); + expect(str, ",["); + node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str)); + } + expect(str, ")"); + } + return node; +} + + Derivation parseDerivation(const Store & store, std::string && s, std::string_view name) { Derivation drv; @@ -288,8 +310,7 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi expect(str, "("); Path drvPath = parsePath(str); expect(str, ",["); - drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false)); - expect(str, ")"); + drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str)); } expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); @@ -376,8 +397,24 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt } +static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap::Node & node) +{ + s += ','; printUnquotedStrings(s, node.value.begin(), node.value.end()); + if (node.childMap.empty()) return; + s += ",["; + bool first = true; + for (auto & [outputName, childNode] : node.childMap) { + if (first) first = false; else s += ','; + s += '('; printUnquotedString(s, outputName); + unparseDerivedPathMapNode(store, s, childNode); + s += ')'; + } + s += "]"; +} + + std::string Derivation::unparse(const Store & store, bool maskOutputs, - std::map * actualInputs) const + DerivedPathMap::ChildMap * actualInputs) const { std::string s; s.reserve(65536); @@ -421,17 +458,17 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, s += "],["; first = true; if (actualInputs) { - for (auto & i : *actualInputs) { + for (auto & [drvHashModulo, childMap] : *actualInputs) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, drvHashModulo); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } else { - for (auto & i : inputDrvs) { + for (auto & [drvPath, childMap] : inputDrvs.map) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, store.printStorePath(i.first)); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, store.printStorePath(drvPath)); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } @@ -654,28 +691,29 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* For other derivations, replace the inputs paths with recursive calls to this function. */ - std::map inputs2; - for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { + DerivedPathMap::ChildMap inputs2; + for (auto & [drvPath, node0] : drv.inputDrvs.map) { // Avoid lambda capture restriction with standard / Clang - auto & inputOutputs = inputOutputs0; + auto & node = node0; const auto & res = pathDerivationModulo(store, drvPath); std::visit(overloaded { // Regular non-CA derivation, replace derivation [&](const DrvHash & drvHash) { kind |= drvHash.kind; - inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs); + inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), node); }, // CA derivation's output hashes [&](const CaOutputHashes & outputHashes) { - std::set justOut = { "out" }; - for (auto & outputName : inputOutputs) { + if (!node.childMap.empty()) + throw Error("Output of fixed output derivation cannot currently be treated as derivation"); + for (auto & outputName : node.value) { /* Put each one in with a single "out" output.. */ - const auto h = get(outputHashes, outputName); + const auto * h = get(outputHashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); inputs2.insert_or_assign( h->to_string(Base16, false), - justOut); + DerivedPathMap::Node { { "out" } }); } }, }, res.raw()); @@ -837,6 +875,8 @@ std::string hashPlaceholder(const std::string_view outputName) static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { + debug("Rewriting the derivation"); + for (auto & rewrite : rewrites) { debug("rewriting %s as %s", rewrite.first, rewrite.second); } @@ -856,7 +896,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { - if (std::holds_alternative(output.raw())) { + if (std::holds_alternative(output.raw())) { auto & h = hashModulo.requireNoFixedNonDeferred(); auto outPath = store.makeOutputPath(outputName, h, drv.name); drv.env[outputName] = store.printStorePath(outPath); @@ -882,14 +922,68 @@ std::optional Derivation::tryResolve(Store & store) const { std::map, StorePath> inputDrvOutputs; - for (auto & input : inputDrvs) - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first)) - if (outputPath) - inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath); + std::function::Node &)> accum; + accum = [&](auto & inputDrv, auto & node) { + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) { + if (outputPath) { + inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); + if (auto p = get(node.childMap, outputName)) + accum(*outputPath, *p); + } + } + }; + + for (auto & [inputDrv, node] : inputDrvs.map) + accum(inputDrv, node); return tryResolve(store, inputDrvOutputs); } +static bool tryResolveInput( + Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites, + const DownstreamPlaceholder * placeholderOpt, + const StorePath & inputDrv, const DerivedPathMap::Node & inputNode, + const std::map, StorePath> & inputDrvOutputs) +{ + auto getOutput = [&](const std::string & outputName) { + auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName }); + if (!actualPathOpt) + warn("output %s of input %s missing, aborting the resolving", + outputName, + store.printStorePath(inputDrv) + ); + return actualPathOpt; + }; + + auto getPlaceholder = [&](const std::string & outputName) { + return placeholderOpt + ? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName) + : DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName); + }; + + for (auto & outputName : inputNode.value) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + inputRewrites.emplace( + getPlaceholder(outputName).render(), + store.printStorePath(actualPath)); + inputSrcs.insert(std::move(actualPath)); + } + + for (auto & [outputName, childNode] : inputNode.childMap) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + auto nextPlaceholder = getPlaceholder(outputName); + if (!tryResolveInput(store, inputSrcs, inputRewrites, + &nextPlaceholder, actualPath, childNode, + inputDrvOutputs)) + return false; + } + return true; +} + std::optional Derivation::tryResolve( Store & store, const std::map, StorePath> & inputDrvOutputs) const @@ -899,21 +993,10 @@ std::optional Derivation::tryResolve( // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & [inputDrv, inputOutputs] : inputDrvs) { - for (auto & outputName : inputOutputs) { - if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { - inputRewrites.emplace( - DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), - store.printStorePath(*actualPath)); - resolved.inputSrcs.insert(*actualPath); - } else { - warn("output '%s' of input '%s' missing, aborting the resolving", - outputName, - store.printStorePath(inputDrv)); - return {}; - } - } - } + for (auto & [inputDrv, inputNode] : inputDrvs.map) + if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, + nullptr, inputDrv, inputNode, inputDrvOutputs)) + return std::nullopt; rewriteDerivation(store, resolved, inputRewrites); @@ -1098,10 +1181,25 @@ nlohmann::json Derivation::toJSON(const Store & store) const } { - auto& inputDrvsObj = res["inputDrvs"]; - inputDrvsObj = nlohmann::json ::object(); - for (auto & input : inputDrvs) - inputDrvsObj[store.printStorePath(input.first)] = input.second; + std::function::Node &)> doInput; + doInput = [&](const auto & inputNode) { + auto value = nlohmann::json::object(); + value["outputs"] = inputNode.value; + { + auto next = nlohmann::json::object(); + for (auto & [outputId, childNode] : inputNode.childMap) + next[outputId] = doInput(childNode); + value["deeper"] = std::move(next); + } + return value; + }; + { + auto& inputDrvsObj = res["inputDrvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : inputDrvs.map) { + inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode); + } + } } res["system"] = platform; @@ -1137,10 +1235,18 @@ Derivation Derivation::fromJSON( } { + std::function::Node(const nlohmann::json &)> doInput; + doInput = [&](const auto & json) { + DerivedPathMap::Node node; + node.value = static_cast(json["outputs"]); + for (auto & [outputId, childNode] : json["deeper"].items()) + node.childMap[outputId] = doInput(childNode); + return node; + }; auto & inputDrvsObj = json["inputDrvs"]; for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) - res.inputDrvs[store.parseStorePath(inputDrvPath)] = - static_cast(inputOutputs); + res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = + doInput(inputOutputs); } res.platform = json["system"]; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 47ede9bcac95..4ea4299f6e96 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -6,7 +6,7 @@ #include "hash.hh" #include "content-address.hh" #include "repair-flag.hh" -#include "derived-path.hh" +#include "derived-path-map.hh" #include "sync.hh" #include "comparator.hh" @@ -318,13 +318,13 @@ struct Derivation : BasicDerivation /** * inputs that are sub-derivations */ - DerivationInputs inputDrvs; + DerivedPathMap inputDrvs; /** * Print a derivation. */ std::string unparse(const Store & store, bool maskOutputs, - std::map * actualInputs = nullptr) const; + DerivedPathMap::ChildMap * actualInputs = nullptr) const; /** * Return the underlying basic derivation but with these changes: @@ -486,6 +486,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m * \todo What is the Hash in this map? */ std::map staticOutputHashes(Store & store, const Derivation & drv); +std::map staticOutputHashesWithDeferral(Store & store, const Derivation & drv); /** * Memoisation of hashDerivationModulo(). diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 6a078539cb44..56b54e1e616d 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -30,4 +30,17 @@ namespace nix { template struct DerivedPathMap>; -} +GENERATE_CMP_EXT( + template<>, + DerivedPathMap>::Node, + me->value, + me->childMap); + +GENERATE_CMP_EXT( + template<>, + DerivedPathMap>, + me->map); + +template struct DerivedPathMap>; + +}; diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 42a9679e136f..5b7fcb4213c0 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -7,17 +7,33 @@ namespace nix { template struct DerivedPathMap { - struct Node; + + struct Node; using ChildMap = std::map; struct Node { V value; ChildMap childMap; + + DECLARE_CMP(Node); }; using Raw = std::map; Raw map; + DECLARE_CMP(DerivedPathMap); + Node & ensureSlot(const SingleDerivedPath & k); + }; + +DECLARE_CMP_EXT( + template<>, + DerivedPathMap>::, + DerivedPathMap>); +DECLARE_CMP_EXT( + template<>, + DerivedPathMap>::Node::, + DerivedPathMap>::Node); + } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index bc610b76878a..64e8375f705c 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -125,14 +125,26 @@ void Store::queryMissing(const std::vector & targets, std::function doPath; + std::function, const DerivedPathMap::Node &)> accumDerivedPath; + + accumDerivedPath = [&](ref inputDrv, const DerivedPathMap::Node & inputNode) { + if (!inputNode.value.empty()) + pool.enqueue(std::bind(doPath, DerivedPath::Built { inputDrv, inputNode.value })); + for (const auto & [outputName, childNode] : inputNode.childMap) + accumDerivedPath( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); state->willBuild.insert(drvPath); } - for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second })); + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) { + accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode); + } }; auto checkOutput = [&]( @@ -292,24 +304,43 @@ std::map drvOutputReferences( { std::set inputRealisations; - for (const auto & [inputDrv, outputNames] : drv.inputDrvs) { - const auto outputHashes = - staticOutputHashes(store, store.readDerivation(inputDrv)); - for (const auto & outputName : outputNames) { - auto outputHash = get(outputHashes, outputName); - if (!outputHash) - throw Error( - "output '%s' of derivation '%s' isn't realised", outputName, - store.printStorePath(inputDrv)); - auto thisRealisation = store.queryRealisation( - DrvOutput{*outputHash, outputName}); - if (!thisRealisation) - throw Error( - "output '%s' of derivation '%s' isn't built", outputName, - store.printStorePath(inputDrv)); - inputRealisations.insert(*thisRealisation); + std::function::Node &)> accumRealisations; + + accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap::Node & inputNode) { + if (!inputNode.value.empty()) { + auto outputHashes = + staticOutputHashes(store, store.readDerivation(inputDrv)); + for (const auto & outputName : inputNode.value) { + auto outputHash = get(outputHashes, outputName); + if (!outputHash) + throw Error( + "output '%s' of derivation '%s' isn't realised", outputName, + store.printStorePath(inputDrv)); + auto thisRealisation = store.queryRealisation( + DrvOutput{*outputHash, outputName}); + if (!thisRealisation) + throw Error( + "output '%s' of derivation '%s' isn’t built", outputName, + store.printStorePath(inputDrv)); + inputRealisations.insert(*thisRealisation); + } } - } + if (!inputNode.value.empty()) { + auto d = makeConstantStorePathRef(inputDrv); + for (const auto & [outputName, childNode] : inputNode.childMap) { + SingleDerivedPath next = SingleDerivedPath::Built { d, outputName }; + accumRealisations( + // FIXME: Someday have "deep" realizations analogous to + // deep placeholders, rather than just straight-up + // resolving the computed drv to a store path. + resolveDerivedPath(store, next), + childNode); + } + } + }; + + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + accumRealisations(inputDrv, inputNode); auto info = store.queryPathInfo(outputPath); diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 0e28c1f08347..6945ed4e86d5 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -167,10 +167,13 @@ TEST_JSON(simple, "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [ - "cat", - "dog" - ] + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "deeper": {}, + "outputs": [ + "cat", + "dog" + ] + } }, "system": "wasm-sel4", "builder": "foo", @@ -190,13 +193,17 @@ TEST_JSON(simple, store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), }; drv.inputDrvs = { - { - store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + .map = { { - "cat", - "dog", + store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + .value = { + "cat", + "dog", + }, + }, }, - } + }, }; drv.platform = "wasm-sel4"; drv.builder = "foo"; diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 7982fdc5e08a..a4d20a675a16 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,21 +1,48 @@ #pragma once ///@file +#define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \ + PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const; +#define DECLARE_EQUAL(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, ==, my_type) +#define DECLARE_LEQ(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, <, my_type) +#define DECLARE_NEQ(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, !=, my_type) + +#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \ + PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \ + __VA_OPT__(const MY_TYPE * me = this;) \ + auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + __VA_OPT__(me = &other;) \ + auto fields2 = std::make_tuple( __VA_ARGS__ ); \ + return fields1 COMPARATOR fields2; \ + } +#define GENERATE_EQUAL(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, ==, my_type, args) +#define GENERATE_LEQ(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, <, my_type, args) +#define GENERATE_NEQ(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, !=, my_type, args) + /** * Declare comparison methods without defining them. */ -#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \ - bool operator COMPARATOR(const MY_TYPE & other) const; -#define DECLARE_EQUAL(my_type) \ - DECLARE_ONE_CMP(==, my_type) -#define DECLARE_LEQ(my_type) \ - DECLARE_ONE_CMP(<, my_type) -#define DECLARE_NEQ(my_type) \ - DECLARE_ONE_CMP(!=, my_type) #define DECLARE_CMP(my_type) \ - DECLARE_EQUAL(my_type) \ - DECLARE_LEQ(my_type) \ - DECLARE_NEQ(my_type) + DECLARE_EQUAL(,,my_type) \ + DECLARE_LEQ(,,my_type) \ + DECLARE_NEQ(,,my_type) + +/** + * @param prefix This is for something before each declaration like + * `template`. + * + * @param my_type the type are defining operators for. + */ +#define DECLARE_CMP_EXT(prefix, qualification, my_type) \ + DECLARE_EQUAL(prefix, qualification, my_type) \ + DECLARE_LEQ(prefix, qualification, my_type) \ + DECLARE_NEQ(prefix, qualification, my_type) /** * Awful hacky generation of the comparison operators by doing a lexicographic @@ -33,18 +60,18 @@ * } * ``` */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \ - bool operator COMPARATOR(const MY_TYPE& other) const { \ - __VA_OPT__(const MY_TYPE* me = this;) \ - auto fields1 = std::make_tuple( __VA_ARGS__ ); \ - __VA_OPT__(me = &other;) \ - auto fields2 = std::make_tuple( __VA_ARGS__ ); \ - return fields1 COMPARATOR fields2; \ - } -#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) -#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) -#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args) #define GENERATE_CMP(args...) \ - GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) \ - GENERATE_NEQ(args) + GENERATE_EQUAL(,,args) \ + GENERATE_LEQ(,,args) \ + GENERATE_NEQ(,,args) + +/** + * @param prefix This is for something before each declaration like + * `template`. + * + * @param my_type the type are defining operators for. + */ +#define GENERATE_CMP_EXT(prefix, my_type, args...) \ + GENERATE_EQUAL(prefix, my_type ::, my_type, args) \ + GENERATE_LEQ(prefix, my_type ::, my_type, args) \ + GENERATE_NEQ(prefix, my_type ::, my_type, args) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 66f319c3e381..d3716c304851 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -406,8 +406,22 @@ static void main_nix_build(int argc, char * * argv) } } + std::function, const DerivedPathMap::Node &)> accumDerivedPath; + + accumDerivedPath = [&](ref inputDrv, const DerivedPathMap::Node & inputNode) { + if (!inputNode.value.empty()) + pathsToBuild.push_back(DerivedPath::Built { + .drvPath = inputDrv, + .outputs = OutputsSpec::Names { inputNode.value }, + }); + for (const auto & [outputName, childNode] : inputNode.childMap) + accumDerivedPath( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + // Build or fetch all dependencies of the derivation. - for (const auto & [inputDrv0, inputOutputs] : drv.inputDrvs) { + for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) { // To get around lambda capturing restrictions in the // standard. const auto & inputDrv = inputDrv0; @@ -416,10 +430,7 @@ static void main_nix_build(int argc, char * * argv) return !std::regex_search(store->printStorePath(inputDrv), std::regex(exclude)); })) { - pathsToBuild.push_back(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(inputDrv), - .outputs = OutputsSpec::Names { inputOutputs }, - }); + accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode); pathsToCopy.insert(inputDrv); } } @@ -482,13 +493,21 @@ static void main_nix_build(int argc, char * * argv) if (env.count("__json")) { StorePathSet inputs; - for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) { - auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath); - for (auto & i : wantedDepOutputs) { + + std::function::Node &)> accumInputClosure; + + accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap::Node & inputNode) { + auto outputs = evalStore->queryPartialDerivationOutputMap(inputDrv); + for (auto & i : inputNode.value) { auto o = outputs.at(i); store->computeFSClosure(*o, inputs); } - } + for (const auto & [outputName, childNode] : inputNode.childMap) + accumInputClosure(*outputs.at(outputName), childNode); + }; + + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + accumInputClosure(inputDrv, inputNode); ParsedDerivation parsedDrv(drvInfo.requireDrvPath(), drv); diff --git a/src/nix/app.cc b/src/nix/app.cc index be5d39fce1c2..d53fa4ffe3b7 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -11,6 +11,25 @@ namespace nix { +static std::variant makePlaceHolder(const SingleBuiltPath & req) +{ + return std::visit(overloaded { + [&](const SingleBuiltPath::Opaque & bo) -> std::variant { + return bo.path; + }, + [&](const SingleBuiltPath::Built & bfd) -> std::variant { + return std::visit(overloaded { + [&](const StorePath & child) { + return DownstreamPlaceholder::unknownCaOutput(child, bfd.output.first); + }, + [&](const DownstreamPlaceholder & child) { + return DownstreamPlaceholder::unknownDerivation(child, bfd.output.first); + }, + }, makePlaceHolder(*bfd.drvPath)); + }, + }, req.raw()); +} + /** * Return the rewrites that are needed to resolve a string whose context is * included in `dependencies`. @@ -21,13 +40,22 @@ StringPairs resolveRewrites( { StringPairs res; for (auto & dep : dependencies) - if (auto drvDep = std::get_if(&dep.path)) - for (auto & [ outputName, outputPath ] : drvDep->outputs) + if (auto drvDep = std::get_if(&dep.path)) { + auto deeper = makePlaceHolder(*drvDep->drvPath); + for (auto & [ outputName_, outputPath ] : drvDep->outputs) { + auto & outputName = outputName_; res.emplace( - DownstreamPlaceholder::unknownCaOutput( - drvDep->drvPath->outPath(), outputName).render(), - store.printStorePath(outputPath) - ); + std::visit(overloaded { + [&](const StorePath & child) { + return DownstreamPlaceholder::unknownCaOutput(child, outputName); + }, + [&](const DownstreamPlaceholder & child) { + return DownstreamPlaceholder::unknownDerivation(child, outputName); + }, + }, deeper).render(), + store.printStorePath(outputPath)); + } + } return res; } diff --git a/tests/dyn-drv/dep-built-drv.sh b/tests/dyn-drv/dep-built-drv.sh index 8c4a45e3b40c..c3daf2c59174 100644 --- a/tests/dyn-drv/dep-built-drv.sh +++ b/tests/dyn-drv/dep-built-drv.sh @@ -6,4 +6,6 @@ out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) clearStore -expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Dependencies on the outputs of dynamic derivations are not yet supported" +out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link) + +diff -r $out1 $out2