Skip to content

Commit

Permalink
Sync up with some changes done to the main CA branch
Browse files Browse the repository at this point in the history
  • Loading branch information
@Ericson2314
Ericson2314 committed Dec 11, 2023
1 parent 8783dd5 commit ebfefb9
Show file tree
Hide file tree
Showing 6 changed files with 93 additions and 104 deletions.
8 changes: 4 additions & 4 deletions flake.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion flake.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
description = "A Nix-based continuous build system";

inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05";
inputs.nix.url = "github:NixOS/nix/2.19.2";
inputs.nix.url = "github:NixOS/nix/2.19-maintenance";
inputs.nix.inputs.nixpkgs.follows = "nixpkgs";

outputs = { self, nixpkgs, nix }:
Expand Down
55 changes: 18 additions & 37 deletions src/hydra-queue-runner/build-remote.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,40 +182,6 @@ static StorePaths reverseTopoSortPaths(const std::map<StorePath, ValidPathInfo>
return sorted;
}

/**
* Replace the input derivations by their output paths to send a minimal closure
* to the builder.
*
* If we can afford it, resolve it, so that the newly generated derivation still
* has some sensible output paths.
*/
BasicDerivation inlineInputDerivations(Store & store, Derivation & drv, const StorePath & drvPath)
{
BasicDerivation ret;
if (!drv.type().hasKnownOutputPaths()) {
auto maybeBasicDrv = drv.tryResolve(store);
if (!maybeBasicDrv)
throw Error(
"the derivation '%s' can’t be resolved. It’s probably "
"missing some outputs",
store.printStorePath(drvPath));
ret = *maybeBasicDrv;
} else {
// If the derivation is a real `InputAddressed` derivation, we must
// resolve it manually to keep the original output paths
ret = BasicDerivation(drv);
for (auto & [drvPath, node] : drv.inputDrvs.map) {
auto drv2 = store.readDerivation(drvPath);
auto drv2Outputs = drv2.outputsAndOptPaths(store);
for (auto & name : node.value) {
auto inputPath = drv2Outputs.at(name);
ret.inputSrcs.insert(*inputPath.second);
}
}
}
return ret;
}

static std::pair<Path, AutoCloseFD> openLogFile(const std::string & logDir, const StorePath & drvPath)
{
std::string base(drvPath.to_string());
Expand Down Expand Up @@ -262,7 +228,22 @@ static BasicDerivation sendInputs(
counter & nrStepsCopyingTo
)
{
BasicDerivation basicDrv = inlineInputDerivations(localStore, *step.drv, step.drvPath);
/* Replace the input derivations by their output paths to send a
minimal closure to the builder.
`tryResolve` currently does *not* rewrite input addresses, so it
is safe to do this in all cases. (It should probably have a mode
to do that, however, but we would not use it here.)
*/
BasicDerivation basicDrv = ({
auto maybeBasicDrv = step.drv->tryResolve(destStore, &localStore);
if (!maybeBasicDrv)
throw Error(
"the derivation '%s' can’t be resolved. It’s probably "
"missing some outputs",
localStore.printStorePath(step.drvPath));
*maybeBasicDrv;
});

/* Ensure that the inputs exist in the destination store. This is
a no-op for regular stores, but for the binary cache store,
Expand Down Expand Up @@ -351,6 +332,8 @@ static BuildResult performBuild(
// far anyways
assert(drv.type().hasKnownOutputPaths());
DerivationOutputsAndOptPaths drvOutputs = drv.outputsAndOptPaths(localStore);
// Since this a `BasicDerivation`, `staticOutputHashes` will not
// do any real work.
auto outputHashes = staticOutputHashes(localStore, drv);
for (auto & [outputName, output] : drvOutputs) {
auto outputPath = output.second;
Expand Down Expand Up @@ -665,14 +648,12 @@ void State::buildRemote(ref<Store> destStore,
auto outputHashes = staticOutputHashes(*localStore, *step->drv);
for (auto & [outputName, realisation] : buildResult.builtOutputs) {
// Register the resolved drv output
localStore->registerDrvOutput(realisation);
destStore->registerDrvOutput(realisation);

// Also register the unresolved one
auto unresolvedRealisation = realisation;
unresolvedRealisation.signatures.clear();
unresolvedRealisation.id.drvHash = outputHashes.at(outputName);
localStore->registerDrvOutput(unresolvedRealisation);
destStore->registerDrvOutput(unresolvedRealisation);
}
}
Expand Down
11 changes: 7 additions & 4 deletions src/hydra-queue-runner/builder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,

if (result.stepStatus == bsSuccess) {
updateStep(ssPostProcessing);
res = getBuildOutput(destStore, narMembers, localStore->queryDerivationOutputMap(step->drvPath));
res = getBuildOutput(destStore, narMembers, destStore->queryDerivationOutputMap(step->drvPath, &*localStore));
}
}

Expand Down Expand Up @@ -277,9 +277,12 @@ State::StepResult State::doBuildStep(nix::ref<Store> destStore,

assert(stepNr);

for (auto & i : localStore->queryPartialDerivationOutputMap(step->drvPath)) {
if (i.second)
addRoot(*i.second);
for (auto & [outputName, optOutputPath] : destStore->queryPartialDerivationOutputMap(step->drvPath, &*localStore)) {
if (!optOutputPath)
throw Error(
"Missing output %s for derivation %d which was supposed to have succeeded",
outputName, localStore->printStorePath(step->drvPath));
addRoot(*optOutputPath);
}

/* Register success in the database for all Build objects that
Expand Down
4 changes: 2 additions & 2 deletions src/hydra-queue-runner/hydra-queue-runner.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID

if (r.affected_rows() == 0) goto restart;

for (auto & [name, output] : localStore->queryPartialDerivationOutputMap(step->drvPath))
for (auto & [name, output] : getDestStore()->queryPartialDerivationOutputMap(step->drvPath, &*localStore))
txn.exec_params0
("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)",
buildId, stepNr, name, output ? localStore->printStorePath(*output) : "");
Expand Down Expand Up @@ -359,7 +359,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result,
assert(res.size());
StorePath drvPath = localStore->parseStorePath(res[0].as<std::string>());
// If we've finished building, all the paths should be known
for (auto & [name, output] : localStore->queryDerivationOutputMap(drvPath))
for (auto & [name, output] : getDestStore()->queryDerivationOutputMap(drvPath, &*localStore))
txn.exec_params0
("update BuildStepOutputs set path = $4 where build = $1 and stepnr = $2 and name = $3",
buildId, stepNr, name, localStore->printStorePath(output));
Expand Down
117 changes: 61 additions & 56 deletions src/hydra-queue-runner/queue-monitor.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,11 +192,11 @@ bool State::getQueuedBuilds(Connection & conn,
if (!res[0].is_null()) propagatedFrom = res[0].as<BuildID>();

if (!propagatedFrom) {
for (auto & i : localStore->queryPartialDerivationOutputMap(ex.step->drvPath)) {
for (auto & [outputName, _] : destStore->queryPartialDerivationOutputMap(ex.step->drvPath, &*localStore)) {
auto res = txn.exec_params
("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where drvPath = $1 and name = $2 and startTime != 0 and stopTime != 0 and status = 1",
localStore->printStorePath(ex.step->drvPath),
i.first);
outputName);
if (!res[0][0].is_null()) {
propagatedFrom = res[0][0].as<BuildID>();
break;
Expand Down Expand Up @@ -237,7 +237,7 @@ bool State::getQueuedBuilds(Connection & conn,
if (!step) {
BuildOutput res = getBuildOutputCached(conn, destStore, build->drvPath);

for (auto & i : localStore->queryDerivationOutputMap(build->drvPath))
for (auto & i : destStore->queryDerivationOutputMap(build->drvPath, &*localStore))
addRoot(i.second);

{
Expand Down Expand Up @@ -481,20 +481,12 @@ Step::ptr State::createStep(ref<Store> destStore,
auto outputHashes = staticOutputHashes(*localStore, *(step->drv));
bool valid = true;
std::map<DrvOutput, std::optional<StorePath>> missing;
for (auto &[outputName, maybeOutputPath] : step->drv->outputsAndOptPaths(*destStore)) {
for (auto & [outputName, maybeOutputPath] : destStore->queryPartialDerivationOutputMap(drvPath, &*localStore)) {
auto outputHash = outputHashes.at(outputName);
if (maybeOutputPath.second) {
if (!destStore->isValidPath(*maybeOutputPath.second)) {
valid = false;
missing.insert({{outputHash, outputName}, maybeOutputPath.second});
}
} else {
experimentalFeatureSettings.require(Xp::CaDerivations);
if (!destStore->queryRealisation(DrvOutput{outputHash, outputName})) {
valid = false;
missing.insert({{outputHash, outputName}, std::nullopt});
}
}
if (maybeOutputPath && destStore->isValidPath(*maybeOutputPath))
continue;
valid = false;
missing.insert({{outputHash, outputName}, maybeOutputPath});
}

/* Try to copy the missing paths from the local store or from
Expand All @@ -503,59 +495,72 @@ Step::ptr State::createStep(ref<Store> destStore,

size_t avail = 0;
for (auto & [i, maybePath] : missing) {
if ((maybePath && localStore->isValidPath(*maybePath)))
avail++;
else if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && localStore->queryRealisation(i)) {
maybePath = localStore->queryRealisation(i)->outPath;
// If we don't know the output path from the destination
// store, see if the local store can tell us.
if (/* localStore != destStore && */ !maybePath && experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
if (auto maybeRealisation = localStore->queryRealisation(i))
maybePath = maybeRealisation->outPath;

if (!maybePath) {
// No hope of getting the store object if we don't know
// the path.
continue;
}
auto & path = *maybePath;

if (/* localStore != destStore && */ localStore->isValidPath(path))
avail++;
} else if (useSubstitutes && maybePath) {
else if (useSubstitutes) {
SubstitutablePathInfos infos;
localStore->querySubstitutablePathInfos({{*maybePath, {}}}, infos);
localStore->querySubstitutablePathInfos({{path, {}}}, infos);
if (infos.size() == 1)
avail++;
}
}

if (missing.size() == avail) {
valid = true;
for (auto & [i, path] : missing) {
if (path) {
try {
time_t startTime = time(0);

if (localStore->isValidPath(*path))
printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
localStore->printStorePath(*path),
localStore->printStorePath(drvPath));
else {
printInfo("substituting output ‘%1%’ of ‘%2%’",
localStore->printStorePath(*path),
localStore->printStorePath(drvPath));
localStore->ensurePath(*path);
// FIXME: should copy directly from substituter to destStore.
}

copyClosure(*localStore, *destStore,
StorePathSet { *path },
NoRepair, CheckSigs, NoSubstitute);
for (auto & [i, maybePath] : missing) {
// If we found everything, then we should know the path
// to every missing store object now.
assert(maybePath);
auto & path = *maybePath;

try {
time_t startTime = time(0);

if (localStore->isValidPath(path))
printInfo("copying output ‘%1%’ of ‘%2%’ from local store",
localStore->printStorePath(path),
localStore->printStorePath(drvPath));
else {
printInfo("substituting output ‘%1%’ of ‘%2%’",
localStore->printStorePath(path),
localStore->printStorePath(drvPath));
localStore->ensurePath(path);
// FIXME: should copy directly from substituter to destStore.
}

time_t stopTime = time(0);
copyClosure(*localStore, *destStore,
StorePathSet { path },
NoRepair, CheckSigs, NoSubstitute);

{
auto mc = startDbUpdate();
pqxx::work txn(conn);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", *path);
txn.commit();
}
time_t stopTime = time(0);

} catch (Error & e) {
printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
localStore->printStorePath(*path),
localStore->printStorePath(drvPath),
e.what());
valid = false;
break;
{
auto mc = startDbUpdate();
pqxx::work txn(conn);
createSubstitutionStep(txn, startTime, stopTime, build, drvPath, *(step->drv), "out", path);
txn.commit();
}

} catch (Error & e) {
printError("while copying/substituting output ‘%s’ of ‘%s’: %s",
localStore->printStorePath(path),
localStore->printStorePath(drvPath),
e.what());
valid = false;
break;
}
}
}
Expand Down

0 comments on commit ebfefb9

Please sign in to comment.