From d3a959510630932daa0c34470b6cbd965afb2138 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 12 Feb 2024 17:12:49 +0100 Subject: [PATCH] Switch to new Nix bindings Implements support for Nix's new Perl bindings[1]. The current state basically does `openStore()`, but always uses `auto` and doesn't support stores at other URIs. Even though the stores are cached inside the Perl implementation, I decided to instantiate those once in the Nix helper module. That way store openings aren't cluttered across the entire codebase. Also, there are two stores used later on - MACHINE_LOCAL_STORE for `auto`, BINARY_CACHE_STORE for the one from `store_uri` in `hydra.conf` - and using consistent names should make the intent clearer then. This doesn't contain any behavioral changes, i.e. the build product availability issue from #1352 isn't fixed. This patch only contains the migration to the new API. [1] https://github.com/NixOS/nix/pull/9863 --- flake.lock | 11 ++++++----- flake.nix | 2 +- src/lib/Hydra/Base/Controller/NixChannel.pm | 3 +-- src/lib/Hydra/Controller/Build.pm | 18 ++++++++---------- src/lib/Hydra/Controller/Root.pm | 2 +- src/lib/Hydra/Helper/Nix.pm | 5 ++++- src/lib/Hydra/Plugin/BazaarInput.pm | 7 +++---- src/lib/Hydra/Plugin/DarcsInput.pm | 7 +++---- src/lib/Hydra/Plugin/GitInput.pm | 6 +++--- src/lib/Hydra/Plugin/MercurialInput.pm | 7 +++---- src/lib/Hydra/Plugin/PathInput.pm | 5 ++--- src/lib/Hydra/Plugin/SubversionInput.pm | 9 ++++----- src/lib/Hydra/View/NARInfo.pm | 9 ++++----- src/script/hydra-eval-jobset | 8 ++++---- src/script/hydra-update-gc-roots | 5 ++--- t/s3-backup-test.pl | 1 - 16 files changed, 49 insertions(+), 56 deletions(-) diff --git a/flake.lock b/flake.lock index f04d657b0..34479856a 100644 --- a/flake.lock +++ b/flake.lock @@ -42,15 +42,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1706629374, - "narHash": "sha256-KyAiLGxJ39fSY0cuq8EWAZQ4vaDdqAItSRP4+vjYvq8=", - "owner": "NixOS", + "lastModified": 1706306797, + "narHash": "sha256-nha3OTHgU/zSLNQhl4tZrkwehHZASd4KWOKZzp/uLew=", + "owner": "obsidiansystems", "repo": "nix", - "rev": "75ebb90a70f6320c1c7a1fca87a0a8adb0716143", + "rev": "e3ccb7e42a55529d8d50544e7e0cebb5cbc606cf", "type": "github" }, "original": { - "owner": "NixOS", + "owner": "obsidiansystems", + "ref": "perl-open-other-store", "repo": "nix", "type": "github" } diff --git a/flake.nix b/flake.nix index 4dc7d25ea..b06a51e7d 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ description = "A Nix-based continuous build system"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; - inputs.nix.url = "github:NixOS/nix"; + inputs.nix.url = "github:obsidiansystems/nix/perl-open-other-store"; inputs.nix.inputs.nixpkgs.follows = "nixpkgs"; # TODO get rid of this once https://github.com/NixOS/nix/pull/9546 is diff --git a/src/lib/Hydra/Base/Controller/NixChannel.pm b/src/lib/Hydra/Base/Controller/NixChannel.pm index 3f8e96096..a5bc2784e 100644 --- a/src/lib/Hydra/Base/Controller/NixChannel.pm +++ b/src/lib/Hydra/Base/Controller/NixChannel.pm @@ -4,7 +4,6 @@ use strict; use warnings; use base 'Hydra::Base::Controller::REST'; use List::SomeUtils qw(any); -use Nix::Store; use Hydra::Helper::Nix; use Hydra::Helper::CatalystUtils; @@ -30,7 +29,7 @@ sub getChannelData { my $outputs = {}; foreach my $output (@outputs) { my $outPath = $output->get_column("outpath"); - next if $checkValidity && !isValidPath($outPath); + next if $checkValidity && !$MACHINE_LOCAL_STORE->isValidPath($outPath); $outputs->{$output->get_column("outname")} = $outPath; push @storePaths, $outPath; # Put the system type in the manifest (for top-level diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index c3869838d..4e249d140 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -10,8 +10,6 @@ use File::Basename; use File::LibMagic; use File::stat; use Data::Dump qw(dump); -use Nix::Store; -use Nix::Config; use List::SomeUtils qw(all); use Encode; use JSON::PP; @@ -82,9 +80,9 @@ sub build_GET { # false because `$_->path` will be empty $c->stash->{available} = $c->stash->{isLocalStore} - ? all { $_->path && isValidPath($_->path) } $build->buildoutputs->all + ? all { $_->path && $MACHINE_LOCAL_STORE->isValidPath($_->path) } $build->buildoutputs->all : 1; - $c->stash->{drvAvailable} = isValidPath $build->drvpath; + $c->stash->{drvAvailable} = $MACHINE_LOCAL_STORE->isValidPath($build->drvpath); if ($build->finished && $build->iscachedbuild) { my $path = ($build->buildoutputs)[0]->path or undef; @@ -308,7 +306,7 @@ sub output : Chained('buildChain') PathPart Args(1) { error($c, "This build is not finished yet.") unless $build->finished; my $output = $build->buildoutputs->find({name => $outputName}); notFound($c, "This build has no output named ‘$outputName’") unless defined $output; - gone($c, "Output is no longer available.") unless isValidPath $output->path; + gone($c, "Output is no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($output->path); $c->response->header('Content-Disposition', "attachment; filename=\"build-${\$build->id}-${\$outputName}.nar.bz2\""); $c->stash->{current_view} = 'NixNAR'; @@ -425,7 +423,7 @@ sub getDependencyGraph { }; $$done{$path} = $node; my @refs; - foreach my $ref (queryReferences($path)) { + foreach my $ref ($MACHINE_LOCAL_STORE->queryReferences($path)) { next if $ref eq $path; next unless $runtime || $ref =~ /\.drv$/; getDependencyGraph($self, $c, $runtime, $done, $ref); @@ -433,7 +431,7 @@ sub getDependencyGraph { } # Show in reverse topological order to flatten the graph. # Should probably do a proper BFS. - my @sorted = reverse topoSortPaths(@refs); + my @sorted = reverse $MACHINE_LOCAL_STORE->topoSortPaths(@refs); $node->{refs} = [map { $$done{$_} } @sorted]; } @@ -446,7 +444,7 @@ sub build_deps : Chained('buildChain') PathPart('build-deps') { my $build = $c->stash->{build}; my $drvPath = $build->drvpath; - error($c, "Derivation no longer available.") unless isValidPath $drvPath; + error($c, "Derivation no longer available.") unless $MACHINE_LOCAL_STORE->isValidPath($drvPath); $c->stash->{buildTimeGraph} = getDependencyGraph($self, $c, 0, {}, $drvPath); @@ -461,7 +459,7 @@ sub runtime_deps : Chained('buildChain') PathPart('runtime-deps') { requireLocalStore($c); - error($c, "Build outputs no longer available.") unless all { isValidPath($_) } @outPaths; + error($c, "Build outputs no longer available.") unless all { $MACHINE_LOCAL_STORE->isValidPath($_) } @outPaths; my $done = {}; $c->stash->{runtimeGraph} = [ map { getDependencyGraph($self, $c, 1, $done, $_) } @outPaths ]; @@ -481,7 +479,7 @@ sub nix : Chained('buildChain') PathPart('nix') CaptureArgs(0) { if (isLocalStore) { foreach my $out ($build->buildoutputs) { notFound($c, "Path " . $out->path . " is no longer available.") - unless isValidPath($out->path); + unless $MACHINE_LOCAL_STORE->isValidPath($out->path); } } diff --git a/src/lib/Hydra/Controller/Root.pm b/src/lib/Hydra/Controller/Root.pm index 548cfac34..162f347ef 100644 --- a/src/lib/Hydra/Controller/Root.pm +++ b/src/lib/Hydra/Controller/Root.pm @@ -395,7 +395,7 @@ sub narinfo :Path :Args(StrMatch[NARINFO_REGEX]) { my ($hash) = $narinfo =~ NARINFO_REGEX; die("Hash length was not 32") if length($hash) != 32; - my $path = queryPathFromHashPart($hash); + my $path = $MACHINE_LOCAL_STORE->queryPathFromHashPart($hash); if (!$path) { $c->response->status(404); diff --git a/src/lib/Hydra/Helper/Nix.pm b/src/lib/Hydra/Helper/Nix.pm index 71a8a7d77..900514cbf 100644 --- a/src/lib/Hydra/Helper/Nix.pm +++ b/src/lib/Hydra/Helper/Nix.pm @@ -40,8 +40,11 @@ our @EXPORT = qw( registerRoot restartBuilds run + $MACHINE_LOCAL_STORE ); +our $MACHINE_LOCAL_STORE = Nix::Store->new(); + sub getHydraHome { my $dir = $ENV{"HYDRA_HOME"} or die "The HYDRA_HOME directory does not exist!\n"; @@ -494,7 +497,7 @@ sub restartBuilds { $builds = $builds->search({ finished => 1 }); foreach my $build ($builds->search({}, { columns => ["drvpath"] })) { - next if !isValidPath($build->drvpath); + next if !$MACHINE_LOCAL_STORE->isValidPath($build->drvpath); registerRoot $build->drvpath; } diff --git a/src/lib/Hydra/Plugin/BazaarInput.pm b/src/lib/Hydra/Plugin/BazaarInput.pm index 230d108b8..b35ed7c8c 100644 --- a/src/lib/Hydra/Plugin/BazaarInput.pm +++ b/src/lib/Hydra/Plugin/BazaarInput.pm @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex); use File::Path; use Hydra::Helper::Exec; use Hydra::Helper::Nix; -use Nix::Store; sub supportedInputTypes { my ($self, $inputTypes) = @_; @@ -38,9 +37,9 @@ sub fetchInput { (my $cachedInput) = $self->{db}->resultset('CachedBazaarInputs')->search( {uri => $uri, revision => $revision}); - addTempRoot($cachedInput->storepath) if defined $cachedInput; + $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput; - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; $sha256 = $cachedInput->sha256hash; } else { @@ -58,7 +57,7 @@ sub fetchInput { ($sha256, $storePath) = split ' ', $stdout; # FIXME: time window between nix-prefetch-bzr and addTempRoot. - addTempRoot($storePath); + $MACHINE_LOCAL_STORE->addTempRoot($storePath); $self->{db}->txn_do(sub { $self->{db}->resultset('CachedBazaarInputs')->create( diff --git a/src/lib/Hydra/Plugin/DarcsInput.pm b/src/lib/Hydra/Plugin/DarcsInput.pm index b7f3db550..a8df63964 100644 --- a/src/lib/Hydra/Plugin/DarcsInput.pm +++ b/src/lib/Hydra/Plugin/DarcsInput.pm @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex); use File::Path; use Hydra::Helper::Exec; use Hydra::Helper::Nix; -use Nix::Store; sub supportedInputTypes { my ($self, $inputTypes) = @_; @@ -58,7 +57,7 @@ sub fetchInput { {uri => $uri, revision => $revision}, {rows => 1}); - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; $sha256 = $cachedInput->sha256hash; $revision = $cachedInput->revision; @@ -75,8 +74,8 @@ sub fetchInput { die "darcs changes --count failed" if $? != 0; system "rm", "-rf", "$tmpDir/export/_darcs"; - $storePath = addToStore("$tmpDir/export", 1, "sha256"); - $sha256 = queryPathHash($storePath); + $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/export", 1, "sha256"); + $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath); $sha256 =~ s/sha256://; $self->{db}->txn_do(sub { diff --git a/src/lib/Hydra/Plugin/GitInput.pm b/src/lib/Hydra/Plugin/GitInput.pm index e5fc7de92..0de021282 100644 --- a/src/lib/Hydra/Plugin/GitInput.pm +++ b/src/lib/Hydra/Plugin/GitInput.pm @@ -186,9 +186,9 @@ sub fetchInput { {uri => $uri, branch => $branch, revision => $revision, isdeepclone => defined($deepClone) ? 1 : 0}, {rows => 1}); - addTempRoot($cachedInput->storepath) if defined $cachedInput; + $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput; - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; $sha256 = $cachedInput->sha256hash; $revision = $cachedInput->revision; @@ -217,7 +217,7 @@ sub fetchInput { ($sha256, $storePath) = split ' ', grab(cmd => ["nix-prefetch-git", $clonePath, $revision], chomp => 1); # FIXME: time window between nix-prefetch-git and addTempRoot. - addTempRoot($storePath); + $MACHINE_LOCAL_STORE->addTempRoot($storePath); $self->{db}->txn_do(sub { $self->{db}->resultset('CachedGitInputs')->update_or_create( diff --git a/src/lib/Hydra/Plugin/MercurialInput.pm b/src/lib/Hydra/Plugin/MercurialInput.pm index 921262ad7..85bd2c708 100644 --- a/src/lib/Hydra/Plugin/MercurialInput.pm +++ b/src/lib/Hydra/Plugin/MercurialInput.pm @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex); use File::Path; use Hydra::Helper::Nix; use Hydra::Helper::Exec; -use Nix::Store; use Fcntl qw(:flock); sub supportedInputTypes { @@ -68,9 +67,9 @@ sub fetchInput { (my $cachedInput) = $self->{db}->resultset('CachedHgInputs')->search( {uri => $uri, branch => $branch, revision => $revision}); - addTempRoot($cachedInput->storepath) if defined $cachedInput; + $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput; - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; $sha256 = $cachedInput->sha256hash; } else { @@ -85,7 +84,7 @@ sub fetchInput { ($sha256, $storePath) = split ' ', $stdout; # FIXME: time window between nix-prefetch-hg and addTempRoot. - addTempRoot($storePath); + $MACHINE_LOCAL_STORE->addTempRoot($storePath); $self->{db}->txn_do(sub { $self->{db}->resultset('CachedHgInputs')->update_or_create( diff --git a/src/lib/Hydra/Plugin/PathInput.pm b/src/lib/Hydra/Plugin/PathInput.pm index d122ff573..c923a03c3 100644 --- a/src/lib/Hydra/Plugin/PathInput.pm +++ b/src/lib/Hydra/Plugin/PathInput.pm @@ -5,7 +5,6 @@ use warnings; use parent 'Hydra::Plugin'; use POSIX qw(strftime); use Hydra::Helper::Nix; -use Nix::Store; sub supportedInputTypes { my ($self, $inputTypes) = @_; @@ -30,7 +29,7 @@ sub fetchInput { {srcpath => $uri, lastseen => {">", $timestamp - $timeout}}, {rows => 1, order_by => "lastseen DESC"}); - if (defined $cachedInput && isValidPath($cachedInput->storepath)) { + if (defined $cachedInput && $MACHINE_LOCAL_STORE->isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; $sha256 = $cachedInput->sha256hash; $timestamp = $cachedInput->timestamp; @@ -46,7 +45,7 @@ sub fetchInput { } chomp $storePath; - $sha256 = (queryPathInfo($storePath, 0))[1] or die; + $sha256 = ($MACHINE_LOCAL_STORE->queryPathInfo($storePath, 0))[1] or die; ($cachedInput) = $self->{db}->resultset('CachedPathInputs')->search( {srcpath => $uri, sha256hash => $sha256}); diff --git a/src/lib/Hydra/Plugin/SubversionInput.pm b/src/lib/Hydra/Plugin/SubversionInput.pm index 456c6892a..83c1f39d2 100644 --- a/src/lib/Hydra/Plugin/SubversionInput.pm +++ b/src/lib/Hydra/Plugin/SubversionInput.pm @@ -7,7 +7,6 @@ use Digest::SHA qw(sha256_hex); use Hydra::Helper::Exec; use Hydra::Helper::Nix; use IPC::Run; -use Nix::Store; sub supportedInputTypes { my ($self, $inputTypes) = @_; @@ -45,7 +44,7 @@ sub fetchInput { (my $cachedInput) = $self->{db}->resultset('CachedSubversionInputs')->search( {uri => $uri, revision => $revision}); - addTempRoot($cachedInput->storepath) if defined $cachedInput; + $MACHINE_LOCAL_STORE->addTempRoot($cachedInput->storepath) if defined $cachedInput; if (defined $cachedInput && isValidPath($cachedInput->storepath)) { $storePath = $cachedInput->storepath; @@ -62,16 +61,16 @@ sub fetchInput { die "error checking out Subversion repo at `$uri':\n$stderr" if $res; if ($type eq "svn-checkout") { - $storePath = addToStore($wcPath, 1, "sha256"); + $storePath = $MACHINE_LOCAL_STORE->addToStore($wcPath, 1, "sha256"); } else { # Hm, if the Nix Perl bindings supported filters in # addToStore(), then we wouldn't need to make a copy here. my $tmpDir = File::Temp->newdir("hydra-svn-export.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die; (system "svn", "export", $wcPath, "$tmpDir/source", "--quiet") == 0 or die "svn export failed"; - $storePath = addToStore("$tmpDir/source", 1, "sha256"); + $storePath = $MACHINE_LOCAL_STORE->addToStore("$tmpDir/source", 1, "sha256"); } - $sha256 = queryPathHash($storePath); $sha256 =~ s/sha256://; + $sha256 = $MACHINE_LOCAL_STORE->queryPathHash($storePath); $sha256 =~ s/sha256://; $self->{db}->txn_do(sub { $self->{db}->resultset('CachedSubversionInputs')->update_or_create( diff --git a/src/lib/Hydra/View/NARInfo.pm b/src/lib/Hydra/View/NARInfo.pm index 44db78b14..bf8711a4a 100644 --- a/src/lib/Hydra/View/NARInfo.pm +++ b/src/lib/Hydra/View/NARInfo.pm @@ -6,8 +6,7 @@ use File::Basename; use Hydra::Helper::CatalystUtils; use MIME::Base64; use Nix::Manifest; -use Nix::Store; -use Nix::Utils; +use Hydra::Helper::Nix; use base qw/Catalyst::View/; sub process { @@ -17,7 +16,7 @@ sub process { $c->response->content_type('text/x-nix-narinfo'); # !!! check MIME type - my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1); + my ($deriver, $narHash, $time, $narSize, $refs) = $MACHINE_LOCAL_STORE->queryPathInfo($storePath, 1); my $info; $info .= "StorePath: $storePath\n"; @@ -28,8 +27,8 @@ sub process { $info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n"; if (defined $deriver) { $info .= "Deriver: " . basename $deriver . "\n"; - if (isValidPath($deriver)) { - my $drv = derivationFromPath($deriver); + if ($MACHINE_LOCAL_STORE->isValidPath($deriver)) { + my $drv = $MACHINE_LOCAL_STORE->derivationFromPath($deriver); $info .= "System: $drv->{platform}\n"; } } diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 7ed7ebe83..72a386f57 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -85,14 +85,14 @@ sub attrsToSQL { # Fetch a store path from 'eval_substituter' if not already present. sub getPath { my ($path) = @_; - return 1 if isValidPath($path); + return 1 if $MACHINE_LOCAL_STORE->isValidPath($path); my $substituter = $config->{eval_substituter}; system("nix", "--experimental-features", "nix-command", "copy", "--from", $substituter, "--", $path) if defined $substituter; - return isValidPath($path); + return $MACHINE_LOCAL_STORE->isValidPath($path); } @@ -143,7 +143,7 @@ sub fetchInputBuild { , version => $version , outputName => $mainOutput->name }; - if (isValidPath($prevBuild->drvpath)) { + if ($MACHINE_LOCAL_STORE->isValidPath($prevBuild->drvpath)) { $result->{drvPath} = $prevBuild->drvpath; } @@ -233,7 +233,7 @@ sub fetchInputEval { my $out = $build->buildoutputs->find({ name => "out" }); next unless defined $out; # FIXME: Should we fail if the path is not valid? - next unless isValidPath($out->path); + next unless $MACHINE_LOCAL_STORE->isValidPath($out->path); $jobs->{$build->get_column('job')} = $out->path; } diff --git a/src/script/hydra-update-gc-roots b/src/script/hydra-update-gc-roots index 11eba7a6a..130cbb682 100755 --- a/src/script/hydra-update-gc-roots +++ b/src/script/hydra-update-gc-roots @@ -5,7 +5,6 @@ use warnings; use File::Path; use File::stat; use File::Basename; -use Nix::Store; use Hydra::Config; use Hydra::Schema; use Hydra::Helper::Nix; @@ -47,7 +46,7 @@ sub keepBuild { $build->finished && ($build->buildstatus == 0 || $build->buildstatus == 6)) { foreach my $path (split / /, $build->get_column('outpaths')) { - if (isValidPath($path)) { + if ($MACHINE_LOCAL_STORE->isValidPath($path)) { addRoot $path; } else { print STDERR " warning: output ", $path, " has disappeared\n" if $build->finished; @@ -55,7 +54,7 @@ sub keepBuild { } } if (!$build->finished || ($keepFailedDrvs && $build->buildstatus != 0)) { - if (isValidPath($build->drvpath)) { + if ($MACHINE_LOCAL_STORE->isValidPath($build->drvpath)) { addRoot $build->drvpath; } else { print STDERR " warning: derivation ", $build->drvpath, " has disappeared\n"; diff --git a/t/s3-backup-test.pl b/t/s3-backup-test.pl index 9332ab15b..a367981fe 100644 --- a/t/s3-backup-test.pl +++ b/t/s3-backup-test.pl @@ -3,7 +3,6 @@ use File::Basename; use Hydra::Model::DB; use Hydra::Helper::Nix; -use Nix::Store; use Cwd; my $db = Hydra::Model::DB->new;