From 010277de84ba988922adf23e8cb1ad3890c5ff62 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 29 Jul 2024 06:45:08 +0800 Subject: [PATCH 1/2] buildGoModule: respect user-specified passthru Prioritize user-specified passthru arguments over those added by buildGoModule. --- pkgs/build-support/go/module.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/go/module.nix b/pkgs/build-support/go/module.nix index afe97d12ea6ab..a9c09f4ccbb5b 100644 --- a/pkgs/build-support/go/module.nix +++ b/pkgs/build-support/go/module.nix @@ -307,7 +307,7 @@ let disallowedReferences = lib.optional (!allowGoReference) go; - passthru = passthru // { inherit go goModules vendorHash; }; + passthru = { inherit go goModules vendorHash; } // passthru; meta = { # Add default meta information From eed069a5bc40ba4d871de7700c7eb8d592e98cb6 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 29 Jul 2024 03:15:12 +0800 Subject: [PATCH 2/2] buildGoModule: fix overrideAttrs overriding Fix overriding of vendorHash and various attributes via the fixed point attribute support of stdenv.mkDerivation. Pass as derivation attributes goModules, modRoot, vendorHash, deleteVendor, and proxyVendor. Move goModules and vendorHash out of passthru. Co-authored-by: Doron Behar --- doc/languages-frameworks/go.section.md | 59 +++++++++++ .../manual/release-notes/rl-2411.section.md | 4 + pkgs/build-support/go/module.nix | 88 +++++++++------ pkgs/test/overriding.nix | 100 ++++++++++++++++++ 4 files changed, 219 insertions(+), 32 deletions(-) diff --git a/doc/languages-frameworks/go.section.md b/doc/languages-frameworks/go.section.md index b20308c7b4ab1..e40b92f952b1c 100644 --- a/doc/languages-frameworks/go.section.md +++ b/doc/languages-frameworks/go.section.md @@ -62,6 +62,65 @@ The following is an example expression using `buildGoModule`: } ``` +### Obtaining and overriding `vendorHash` for `buildGoModule` {#buildGoModule-vendorHash} + +We can use `nix-prefetch` to obtain the actual hash. The following command gets the value of `vendorHash` for package `pet`: + +```sh +cd path/to/nixpkgs +nix-prefetch -E "{ sha256 }: ((import ./. { }).my-package.overrideAttrs { vendorHash = sha256; }).goModules" +``` + +To obtain the hash without external tools, set `vendorHash = lib.fakeHash;` and run the build. ([more details here](#sec-source-hashes)). + +`vendorHash` can be overridden with `overrideAttrs`. Override the above example like this: + +```nix +{ + pet_0_4_0 = pet.overrideAttrs ( + finalAttrs: previousAttrs: { + version = "0.4.0"; + src = fetchFromGitHub { + inherit (previousAttrs.src) owner repo; + rev = "v${finalAttrs.version}"; + hash = "sha256-gVTpzmXekQxGMucDKskGi+e+34nJwwsXwvQTjRO6Gdg="; + }; + vendorHash = "sha256-dUvp7FEW09V0xMuhewPGw3TuAic/sD7xyXEYviZ2Ivs="; + } + ); +} +``` + +### Overriding `goModules` (#buildGoModule-goModules-override) + +Overriding `.goModules` by calling `goModules.overrideAttrs` is unsupported. Still, it is possible to override the `vendorHash` (`goModules`'s `outputHash`) and the `pre`/`post` hooks for both the build and patch phases of the primary and `goModules` derivation. Alternatively, the primary derivation provides an overridable `passthru.overrideModAttrs` function to store the attribute overlay implicitly taken by `goModules.overrideAttrs`. Here's an example usage of `overrideModAttrs`: + +```nix +{ + pet-overridden = pet.overrideAttrs ( + finalAttrs: previousAttrs: { + passthru = previousAttrs.passthru // { + # If the original package has an `overrideModAttrs` attribute set, you'd + # want to extend it, and not replace it. Hence we use + # `lib.composeExtensions`. If you are sure the `overrideModAttrs` of the + # original package trivially does nothing, you can safely replace it + # with your own by not using `lib.composeExtensions`. + overrideModAttrs = lib.composeExtensions previousAttrs.passthru.overrideModAttrs ( + finalModAttrs: previousModAttrs: { + # goModules-specific overriding goes here + postBuild = '' + # Here you have access to the `vendor` directory. + substituteInPlace vendor/github.com/example/repo/file.go \ + --replace-fail "panic(err)" "" + ''; + } + ); + }; + } + ); +} +``` + ## `buildGoPackage` (legacy) {#ssec-go-legacy} The function `buildGoPackage` builds legacy Go programs, not supporting Go modules. diff --git a/nixos/doc/manual/release-notes/rl-2411.section.md b/nixos/doc/manual/release-notes/rl-2411.section.md index 3185137f971fd..acc2f2177f949 100644 --- a/nixos/doc/manual/release-notes/rl-2411.section.md +++ b/nixos/doc/manual/release-notes/rl-2411.section.md @@ -272,6 +272,10 @@ [buildRustPackage: Compiling Rust applications with Cargo](https://nixos.org/manual/nixpkgs/unstable/#compiling-rust-applications-with-cargo) for more information. +- The `vendorHash` of Go packages built with `buildGoModule` can now be overridden with `overrideAttrs`. + `goModules`, `modRoot`, `vendorHash`, `deleteVendor`, and `proxyVendor` are now passed as derivation attributes. + `goModules` and `vendorHash` are no longer placed t under `passthru`. + - `hareHook` has been added as the language framework for Hare. From now on, it, not the `hare` package, should be added to `nativeBuildInputs` when building Hare programs. diff --git a/pkgs/build-support/go/module.nix b/pkgs/build-support/go/module.nix index a9c09f4ccbb5b..214b2603ff7f3 100644 --- a/pkgs/build-support/go/module.nix +++ b/pkgs/build-support/go/module.nix @@ -7,7 +7,7 @@ , patches ? [ ] # A function to override the goModules derivation -, overrideModAttrs ? (_oldAttrs: { }) +, overrideModAttrs ? (finalAttrs: previousAttrs: { }) # path to go.mod and go.sum directory , modRoot ? "./" @@ -58,18 +58,38 @@ assert goPackagePath != "" -> throw "`goPackagePath` is not needed with `buildGoModule`"; let - args = removeAttrs args' [ "overrideModAttrs" "vendorSha256" "vendorHash" ]; + args = removeAttrs args' [ "overrideModAttrs" "vendorSha256" ]; GO111MODULE = "on"; GOTOOLCHAIN = "local"; - goModules = if (vendorHash == null) then "" else + toExtension = + overlay0: + if lib.isFunction overlay0 then + final: prev: + if lib.isFunction (overlay0 prev) then + # `overlay0` is `final: prev: { ... }` + overlay0 final prev + else + # `overlay0` is `prev: { ... }` + overlay0 prev + else + # `overlay0` is `{ ... }` + final: prev: overlay0; + +in +(stdenv.mkDerivation (finalAttrs: + args + // { + + inherit modRoot vendorHash deleteVendor proxyVendor; + goModules = if (finalAttrs.vendorHash == null) then "" else (stdenv.mkDerivation { - name = "${name}-go-modules"; + name = "${finalAttrs.name or "${finalAttrs.pname}-${finalAttrs.version}"}-go-modules"; - nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ go git cacert ]; + nativeBuildInputs = (finalAttrs.nativeBuildInputs or [ ]) ++ [ go git cacert ]; - inherit (args) src; + inherit (finalAttrs) src modRoot; inherit (go) GOOS GOARCH; inherit GO111MODULE GOTOOLCHAIN; @@ -77,15 +97,15 @@ let # argue it's not ideal. Changing it may break vendor hashes in Nixpkgs and # out in the wild. In anycase, it's documented in: # doc/languages-frameworks/go.section.md - prePatch = args.prePatch or ""; - patches = args.patches or [ ]; - patchFlags = args.patchFlags or [ ]; - postPatch = args.postPatch or ""; - preBuild = args.preBuild or ""; - postBuild = args.modPostBuild or ""; - sourceRoot = args.sourceRoot or ""; - setSourceRoot = args.setSourceRoot or ""; - env = args.env or { }; + prePatch = finalAttrs.prePatch or ""; + patches = finalAttrs.patches or [ ]; + patchFlags = finalAttrs.patchFlags or [ ]; + postPatch = finalAttrs.postPatch or ""; + preBuild = finalAttrs.preBuild or ""; + postBuild = finalAttrs.modPostBuild or ""; + sourceRoot = finalAttrs.sourceRoot or ""; + setSourceRoot = finalAttrs.setSourceRoot or ""; + env = finalAttrs.env or { }; impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ "GIT_PROXY_COMMAND" @@ -97,13 +117,13 @@ let runHook preConfigure export GOCACHE=$TMPDIR/go-cache export GOPATH="$TMPDIR/go" - cd "${modRoot}" + cd "$modRoot" runHook postConfigure ''; buildPhase = args.modBuildPhase or ('' runHook preBuild - '' + lib.optionalString deleteVendor '' + '' + lib.optionalString finalAttrs.deleteVendor '' if [ ! -d vendor ]; then echo "vendor folder does not exist, 'deleteVendor' is not needed" exit 10 @@ -116,7 +136,7 @@ let exit 10 fi - ${if proxyVendor then '' + ${if finalAttrs.proxyVendor then '' mkdir -p "''${GOPATH}/pkg/mod/cache/download" go mod download '' else '' @@ -134,7 +154,7 @@ let installPhase = args.modInstallPhase or '' runHook preInstall - ${if proxyVendor then '' + ${if finalAttrs.proxyVendor then '' rm -rf "''${GOPATH}/pkg/mod/cache/download/sumdb" cp -r --reflink=auto "''${GOPATH}/pkg/mod/cache/download" $out '' else '' @@ -152,20 +172,19 @@ let dontFixup = true; outputHashMode = "recursive"; - outputHash = vendorHash; + outputHash = finalAttrs.vendorHash; # Handle empty vendorHash; avoid # error: empty hash requires explicit hash algorithm - outputHashAlgo = if vendorHash == "" then "sha256" else null; - }).overrideAttrs overrideModAttrs; + outputHashAlgo = if finalAttrs.vendorHash == "" then "sha256" else null; + }).overrideAttrs finalAttrs.passthru.overrideModAttrs; - package = stdenv.mkDerivation (args // { nativeBuildInputs = [ go ] ++ nativeBuildInputs; inherit (go) GOOS GOARCH; GOFLAGS = GOFLAGS ++ lib.warnIf (lib.any (lib.hasPrefix "-mod=") GOFLAGS) "use `proxyVendor` to control Go module/vendor behavior instead of setting `-mod=` in GOFLAGS" - (lib.optional (!proxyVendor) "-mod=vendor") + (lib.optional (!finalAttrs.proxyVendor) "-mod=vendor") ++ lib.warnIf (builtins.elem "-trimpath" GOFLAGS) "`-trimpath` is added by default to GOFLAGS by buildGoModule when allowGoReference isn't set to true" (lib.optional (!allowGoReference) "-trimpath"); inherit CGO_ENABLED enableParallelBuilding GO111MODULE GOTOOLCHAIN; @@ -181,12 +200,12 @@ let export GOPROXY=off export GOSUMDB=off cd "$modRoot" - '' + lib.optionalString (vendorHash != null) '' - ${if proxyVendor then '' - export GOPROXY=file://${goModules} + '' + lib.optionalString (finalAttrs.vendorHash != null) '' + ${if finalAttrs.proxyVendor then '' + export GOPROXY="file://$goModules" '' else '' rm -rf vendor - cp -r --reflink=auto ${goModules} vendor + cp -r --reflink=auto "$goModules" vendor ''} '' + '' @@ -307,12 +326,17 @@ let disallowedReferences = lib.optional (!allowGoReference) go; - passthru = { inherit go goModules vendorHash; } // passthru; + passthru = { + inherit go; + # Canonicallize `overrideModAttrs` as an attribute overlay. + # `passthru.overrideModAttrs` will be overridden + # when users want to override `goModules`. + overrideModAttrs = toExtension overrideModAttrs; + } // passthru; meta = { # Add default meta information platforms = go.meta.platforms or lib.platforms.all; } // meta; - }); -in -package + } +)) diff --git a/pkgs/test/overriding.nix b/pkgs/test/overriding.nix index e4c1f257c8190..a9fa482e4e588 100644 --- a/pkgs/test/overriding.nix +++ b/pkgs/test/overriding.nix @@ -27,6 +27,38 @@ let expr = ((stdenvNoCC.mkDerivation { pname = "hello-no-final-attrs"; }).overrideAttrs { pname = "hello-no-final-attrs-overridden"; }).pname == "hello-no-final-attrs-overridden"; expected = true; }; + buildGoModule-overrideAttrs = { + expr = lib.all ( + attrPath: + let + attrPathPretty = lib.concatStringsSep "." attrPath; + valueNative = lib.getAttrFromPath attrPath pet_0_4_0; + valueOverridden = lib.getAttrFromPath attrPath pet_0_4_0-overridden; + in + lib.warnIfNot + (valueNative == valueOverridden) + "pet_0_4_0.${attrPathPretty} (${valueNative}) does not equal pet_0_4_0-overridden.${attrPathPretty} (${valueOverridden})" + true + ) [ + [ "drvPath" ] + [ "name" ] + [ "pname" ] + [ "version" ] + [ "vendorHash" ] + [ "goModules" "drvPath" ] + [ "goModules" "name" ] + [ "goModules" "outputHash" ] + ]; + expected = true; + }; + buildGoModule-goModules-overrideAttrs = { + expr = pet-foo.goModules.FOO == "foo"; + expected = true; + }; + buildGoModule-goModules-overrideAttrs-vendored = { + expr = lib.isString pet-vendored.drvPath; + expected = true; + }; }; addEntangled = origOverrideAttrs: f: @@ -51,6 +83,74 @@ let overrides1 = example.overrideAttrs (_: super: { pname = "a-better-${super.pname}"; }); repeatedOverrides = overrides1.overrideAttrs (_: super: { pname = "${super.pname}-with-blackjack"; }); + + pet_0_3_4 = pkgs.buildGoModule rec { + pname = "pet"; + version = "0.3.4"; + + src = pkgs.fetchFromGitHub { + owner = "knqyf263"; + repo = "pet"; + rev = "v${version}"; + hash = "sha256-Gjw1dRrgM8D3G7v6WIM2+50r4HmTXvx0Xxme2fH9TlQ="; + }; + + vendorHash = "sha256-ciBIR+a1oaYH+H1PcC8cD8ncfJczk1IiJ8iYNM+R6aA="; + + meta = { + description = "Simple command-line snippet manager, written in Go"; + homepage = "https://github.com/knqyf263/pet"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ kalbasit ]; + }; + }; + + pet_0_4_0 = pkgs.buildGoModule rec { + pname = "pet"; + version = "0.4.0"; + + src = pkgs.fetchFromGitHub { + owner = "knqyf263"; + repo = "pet"; + rev = "v${version}"; + hash = "sha256-gVTpzmXekQxGMucDKskGi+e+34nJwwsXwvQTjRO6Gdg="; + }; + + vendorHash = "sha256-dUvp7FEW09V0xMuhewPGw3TuAic/sD7xyXEYviZ2Ivs="; + + meta = { + description = "Simple command-line snippet manager, written in Go"; + homepage = "https://github.com/knqyf263/pet"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ kalbasit ]; + }; + }; + + pet_0_4_0-overridden = pet_0_3_4.overrideAttrs (finalAttrs: previousAttrs: { + version = "0.4.0"; + + src = pkgs.fetchFromGitHub { + inherit (previousAttrs.src) owner repo; + rev = "v${finalAttrs.version}"; + hash = "sha256-gVTpzmXekQxGMucDKskGi+e+34nJwwsXwvQTjRO6Gdg="; + }; + + vendorHash = "sha256-dUvp7FEW09V0xMuhewPGw3TuAic/sD7xyXEYviZ2Ivs="; + }); + + pet-foo = pet_0_3_4.overrideAttrs ( + finalAttrs: previousAttrs: { + passthru = previousAttrs.passthru // { + overrideModAttrs = lib.composeExtensions previousAttrs.passthru.overrideModAttrs ( + finalModAttrs: previousModAttrs: { + FOO = "foo"; + } + ); + }; + } + ); + + pet-vendored = pet-foo.overrideAttrs { vendorHash = null; }; in stdenvNoCC.mkDerivation {