Merge pull request #225051 from ShamrockLee/go-module-overlay-stdenv

buildGoModule: Fix overriding with overlay-style stdenv

doc/languages-frameworks/go.section.md

@@ -62,6 +62,65 @@ The following is an example expression using `buildGoModule`:
 }
 ```
 
+### Obtaining and overriding `vendorHash` for `buildGoModule` {#buildGoModule-vendorHash}
+
+We can use `nix-prefetch` to obtain the actual hash. The following command gets the value of `vendorHash` for package `pet`:
+
+```sh
+cd path/to/nixpkgs
+nix-prefetch -E "{ sha256 }: ((import ./. { }).my-package.overrideAttrs { vendorHash = sha256; }).goModules"
+```
+
+To obtain the hash without external tools, set `vendorHash = lib.fakeHash;` and run the build. ([more details here](#sec-source-hashes)).
+
+`vendorHash` can be overridden with `overrideAttrs`. Override the above example like this:
+
+```nix
+{
+  pet_0_4_0 = pet.overrideAttrs (
+    finalAttrs: previousAttrs: {
+      version = "0.4.0";
+      src = fetchFromGitHub {
+        inherit (previousAttrs.src) owner repo;
+        rev = "v${finalAttrs.version}";
+        hash = "sha256-gVTpzmXekQxGMucDKskGi+e+34nJwwsXwvQTjRO6Gdg=";
+      };
+      vendorHash = "sha256-dUvp7FEW09V0xMuhewPGw3TuAic/sD7xyXEYviZ2Ivs=";
+    }
+  );
+}
+```
+
+### Overriding `goModules` (#buildGoModule-goModules-override)
+
+Overriding `<pkg>.goModules` by calling `goModules.overrideAttrs` is unsupported. Still, it is possible to override the `vendorHash` (`goModules`'s `outputHash`) and the `pre`/`post` hooks for both the build and patch phases of the primary and `goModules` derivation. Alternatively, the primary derivation provides an overridable `passthru.overrideModAttrs` function to store the attribute overlay implicitly taken by `goModules.overrideAttrs`. Here's an example usage of `overrideModAttrs`:
+
+```nix
+{
+  pet-overridden = pet.overrideAttrs (
+    finalAttrs: previousAttrs: {
+      passthru = previousAttrs.passthru // {
+        # If the original package has an `overrideModAttrs` attribute set, you'd
+        # want to extend it, and not replace it. Hence we use
+        # `lib.composeExtensions`. If you are sure the `overrideModAttrs` of the
+        # original package trivially does nothing, you can safely replace it
+        # with your own by not using `lib.composeExtensions`.
+        overrideModAttrs = lib.composeExtensions previousAttrs.passthru.overrideModAttrs (
+          finalModAttrs: previousModAttrs: {
+            # goModules-specific overriding goes here
+            postBuild = ''
+              # Here you have access to the `vendor` directory.
+              substituteInPlace vendor/github.com/example/repo/file.go \
+                --replace-fail "panic(err)" ""
+            '';
+          }
+        );
+      };
+    }
+  );
+}
+```
+
 ## `buildGoPackage` (legacy) {#ssec-go-legacy}
 
 The function `buildGoPackage` builds legacy Go programs, not supporting Go modules.

nixos/doc/manual/release-notes/rl-2411.section.md

@@ -328,6 +328,10 @@
   [buildRustPackage: Compiling Rust applications with Cargo](https://nixos.org/manual/nixpkgs/unstable/#compiling-rust-applications-with-cargo)
   for more information.
 
+- The `vendorHash` of Go packages built with `buildGoModule` can now be overridden with `overrideAttrs`.
+  `goModules`, `modRoot`, `vendorHash`, `deleteVendor`, and `proxyVendor` are now passed as derivation attributes.
+  `goModules` and `vendorHash` are no longer placed t under `passthru`.
+
 - `hareHook` has been added as the language framework for Hare. From now on, it,
   not the `hare` package, should be added to `nativeBuildInputs` when building
   Hare programs.

pkgs/build-support/go/module.nix

@@ -7,7 +7,7 @@
 , patches ? [ ]
 
   # A function to override the goModules derivation
-, overrideModAttrs ? (_oldAttrs: { })
+, overrideModAttrs ? (finalAttrs: previousAttrs: { })
 
   # path to go.mod and go.sum directory
 , modRoot ? "./"
@@ -58,34 +58,54 @@
 assert goPackagePath != "" -> throw "`goPackagePath` is not needed with `buildGoModule`";
 
 let
-  args = removeAttrs args' [ "overrideModAttrs" "vendorSha256" "vendorHash" ];
+  args = removeAttrs args' [ "overrideModAttrs" "vendorSha256" ];
 
   GO111MODULE = "on";
   GOTOOLCHAIN = "local";
 
-  goModules = if (vendorHash == null) then "" else
+  toExtension =
+    overlay0:
+    if lib.isFunction overlay0 then
+      final: prev:
+      if lib.isFunction (overlay0 prev) then
+        # `overlay0` is `final: prev: { ... }`
+        overlay0 final prev
+      else
+        # `overlay0` is `prev: { ... }`
+        overlay0 prev
+    else
+      # `overlay0` is `{ ... }`
+      final: prev: overlay0;
+
+in
+(stdenv.mkDerivation (finalAttrs:
+  args
+  // {
+
+  inherit modRoot vendorHash deleteVendor proxyVendor;
+  goModules = if (finalAttrs.vendorHash == null) then "" else
   (stdenv.mkDerivation {
-    name = "${name}-go-modules";
+    name = "${finalAttrs.name or "${finalAttrs.pname}-${finalAttrs.version}"}-go-modules";
 
-    nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ go git cacert ];
+    nativeBuildInputs = (finalAttrs.nativeBuildInputs or [ ]) ++ [ go git cacert ];
 
-    inherit (args) src;
+    inherit (finalAttrs) src modRoot;
     inherit (go) GOOS GOARCH;
     inherit GO111MODULE GOTOOLCHAIN;
 
     # The following inheritence behavior is not trivial to expect, and some may
     # argue it's not ideal. Changing it may break vendor hashes in Nixpkgs and
     # out in the wild. In anycase, it's documented in:
     # doc/languages-frameworks/go.section.md
-    prePatch = args.prePatch or "";
-    patches = args.patches or [ ];
-    patchFlags = args.patchFlags or [ ];
-    postPatch = args.postPatch or "";
-    preBuild = args.preBuild or "";
-    postBuild = args.modPostBuild or "";
-    sourceRoot = args.sourceRoot or "";
-    setSourceRoot = args.setSourceRoot or "";
-    env = args.env or { };
+    prePatch = finalAttrs.prePatch or "";
+    patches = finalAttrs.patches or [ ];
+    patchFlags = finalAttrs.patchFlags or [ ];
+    postPatch = finalAttrs.postPatch or "";
+    preBuild = finalAttrs.preBuild or "";
+    postBuild = finalAttrs.modPostBuild or "";
+    sourceRoot = finalAttrs.sourceRoot or "";
+    setSourceRoot = finalAttrs.setSourceRoot or "";
+    env = finalAttrs.env or { };
 
     impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [
       "GIT_PROXY_COMMAND"
@@ -97,13 +117,13 @@ let
       runHook preConfigure
       export GOCACHE=$TMPDIR/go-cache
       export GOPATH="$TMPDIR/go"
-      cd "${modRoot}"
+      cd "$modRoot"
       runHook postConfigure
     '';
 
     buildPhase = args.modBuildPhase or (''
       runHook preBuild
-    '' + lib.optionalString deleteVendor ''
+    '' + lib.optionalString finalAttrs.deleteVendor ''
       if [ ! -d vendor ]; then
         echo "vendor folder does not exist, 'deleteVendor' is not needed"
         exit 10
@@ -116,7 +136,7 @@ let
         exit 10
       fi
 
-      ${if proxyVendor then ''
+      ${if finalAttrs.proxyVendor then ''
         mkdir -p "''${GOPATH}/pkg/mod/cache/download"
         go mod download
       '' else ''
@@ -134,7 +154,7 @@ let
     installPhase = args.modInstallPhase or ''
       runHook preInstall
 
-      ${if proxyVendor then ''
+      ${if finalAttrs.proxyVendor then ''
         rm -rf "''${GOPATH}/pkg/mod/cache/download/sumdb"
         cp -r --reflink=auto "''${GOPATH}/pkg/mod/cache/download" $out
       '' else ''
@@ -152,20 +172,19 @@ let
     dontFixup = true;
 
     outputHashMode = "recursive";
-    outputHash = vendorHash;
+    outputHash = finalAttrs.vendorHash;
     # Handle empty vendorHash; avoid
     # error: empty hash requires explicit hash algorithm
-    outputHashAlgo = if vendorHash == "" then "sha256" else null;
-  }).overrideAttrs overrideModAttrs;
+    outputHashAlgo = if finalAttrs.vendorHash == "" then "sha256" else null;
+  }).overrideAttrs finalAttrs.passthru.overrideModAttrs;
 
-  package = stdenv.mkDerivation (args // {
     nativeBuildInputs = [ go ] ++ nativeBuildInputs;
 
     inherit (go) GOOS GOARCH;
 
     GOFLAGS = GOFLAGS
       ++ lib.warnIf (lib.any (lib.hasPrefix "-mod=") GOFLAGS) "use `proxyVendor` to control Go module/vendor behavior instead of setting `-mod=` in GOFLAGS"
-        (lib.optional (!proxyVendor) "-mod=vendor")
+        (lib.optional (!finalAttrs.proxyVendor) "-mod=vendor")
       ++ lib.warnIf (builtins.elem "-trimpath" GOFLAGS) "`-trimpath` is added by default to GOFLAGS by buildGoModule when allowGoReference isn't set to true"
         (lib.optional (!allowGoReference) "-trimpath");
     inherit CGO_ENABLED enableParallelBuilding GO111MODULE GOTOOLCHAIN;
@@ -181,12 +200,12 @@ let
       export GOPROXY=off
       export GOSUMDB=off
       cd "$modRoot"
-    '' + lib.optionalString (vendorHash != null) ''
-      ${if proxyVendor then ''
-        export GOPROXY=file://${goModules}
+    '' + lib.optionalString (finalAttrs.vendorHash != null) ''
+      ${if finalAttrs.proxyVendor then ''
+        export GOPROXY="file://$goModules"
       '' else ''
         rm -rf vendor
-        cp -r --reflink=auto ${goModules} vendor
+        cp -r --reflink=auto "$goModules" vendor
       ''}
     '' + ''
 
@@ -307,12 +326,17 @@ let
 
     disallowedReferences = lib.optional (!allowGoReference) go;
 
-    passthru = passthru // { inherit go goModules vendorHash; };
+    passthru = {
+      inherit go;
+      # Canonicallize `overrideModAttrs` as an attribute overlay.
+      # `passthru.overrideModAttrs` will be overridden
+      # when users want to override `goModules`.
+      overrideModAttrs = toExtension overrideModAttrs;
+    } // passthru;
 
     meta = {
       # Add default meta information
       platforms = go.meta.platforms or lib.platforms.all;
     } // meta;
-  });
-in
-package
+  }
+))

pkgs/test/overriding.nix

@@ -27,6 +27,38 @@ let
         expr = ((stdenvNoCC.mkDerivation { pname = "hello-no-final-attrs"; }).overrideAttrs { pname = "hello-no-final-attrs-overridden"; }).pname == "hello-no-final-attrs-overridden";
         expected = true;
       };
+      buildGoModule-overrideAttrs = {
+        expr = lib.all (
+          attrPath:
+          let
+            attrPathPretty = lib.concatStringsSep "." attrPath;
+            valueNative = lib.getAttrFromPath attrPath pet_0_4_0;
+            valueOverridden = lib.getAttrFromPath attrPath pet_0_4_0-overridden;
+          in
+          lib.warnIfNot
+            (valueNative == valueOverridden)
+            "pet_0_4_0.${attrPathPretty} (${valueNative}) does not equal pet_0_4_0-overridden.${attrPathPretty} (${valueOverridden})"
+            true
+        ) [
+          [ "drvPath" ]
+          [ "name" ]
+          [ "pname" ]
+          [ "version" ]
+          [ "vendorHash" ]
+          [ "goModules" "drvPath" ]
+          [ "goModules" "name" ]
+          [ "goModules" "outputHash" ]
+        ];
+        expected = true;
+      };
+      buildGoModule-goModules-overrideAttrs = {
+        expr = pet-foo.goModules.FOO == "foo";
+        expected = true;
+      };
+      buildGoModule-goModules-overrideAttrs-vendored = {
+        expr = lib.isString pet-vendored.drvPath;
+        expected = true;
+      };
     };
 
   addEntangled = origOverrideAttrs: f:
@@ -51,6 +83,74 @@ let
   overrides1 = example.overrideAttrs (_: super: { pname = "a-better-${super.pname}"; });
 
   repeatedOverrides = overrides1.overrideAttrs (_: super: { pname = "${super.pname}-with-blackjack"; });
+
+  pet_0_3_4 = pkgs.buildGoModule rec {
+    pname = "pet";
+    version = "0.3.4";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "knqyf263";
+      repo = "pet";
+      rev = "v${version}";
+      hash = "sha256-Gjw1dRrgM8D3G7v6WIM2+50r4HmTXvx0Xxme2fH9TlQ=";
+    };
+
+    vendorHash = "sha256-ciBIR+a1oaYH+H1PcC8cD8ncfJczk1IiJ8iYNM+R6aA=";
+
+    meta = {
+      description = "Simple command-line snippet manager, written in Go";
+      homepage = "https://github.com/knqyf263/pet";
+      license = lib.licenses.mit;
+      maintainers = with lib.maintainers; [ kalbasit ];
+    };
+  };
+
+  pet_0_4_0 = pkgs.buildGoModule rec {
+    pname = "pet";
+    version = "0.4.0";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "knqyf263";
+      repo = "pet";
+      rev = "v${version}";
+      hash = "sha256-gVTpzmXekQxGMucDKskGi+e+34nJwwsXwvQTjRO6Gdg=";
+    };
+
+    vendorHash = "sha256-dUvp7FEW09V0xMuhewPGw3TuAic/sD7xyXEYviZ2Ivs=";
+
+    meta = {
+      description = "Simple command-line snippet manager, written in Go";
+      homepage = "https://github.com/knqyf263/pet";
+      license = lib.licenses.mit;
+      maintainers = with lib.maintainers; [ kalbasit ];
+    };
+  };
+
+  pet_0_4_0-overridden = pet_0_3_4.overrideAttrs (finalAttrs: previousAttrs: {
+  version = "0.4.0";
+
+  src = pkgs.fetchFromGitHub {
+    inherit (previousAttrs.src) owner repo;
+    rev = "v${finalAttrs.version}";
+    hash = "sha256-gVTpzmXekQxGMucDKskGi+e+34nJwwsXwvQTjRO6Gdg=";
+  };
+
+  vendorHash = "sha256-dUvp7FEW09V0xMuhewPGw3TuAic/sD7xyXEYviZ2Ivs=";
+  });
+
+  pet-foo = pet_0_3_4.overrideAttrs (
+    finalAttrs: previousAttrs: {
+      passthru = previousAttrs.passthru // {
+        overrideModAttrs = lib.composeExtensions previousAttrs.passthru.overrideModAttrs (
+          finalModAttrs: previousModAttrs: {
+            FOO = "foo";
+          }
+        );
+      };
+    }
+  );
+
+  pet-vendored = pet-foo.overrideAttrs { vendorHash = null; };
 in
 
 stdenvNoCC.mkDerivation {