dotnet: infrastructure improvements

doc/languages-frameworks/dotnet.section.md

@@ -219,7 +219,7 @@ buildDotnetGlobalTool {
 ## Generating and updating NuGet dependencies {#generating-and-updating-nuget-dependencies}
 
 When writing a new expression, you can use the generated `fetch-deps` script to initialise the lockfile.
-After creating a blank `deps.nix` and pointing `nugetDeps` to it,
+After setting `nugetDeps` to the desired location of the lockfile (e.g. `./deps.nix`),
 build the script with `nix-build -A package.fetch-deps` and then run the result.
 (When the root attr is your package, it's simply `nix-build -A fetch-deps`.)
 

maintainers/scripts/update-dotnet-lockfiles.nix

@@ -9,6 +9,7 @@
   infrastructure. Regular updates should be done through the individual packages
   update scripts.
  */
+{ startWith ? null }:
 let
   pkgs = import ../.. { config.allowAliases = false; };
 
@@ -17,28 +18,31 @@ let
   packagesWith = cond: pkgs:
     let
       packagesWithInner = attrs:
-        lib.unique (
-          lib.concatLists (
-            lib.mapAttrsToList (name: elem:
-              let
-                result = builtins.tryEval elem;
-              in
-                if result.success then
-                  let
-                    value = result.value;
-                  in
-                    if lib.isDerivation value then
-                      lib.optional (cond value) value
-                    else
-                      if lib.isAttrs value && (value.recurseForDerivations or false || value.recurseForRelease or false) then
-                        packagesWithInner value
-                      else []
-                else []) attrs));
+        lib.concatLists (
+          lib.mapAttrsToList (name: elem:
+            let
+              result = builtins.tryEval elem;
+            in
+              if result.success then
+                let
+                  value = result.value;
+                in
+                  if lib.isDerivation value then
+                    lib.optional (cond value) value
+                  else
+                    if lib.isAttrs value && (value.recurseForDerivations or false || value.recurseForRelease or false) then
+                      packagesWithInner value
+                    else []
+              else []) attrs);
     in
       packagesWithInner pkgs;
 
-  packages =
-    packagesWith (pkgs: pkgs ? fetch-deps) pkgs;
+  packages = lib.unique
+    (lib.filter (p:
+      (builtins.tryEval p.outPath).success ||
+      builtins.trace "warning: skipping ${p.name} because it failed to evaluate" false)
+    ((pkgs: (lib.drop (lib.lists.findFirstIndex (p: p.name == startWith) 0 pkgs) pkgs))
+    (packagesWith (p: p ? fetch-deps) pkgs)));
 
   helpText = ''
     Please run:

pkgs/applications/audio/openutau/default.nix

@@ -22,6 +22,10 @@ buildDotnetModule rec {
   dotnet-sdk = dotnetCorePackages.sdk_7_0;
   dotnet-runtime = dotnetCorePackages.runtime_7_0;
 
+  # [...]/Microsoft.NET.Sdk.targets(157,5): error MSB4018: The "GenerateDepsFile" task failed unexpectedly. [[...]/OpenUtau.Core.csproj]
+  # [...]/Microsoft.NET.Sdk.targets(157,5): error MSB4018: System.IO.IOException: The process cannot access the file '[...]/OpenUtau.Core.deps.json' because it is being used by another process. [[...]/OpenUtau.Core.csproj]
+  enableParallelBuilding = false;
+
   projectFile = "OpenUtau.sln";
   nugetDeps = ./deps.nix;
 

pkgs/build-support/dotnet/add-nuget-deps/default.nix

@@ -0,0 +1,108 @@
+{
+  writeShellScript,
+  runtimeShell,
+  nix,
+  lib,
+  substituteAll,
+  nuget-to-nix,
+  cacert,
+  fetchNupkg,
+  callPackage,
+}:
+
+{
+  nugetDeps,
+  overrideFetchAttrs ? x: { },
+}:
+fnOrAttrs: finalAttrs:
+let
+  attrs = if builtins.isFunction fnOrAttrs then fnOrAttrs finalAttrs else fnOrAttrs;
+
+  deps =
+    if (nugetDeps != null) then
+      if lib.isDerivation nugetDeps then
+        [ nugetDeps ]
+      else if lib.isList nugetDeps then
+        nugetDeps
+      else
+        assert (lib.isPath nugetDeps);
+        callPackage nugetDeps { fetchNuGet = fetchNupkg; }
+    else
+      [ ];
+
+  finalPackage = finalAttrs.finalPackage;
+
+in
+attrs
+// {
+  buildInputs = attrs.buildInputs or [ ] ++ deps;
+
+  passthru =
+    attrs.passthru or { }
+    // {
+      nugetDeps = deps;
+    }
+    // lib.optionalAttrs (nugetDeps == null || lib.isPath nugetDeps) rec {
+      fetch-drv =
+        let
+          pkg' = finalPackage.overrideAttrs (old: {
+            buildInputs = attrs.buildInputs or [ ];
+            nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [ cacert ];
+            keepNugetConfig = true;
+            dontBuild = true;
+            doCheck = false;
+            dontInstall = true;
+            doInstallCheck = false;
+            dontFixup = true;
+            doDist = false;
+          });
+        in
+        pkg'.overrideAttrs overrideFetchAttrs;
+      fetch-deps =
+        let
+          drv = builtins.unsafeDiscardOutputDependency fetch-drv.drvPath;
+
+          innerScript = substituteAll {
+            src = ./fetch-deps.sh;
+            isExecutable = true;
+            inherit cacert;
+            nugetToNix = nuget-to-nix;
+          };
+
+          defaultDepsFile =
+            # Wire in the depsFile such that running the script with no args
+            # runs it agains the correct deps file by default.
+            # Note that toString is necessary here as it results in the path at
+            # eval time (i.e. to the file in your local Nixpkgs checkout) rather
+            # than the Nix store path of the path after it's been imported.
+            if lib.isPath nugetDeps && !lib.isStorePath nugetDeps then
+              toString nugetDeps
+            else
+              ''$(mktemp -t "${finalAttrs.pname or finalPackage.name}-deps-XXXXXX.nix")'';
+
+        in
+        writeShellScript "${finalPackage.name}-fetch-deps" ''
+          set -eu
+
+          echo 'fetching dependencies for' ${lib.escapeShellArg finalPackage.name} >&2
+
+          # this needs to be before TMPDIR is changed, so the output isn't deleted
+          # if it uses mktemp
+          depsFile=$(realpath "''${1:-${lib.escapeShellArg defaultDepsFile}}")
+
+          export TMPDIR
+          TMPDIR=$(mktemp -d -t fetch-deps-${lib.escapeShellArg finalPackage.name}.XXXXXX)
+          trap 'chmod -R +w "$TMPDIR" && rm -fr "$TMPDIR"' EXIT
+
+          export NUGET_HTTP_CACHE_PATH=''${NUGET_HTTP_CACHE_PATH-~/.local/share/NuGet/v3-cache}
+
+          HOME=$TMPDIR/home
+          mkdir "$HOME"
+
+          cd "$TMPDIR"
+
+          NIX_BUILD_SHELL=${lib.escapeShellArg runtimeShell} ${nix}/bin/nix-shell \
+            --pure --keep NUGET_HTTP_CACHE_PATH --run 'source '${lib.escapeShellArg innerScript}' '"''${depsFile@Q}" "${drv}"
+        '';
+    };
+}

pkgs/build-support/dotnet/add-nuget-deps/fetch-deps.sh

@@ -0,0 +1,11 @@
+set -e
+
+genericBuild
+
+(
+    echo -e "# This file was automatically generated by passthru.fetch-deps.\n# Please dont edit it manually, your changes might get overwritten!\n"
+    @nugetToNix@/bin/nuget-to-nix "${NUGET_PACKAGES%/}"
+) > deps.nix
+
+mv deps.nix "$1"
+echo "Succesfully wrote lockfile to $1"

pkgs/build-support/dotnet/build-dotnet-module/default.nix

@@ -7,12 +7,8 @@
   writeShellScript,
   makeWrapper,
   dotnetCorePackages,
-  fetchNupkg,
-  nuget-to-nix,
   cacert,
-  unzip,
-  yq,
-  nix,
+  addNuGetDeps,
 }:
 let
   transformArgs =
@@ -109,39 +105,9 @@ let
         dotnetFixupHook
         ;
 
-      _nugetDeps =
-        if (nugetDeps != null) then
-          if lib.isDerivation nugetDeps then
-            [ nugetDeps ]
-          else if lib.isList nugetDeps then
-            nugetDeps
-          else
-            assert (lib.isPath nugetDeps);
-            callPackage nugetDeps { fetchNuGet = fetchNupkg; }
-        else
-          [ ];
-
-      nugetDepsFile = if lib.isPath nugetDeps then nugetDeps else null;
-
       inherit (dotnetCorePackages) systemToDotnetRid;
     in
-    # Not all args need to be passed through to mkDerivation
-    # TODO: We should probably filter out even more attrs
-    removeAttrs args [
-      "nugetDeps"
-      "installPath"
-      "executables"
-      "projectFile"
-      "projectReferences"
-      "runtimeDeps"
-      "runtimeId"
-      "disabledTests"
-      "testProjectFile"
-      "buildType"
-      "selfContainedBuild"
-      "useDotnet"
-      "useAppHost"
-    ]
+    args
     // {
       dotnetInstallPath = installPath;
       dotnetExecutables = executables;
@@ -167,6 +133,8 @@ let
         dotnetFlags
         packNupkg
         useDotnetFromEnv
+        nugetDeps
+        runtimeId
         ;
 
       nativeBuildInputs = args.nativeBuildInputs or [ ] ++ [
@@ -179,11 +147,14 @@ let
         cacert
         makeWrapper
         dotnet-sdk
-        unzip
-        yq
       ];
 
-      buildInputs = args.buildInputs or [ ] ++ [ dotnet-sdk.packages ] ++ _nugetDeps ++ projectReferences;
+      buildInputs =
+        args.buildInputs or [ ]
+        ++ [
+          dotnet-sdk.packages
+        ]
+        ++ projectReferences;
 
       # Parse the version attr into a format acceptable for the Version msbuild property
       # The actual version attr is saved in InformationalVersion, which accepts an arbitrary string
@@ -223,60 +194,45 @@ let
       # executables
       propagatedSandboxProfile = toString dotnet-runtime.__propagatedSandboxProfile;
 
-      passthru =
-        {
-          nugetDeps = _nugetDeps;
-        }
-        // lib.optionalAttrs (nugetDeps == null || lib.isPath nugetDeps) {
-          fetch-deps =
-            let
-              pkg = finalAttrs.finalPackage.overrideAttrs (
-                old:
-                {
-                  buildInputs = lib.subtractLists _nugetDeps old.buildInputs;
-                  keepNugetConfig = true;
-                }
-                // lib.optionalAttrs (runtimeId == null) {
-                  dotnetRuntimeIds = map (system: systemToDotnetRid system) platforms;
-                }
-              );
-
-              drv = builtins.unsafeDiscardOutputDependency pkg.drvPath;
-
-              innerScript = substituteAll {
-                src = ./fetch-deps.sh;
-                isExecutable = true;
-                defaultDepsFile =
-                  # Wire in the nugetDeps file such that running the script with no args
-                  # runs it agains the correct deps file by default.
-                  # Note that toString is necessary here as it results in the path at
-                  # eval time (i.e. to the file in your local Nixpkgs checkout) rather
-                  # than the Nix store path of the path after it's been imported.
-                  if lib.isPath nugetDeps && !lib.isStorePath nugetDepsFile then
-                    toString nugetDepsFile
-                  else
-                    ''$(mktemp -t "${finalAttrs.pname or finalAttrs.finalPackage.name}-deps-XXXXXX.nix")'';
-                nugetToNix = (nuget-to-nix.override { inherit dotnet-sdk; });
-              };
-
-            in
-            writeShellScript "${finalAttrs.finalPackage.name}-fetch-deps" ''
-              NIX_BUILD_SHELL="${runtimeShell}" exec ${nix}/bin/nix-shell \
-                --pure --run 'source "${innerScript}"' "${drv}"
-            '';
-        }
-        // args.passthru or { };
-
       meta = (args.meta or { }) // {
         inherit platforms;
       };
     };
+
 in
 fnOrAttrs:
 stdenvNoCC.mkDerivation (
   finalAttrs:
   let
-    args = if lib.isFunction fnOrAttrs then fnOrAttrs (args // finalAttrs) else fnOrAttrs;
+    args = if lib.isFunction fnOrAttrs then fnOrAttrs (args' // finalAttrs) else fnOrAttrs;
+    args' = transformArgs finalAttrs args;
+    inherit (args') nugetDeps runtimeId meta;
+    args'' = removeAttrs args' [
+      "nugetDeps"
+      "runtimeId"
+      "installPath"
+      "executables"
+      "projectFile"
+      "projectReferences"
+      "runtimeDeps"
+      "runtimeId"
+      "disabledTests"
+      "testProjectFile"
+      "buildType"
+      "selfContainedBuild"
+      "useDotnet"
+      "useAppHost"
+    ];
   in
-  transformArgs finalAttrs args
+  if nugetDeps != null then
+    addNuGetDeps {
+      inherit nugetDeps;
+      overrideFetchAttrs =
+        a:
+        lib.optionalAttrs ((args'.runtimeId or null) == null) {
+          dotnetRuntimeIds = map (system: dotnetCorePackages.systemToDotnetRid system) meta.platforms;
+        };
+    } args'' finalAttrs
+  else
+    args''
 )