Skip to content

Commit

Permalink
dotnet: infrastructure improvements (#336824)
Browse files Browse the repository at this point in the history
  • Loading branch information
corngood committed Sep 17, 2024
2 parents 658e722 + d8bd63a commit 1b7f8c9
Show file tree
Hide file tree
Showing 37 changed files with 5,741 additions and 2,834 deletions.
2 changes: 1 addition & 1 deletion doc/languages-frameworks/dotnet.section.md
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ buildDotnetGlobalTool {
## Generating and updating NuGet dependencies {#generating-and-updating-nuget-dependencies}

When writing a new expression, you can use the generated `fetch-deps` script to initialise the lockfile.
After creating a blank `deps.nix` and pointing `nugetDeps` to it,
After setting `nugetDeps` to the desired location of the lockfile (e.g. `./deps.nix`),
build the script with `nix-build -A package.fetch-deps` and then run the result.
(When the root attr is your package, it's simply `nix-build -A fetch-deps`.)

Expand Down
42 changes: 23 additions & 19 deletions maintainers/scripts/update-dotnet-lockfiles.nix
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
infrastructure. Regular updates should be done through the individual packages
update scripts.
*/
{ startWith ? null }:
let
pkgs = import ../.. { config.allowAliases = false; };

Expand All @@ -17,28 +18,31 @@ let
packagesWith = cond: pkgs:
let
packagesWithInner = attrs:
lib.unique (
lib.concatLists (
lib.mapAttrsToList (name: elem:
let
result = builtins.tryEval elem;
in
if result.success then
let
value = result.value;
in
if lib.isDerivation value then
lib.optional (cond value) value
else
if lib.isAttrs value && (value.recurseForDerivations or false || value.recurseForRelease or false) then
packagesWithInner value
else []
else []) attrs));
lib.concatLists (
lib.mapAttrsToList (name: elem:
let
result = builtins.tryEval elem;
in
if result.success then
let
value = result.value;
in
if lib.isDerivation value then
lib.optional (cond value) value
else
if lib.isAttrs value && (value.recurseForDerivations or false || value.recurseForRelease or false) then
packagesWithInner value
else []
else []) attrs);
in
packagesWithInner pkgs;

packages =
packagesWith (pkgs: pkgs ? fetch-deps) pkgs;
packages = lib.unique
(lib.filter (p:
(builtins.tryEval p.outPath).success ||
builtins.trace "warning: skipping ${p.name} because it failed to evaluate" false)
((pkgs: (lib.drop (lib.lists.findFirstIndex (p: p.name == startWith) 0 pkgs) pkgs))
(packagesWith (p: p ? fetch-deps) pkgs)));

helpText = ''
Please run:
Expand Down
4 changes: 4 additions & 0 deletions pkgs/applications/audio/openutau/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ buildDotnetModule rec {
dotnet-sdk = dotnetCorePackages.sdk_7_0;
dotnet-runtime = dotnetCorePackages.runtime_7_0;

# [...]/Microsoft.NET.Sdk.targets(157,5): error MSB4018: The "GenerateDepsFile" task failed unexpectedly. [[...]/OpenUtau.Core.csproj]
# [...]/Microsoft.NET.Sdk.targets(157,5): error MSB4018: System.IO.IOException: The process cannot access the file '[...]/OpenUtau.Core.deps.json' because it is being used by another process. [[...]/OpenUtau.Core.csproj]
enableParallelBuilding = false;

projectFile = "OpenUtau.sln";
nugetDeps = ./deps.nix;

Expand Down
108 changes: 108 additions & 0 deletions pkgs/build-support/dotnet/add-nuget-deps/default.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
{
writeShellScript,
runtimeShell,
nix,
lib,
substituteAll,
nuget-to-nix,
cacert,
fetchNupkg,
callPackage,
}:

{
nugetDeps,
overrideFetchAttrs ? x: { },
}:
fnOrAttrs: finalAttrs:
let
attrs = if builtins.isFunction fnOrAttrs then fnOrAttrs finalAttrs else fnOrAttrs;

deps =
if (nugetDeps != null) then
if lib.isDerivation nugetDeps then
[ nugetDeps ]
else if lib.isList nugetDeps then
nugetDeps
else
assert (lib.isPath nugetDeps);
callPackage nugetDeps { fetchNuGet = fetchNupkg; }
else
[ ];

finalPackage = finalAttrs.finalPackage;

in
attrs
// {
buildInputs = attrs.buildInputs or [ ] ++ deps;

passthru =
attrs.passthru or { }
// {
nugetDeps = deps;
}
// lib.optionalAttrs (nugetDeps == null || lib.isPath nugetDeps) rec {
fetch-drv =
let
pkg' = finalPackage.overrideAttrs (old: {
buildInputs = attrs.buildInputs or [ ];
nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [ cacert ];
keepNugetConfig = true;
dontBuild = true;
doCheck = false;
dontInstall = true;
doInstallCheck = false;
dontFixup = true;
doDist = false;
});
in
pkg'.overrideAttrs overrideFetchAttrs;
fetch-deps =
let
drv = builtins.unsafeDiscardOutputDependency fetch-drv.drvPath;

innerScript = substituteAll {
src = ./fetch-deps.sh;
isExecutable = true;
inherit cacert;
nugetToNix = nuget-to-nix;
};

defaultDepsFile =
# Wire in the depsFile such that running the script with no args
# runs it agains the correct deps file by default.
# Note that toString is necessary here as it results in the path at
# eval time (i.e. to the file in your local Nixpkgs checkout) rather
# than the Nix store path of the path after it's been imported.
if lib.isPath nugetDeps && !lib.isStorePath nugetDeps then
toString nugetDeps
else
''$(mktemp -t "${finalAttrs.pname or finalPackage.name}-deps-XXXXXX.nix")'';

in
writeShellScript "${finalPackage.name}-fetch-deps" ''
set -eu
echo 'fetching dependencies for' ${lib.escapeShellArg finalPackage.name} >&2
# this needs to be before TMPDIR is changed, so the output isn't deleted
# if it uses mktemp
depsFile=$(realpath "''${1:-${lib.escapeShellArg defaultDepsFile}}")
export TMPDIR
TMPDIR=$(mktemp -d -t fetch-deps-${lib.escapeShellArg finalPackage.name}.XXXXXX)
trap 'chmod -R +w "$TMPDIR" && rm -fr "$TMPDIR"' EXIT
export NUGET_HTTP_CACHE_PATH=''${NUGET_HTTP_CACHE_PATH-~/.local/share/NuGet/v3-cache}
HOME=$TMPDIR/home
mkdir "$HOME"
cd "$TMPDIR"
NIX_BUILD_SHELL=${lib.escapeShellArg runtimeShell} ${nix}/bin/nix-shell \
--pure --keep NUGET_HTTP_CACHE_PATH --run 'source '${lib.escapeShellArg innerScript}' '"''${depsFile@Q}" "${drv}"
'';
};
}
11 changes: 11 additions & 0 deletions pkgs/build-support/dotnet/add-nuget-deps/fetch-deps.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
set -e

genericBuild

(
echo -e "# This file was automatically generated by passthru.fetch-deps.\n# Please dont edit it manually, your changes might get overwritten!\n"
@nugetToNix@/bin/nuget-to-nix "${NUGET_PACKAGES%/}"
) > deps.nix

mv deps.nix "$1"
echo "Succesfully wrote lockfile to $1"
126 changes: 41 additions & 85 deletions pkgs/build-support/dotnet/build-dotnet-module/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,8 @@
writeShellScript,
makeWrapper,
dotnetCorePackages,
fetchNupkg,
nuget-to-nix,
cacert,
unzip,
yq,
nix,
addNuGetDeps,
}:
let
transformArgs =
Expand Down Expand Up @@ -109,39 +105,9 @@ let
dotnetFixupHook
;

_nugetDeps =
if (nugetDeps != null) then
if lib.isDerivation nugetDeps then
[ nugetDeps ]
else if lib.isList nugetDeps then
nugetDeps
else
assert (lib.isPath nugetDeps);
callPackage nugetDeps { fetchNuGet = fetchNupkg; }
else
[ ];

nugetDepsFile = if lib.isPath nugetDeps then nugetDeps else null;

inherit (dotnetCorePackages) systemToDotnetRid;
in
# Not all args need to be passed through to mkDerivation
# TODO: We should probably filter out even more attrs
removeAttrs args [
"nugetDeps"
"installPath"
"executables"
"projectFile"
"projectReferences"
"runtimeDeps"
"runtimeId"
"disabledTests"
"testProjectFile"
"buildType"
"selfContainedBuild"
"useDotnet"
"useAppHost"
]
args
// {
dotnetInstallPath = installPath;
dotnetExecutables = executables;
Expand All @@ -167,6 +133,8 @@ let
dotnetFlags
packNupkg
useDotnetFromEnv
nugetDeps
runtimeId
;

nativeBuildInputs = args.nativeBuildInputs or [ ] ++ [
Expand All @@ -179,11 +147,14 @@ let
cacert
makeWrapper
dotnet-sdk
unzip
yq
];

buildInputs = args.buildInputs or [ ] ++ [ dotnet-sdk.packages ] ++ _nugetDeps ++ projectReferences;
buildInputs =
args.buildInputs or [ ]
++ [
dotnet-sdk.packages
]
++ projectReferences;

# Parse the version attr into a format acceptable for the Version msbuild property
# The actual version attr is saved in InformationalVersion, which accepts an arbitrary string
Expand Down Expand Up @@ -223,60 +194,45 @@ let
# executables
propagatedSandboxProfile = toString dotnet-runtime.__propagatedSandboxProfile;

passthru =
{
nugetDeps = _nugetDeps;
}
// lib.optionalAttrs (nugetDeps == null || lib.isPath nugetDeps) {
fetch-deps =
let
pkg = finalAttrs.finalPackage.overrideAttrs (
old:
{
buildInputs = lib.subtractLists _nugetDeps old.buildInputs;
keepNugetConfig = true;
}
// lib.optionalAttrs (runtimeId == null) {
dotnetRuntimeIds = map (system: systemToDotnetRid system) platforms;
}
);

drv = builtins.unsafeDiscardOutputDependency pkg.drvPath;

innerScript = substituteAll {
src = ./fetch-deps.sh;
isExecutable = true;
defaultDepsFile =
# Wire in the nugetDeps file such that running the script with no args
# runs it agains the correct deps file by default.
# Note that toString is necessary here as it results in the path at
# eval time (i.e. to the file in your local Nixpkgs checkout) rather
# than the Nix store path of the path after it's been imported.
if lib.isPath nugetDeps && !lib.isStorePath nugetDepsFile then
toString nugetDepsFile
else
''$(mktemp -t "${finalAttrs.pname or finalAttrs.finalPackage.name}-deps-XXXXXX.nix")'';
nugetToNix = (nuget-to-nix.override { inherit dotnet-sdk; });
};

in
writeShellScript "${finalAttrs.finalPackage.name}-fetch-deps" ''
NIX_BUILD_SHELL="${runtimeShell}" exec ${nix}/bin/nix-shell \
--pure --run 'source "${innerScript}"' "${drv}"
'';
}
// args.passthru or { };

meta = (args.meta or { }) // {
inherit platforms;
};
};

in
fnOrAttrs:
stdenvNoCC.mkDerivation (
finalAttrs:
let
args = if lib.isFunction fnOrAttrs then fnOrAttrs (args // finalAttrs) else fnOrAttrs;
args = if lib.isFunction fnOrAttrs then fnOrAttrs (args' // finalAttrs) else fnOrAttrs;
args' = transformArgs finalAttrs args;
inherit (args') nugetDeps runtimeId meta;
args'' = removeAttrs args' [
"nugetDeps"
"runtimeId"
"installPath"
"executables"
"projectFile"
"projectReferences"
"runtimeDeps"
"runtimeId"
"disabledTests"
"testProjectFile"
"buildType"
"selfContainedBuild"
"useDotnet"
"useAppHost"
];
in
transformArgs finalAttrs args
if nugetDeps != null then
addNuGetDeps {
inherit nugetDeps;
overrideFetchAttrs =
a:
lib.optionalAttrs ((args'.runtimeId or null) == null) {
dotnetRuntimeIds = map (system: dotnetCorePackages.systemToDotnetRid system) meta.platforms;
};
} args'' finalAttrs
else
args''
)
24 changes: 0 additions & 24 deletions pkgs/build-support/dotnet/build-dotnet-module/fetch-deps.sh

This file was deleted.

Loading

0 comments on commit 1b7f8c9

Please sign in to comment.