Add CUDA 13 support

Load the appropriate graphics modules and firmware and make
nvidia-jetpack.cudaPackages_13 an alias to the upstream cudaPackages_13.
Convergence is real!

Co-Authored-By: Connor Baker <[email protected]>

Author: elliotberman

mk-overlay.nix

@@ -44,7 +44,7 @@ let
 in
 makeScope final.newScope (self: {
   inherit (sourceInfo) debs gitRepos;
-  inherit jetpackMajorMinorPatchVersion l4tMajorMinorPatchVersion cudaMajorMinorVersion;
+  inherit jetpackMajorMinorPatchVersion l4tMajorMinorPatchVersion cudaMajorMinorVersion cudaDriverMajorMinorVersion;
   inherit l4tAtLeast l4tOlder;
 
   callPackages = callPackagesWith (final // self);
@@ -230,10 +230,13 @@ makeScope final.newScope (self: {
       ]
       ++ _cuda.extensions);
     in
+    if final.lib.versionAtLeast cudaMajorMinorPatchVersion "13" then
+      final."cudaPackages_${cudaLib.dotsToUnderscores cudaMajorMinorVersion}"
+    else
     # NOTE: We must ensure the scope allows us to draw on the contents of nvidia-jetpack.
-    makeScope pkgs'.nvidia-jetpack.newScope (
-      extends composedExtensions passthruFunction
-    );
+      makeScope pkgs'.nvidia-jetpack.newScope (
+        extends composedExtensions passthruFunction
+      );
 
   samples = makeScope self.newScope (finalSamples: {
     callPackages = callPackagesWith (self // finalSamples);

modules/default.nix

@@ -36,6 +36,9 @@ let
   };
 
   jetpackAtLeast = lib.versionAtLeast cfg.majorVersion;
+
+  getDriverDebs = prefix: (lib.filter (drv: lib.hasPrefix prefix (drv.pname or "")) (lib.attrValues pkgs.nvidia-jetpack.driverDebs));
+  nvidiaDriverFirmwareDebs = getDriverDebs "nvidia-firmware-";
 in
 {
   imports = [
@@ -226,8 +229,8 @@ in
               message = "JetPack NixOS 6 supports CUDA 12.4 (natively) - 12.9 (with `cuda_compat`): `pkgs.cudaPackages` has version ${cudaMajorMinorVersion}.";
             }
             {
-              assertion = !cudaAtLeast "13.0";
-              message = "JetPack NixOS does not support CUDA 13.0 or later: `pkgs.cudaPackages` has version ${cudaMajorMinorVersion}.";
+              assertion = cfg.majorVersion == "7" -> cudaAtLeast "13.0";
+              message = "JetPack NixOS 7 suppoers CUDA 13.0 (natively): `pkgs.cudaPackages` has version ${cudaMajorMinorVersion}.";
             }
           ]
         )
@@ -274,8 +277,11 @@ in
             isGeneric = cfg.som == "generic";
             isXavier = lib.hasPrefix "xavier-" cfg.som;
             isOrin = lib.hasPrefix "orin-" cfg.som;
+            isThor = lib.hasPrefix "thor-" cfg.som;
           in
-          lib.optionals (isXavier || isGeneric) [ "7.2" ] ++ lib.optionals (isOrin || isGeneric) [ "8.7" ];
+          lib.optionals (isXavier || isGeneric) [ "7.2" ] ++
+          lib.optionals (isOrin || isGeneric) [ "8.7" ] ++
+          lib.optionals isThor [ "11.0" ];
       });
 
       hardware.nvidia-jetpack.console.args = lib.mkMerge [
@@ -356,7 +362,7 @@ in
       ];
 
       boot.kernelModules =
-        [ "nvgpu" ]
+        (if (jetpackAtLeast "7") then [ "nvidia-uvm" ] else [ "nvgpu" ])
         ++ lib.optionals (cfg.modesetting.enable && checkValidSoms [ "xavier" ]) [ "tegra-udrm" ]
         ++ lib.optionals (cfg.modesetting.enable && checkValidSoms [ "orin" ]) [ "nvidia-drm" ];
 
@@ -365,6 +371,10 @@ in
       '' + lib.optionalString cfg.modesetting.enable ''
         options tegra-udrm modeset=1
         options nvidia-drm modeset=1 ${lib.optionalString (cfg.majorVersion == "6") "fbdev=1"}
+      '' + lib.optionalString (jetpackAtLeast "7") ''
+        # from L4T-Ubuntu /etc/modprobe.d/nvidia-unifiedgpudisp.conf
+        options nvidia NVreg_RegistryDwords="RMExecuteDevinitOnPmu=0;RMEnableAcr=1;RmCePceMap=0xffffff20;RmCePceMap1=0xffffffff;RmCePceMap2=0xffffffff;RmCePceMap3=0xffffffff;" NVreg_TegraGpuPgMask=512
+        softdep nvidia pre: governor_pod_scaling post: nvidia-uvm
       '';
 
       boot.extraModulePackages =
@@ -382,7 +392,9 @@ in
         vpi-firmware # Optional, but needed for pva_auth_allowlist firmware file used by VPI2
       ] ++ lib.optionals (l4tOlder "36") [
         l4t-xusb-firmware # usb firmware also present in linux-firmware package, but that package is huge and has much more than needed
-      ];
+      ] ++ lib.optionals (l4tAtLeast "38") (nvidiaDriverFirmwareDebs ++ [ l4t-firmware-openrm ]);
+
+      boot.blacklistedKernelModules = [ "nouveau" ];
 
       hardware.deviceTree.enable = true;
       hardware.deviceTree.dtboBuildExtraIncludePaths = {

overlay.nix

@@ -52,7 +52,7 @@ in
     {
       jetpackMajorMinorPatchVersion = "7.0";
       l4tMajorMinorPatchVersion = "38.2.1";
-      cudaMajorMinorPatchVersion = "12.6.10"; #TODO
+      cudaMajorMinorPatchVersion = "13.0.2";
 
       cudaDriverMajorMinorVersion = "580.00";
 
@@ -102,7 +102,9 @@ in
     if (redistSystem final.nvidia-jetpack5.cudaMajorMinorVersion) == "linux-aarch64" then
       final.cudaPackages_11_4
     else
-      prev.cudaPackages_11;
+    # note: cudaPackages_11 removed upstream cuda-legacy re-introduces package sets with major and minor versions but does not change or create aliases for major versions
+    # default to cudaPackages_11 if it exists or 11.8 as that was the default
+      prev.cudaPackages_11 or prev.cudaPackages_11_8;
 
   cudaPackages_12_6 =
     if (redistSystem final.nvidia-jetpack6.cudaMajorMinorVersion) == "linux-aarch64" then

pkgs/l4t/default.nix

@@ -8,6 +8,7 @@
 , autoPatchelfHook
 , dpkg
 , expat
+, libgcc
 , libglvnd
 , egl-wayland
 , xorg
@@ -396,7 +397,12 @@ let
 
   l4t-nvsci = buildFromDeb {
     name = "nvidia-l4t-nvsci";
-    buildInputs = [ l4t-core ];
+    buildInputs = [ l4t-core ] ++ lib.optionals (l4tAtLeast "38") [ l4t-adaruntime ];
+  };
+
+  l4t-adaruntime = buildFromDeb {
+    name = "nvidia-l4t-adaruntime";
+    buildInputs = [ libgcc ];
   };
 
   # Programmable Vision Accelerator
@@ -437,7 +443,7 @@ let
 
   l4t-wayland = buildFromDeb {
     name = "nvidia-l4t-wayland";
-    buildInputs = [ wayland ];
+    buildInputs = [ wayland ] ++ lib.optionals (l4tAtLeast "38") [ l4t-core ];
     postPatch = ''
       sed -i -E "s#(libnvidia-egl-wayland)#$out/lib/\\1#" share/egl/egl_external_platform.d/nvidia_wayland.json
     '';
@@ -449,15 +455,41 @@ let
     meta.platforms = [ "aarch64-linux" "x86_64-linux" ];
   };
 
+  l4t-firmware-openrm = buildFromDeb {
+    name = "nvidia-l4t-firmware-openrm";
+    autoPatchelf = false;
+    meta.platforms = [ "aarch64-linux" "x86_64-linux" ];
+  };
+
+  l4t-nvml = buildFromDeb {
+    name = "nvidia-l4t-nvml";
+    buildInputs = [ l4t-core ];
+  };
+
   nvidia-smi = buildFromDeb {
     name = "nvidia-smi";
     src = debs.${defaultSomRepo}.nvidia-l4t-nvml.src;
     version = debs.${defaultSomRepo}.nvidia-l4t-nvml.version;
     buildInputs = [ l4t-core ];
     # nvidia-smi will dlopen libnvidia-ml.so.1
-    appendRunpaths = [ "${placeholder "out"}/lib" ];
+    appendRunpaths = [ "${placeholder "out"}/lib" "/run/opengl-driver/lib" ];
   };
 
+  findDriverDebNames = majorVersion: lib.filter (lib.hasSuffix "-${majorVersion}") (lib.attrNames debs.common);
+  driverDebNames = findDriverDebNames (lib.versions.major cudaDriverMajorMinorVersion);
+  driverDebs = lib.genAttrs driverDebNames (name: buildFromDeb {
+    inherit name;
+    repo = "common";
+
+    buildInputs = lib.attrByPath [ name ] [ ] {
+      libnvidia-compute-580 = [ driverDebs.libnvidia-decode-580 libgcc ];
+      libnvidia-decode-580 = [ xorg.libX11 xorg.libXext ];
+      libnvidia-encode-580 = [ driverDebs.libnvidia-decode-580 ];
+      libnvidia-fbc1-580 = [ xorg.libX11 xorg.libXext ];
+      libnvidia-gl-580 = [ xorg.libX11 xorg.libXext libgcc driverDebs.libnvidia-gpucomp-580 ];
+    };
+  });
+
   vpiMajorVersion = {
     "35" = "2";
     "36" = "3";
@@ -494,9 +526,10 @@ in
   inherit
     l4t-dla-compiler
     nvidia-smi
+    l4t-nvml
     ;
 } // lib.optionalAttrs (l4tOlder "38") {
   inherit l4t-xusb-firmware; # L4T 38+ uses upstream firmware
 } // lib.optionalAttrs (l4tAtLeast "38") {
-  inherit l4t-bootloader-utils;
+  inherit l4t-bootloader-utils l4t-adaruntime driverDebs l4t-firmware-openrm;
 }