python3Packages.mkPythonEditablePackage: init (#339228)

doc/languages-frameworks/python.section.md

@@ -374,6 +374,50 @@ mkPythonMetaPackage {
 }
 ```
 
+#### `mkPythonEditablePackage` function {#mkpythoneditablepackage-function}
+
+When developing Python packages it's common to install packages in [editable mode](https://setuptools.pypa.io/en/latest/userguide/development_mode.html).
+Like `mkPythonMetaPackage` this function exists to create an otherwise empty package, but also containing a pointer to an impure location outside the Nix store that can be changed without rebuilding.
+
+The editable root is passed as a string. Normally `.pth` files contains absolute paths to the mutable location. This isn't always ergonomic with Nix, so environment variables are expanded at runtime.
+This means that a shell hook setting up something like a `$REPO_ROOT` variable can be used as the relative package root.
+
+As an implementation detail, the [PEP-518](https://peps.python.org/pep-0518/) `build-system` specified won't be used, but instead the editable package will be built using [hatchling](https://pypi.org/project/hatchling/).
+The `build-system`'s provided will instead become runtime dependencies of the editable package.
+
+Note that overriding packages deeper in the dependency graph _can_ work, but it's not the primary use case and overriding existing packages can make others break in unexpected ways.
+
+``` nix
+{ pkgs ? import <nixpkgs> { } }:
+
+let
+  pyproject = pkgs.lib.importTOML ./pyproject.toml;
+
+  myPython = pkgs.python.override {
+    self = myPython;
+    packageOverrides = pyfinal: pyprev: {
+      # An editable package with a script that loads our mutable location
+      my-editable = pyfinal.mkPythonEditablePackage {
+        # Inherit project metadata from pyproject.toml
+        pname = pyproject.project.name;
+        inherit (pyproject.project) version;
+
+        # The editable root passed as a string
+        root = "$REPO_ROOT/src"; # Use environment variable expansion at runtime
+
+        # Inject a script (other PEP-621 entrypoints are also accepted)
+        inherit (pyproject.project) scripts;
+      };
+    };
+  };
+
+  pythonEnv =  testPython.withPackages (ps: [ ps.my-editable ]);
+
+in pkgs.mkShell {
+  packages = [ pythonEnv ];
+}
+```
+
 #### `python.buildEnv` function {#python.buildenv-function}
 
 Python environments can be created using the low-level `pkgs.buildEnv` function.

pkgs/development/interpreters/python/editable.nix

@@ -0,0 +1,99 @@
+{
+  buildPythonPackage,
+  lib,
+  hatchling,
+  tomli-w,
+}:
+{
+  pname,
+  version,
+
+  # Editable root as string.
+  # Environment variables will be expanded at runtime using os.path.expandvars.
+  root,
+
+  # Arguments passed on verbatim to buildPythonPackage
+  derivationArgs ? { },
+
+  # Python dependencies
+  dependencies ? [ ],
+  optional-dependencies ? { },
+
+  # PEP-518 build-system https://peps.python.org/pep-518
+  build-system ? [ ],
+
+  # PEP-621 entry points https://peps.python.org/pep-0621/#entry-points
+  scripts ? { },
+  gui-scripts ? { },
+  entry-points ? { },
+
+  passthru ? { },
+  meta ? { },
+}:
+
+# Create a PEP-660 (https://peps.python.org/pep-0660/) editable package pointing to an impure location outside the Nix store.
+# The primary use case of this function is to enable local development workflows where the local package is installed into a virtualenv-like environment using withPackages.
+
+assert lib.isString root;
+let
+  # In editable mode build-system's are considered to be runtime dependencies.
+  dependencies' = dependencies ++ build-system;
+
+  pyproject = {
+    # PEP-621 project table
+    project = {
+      name = pname;
+      inherit
+        version
+        scripts
+        gui-scripts
+        entry-points
+        ;
+      dependencies = map lib.getName dependencies';
+      optional-dependencies = lib.mapAttrs (_: lib.getName) optional-dependencies;
+    };
+
+    # Allow empty package
+    tool.hatch.build.targets.wheel.bypass-selection = true;
+
+    # Include our editable pointer file in build
+    tool.hatch.build.targets.wheel.force-include."_${pname}.pth" = "_${pname}.pth";
+
+    # Build editable package using hatchling
+    build-system = {
+      requires = [ "hatchling" ];
+      build-backend = "hatchling.build";
+    };
+  };
+
+in
+buildPythonPackage (
+  {
+    inherit
+      pname
+      version
+      optional-dependencies
+      passthru
+      meta
+      ;
+    dependencies = dependencies';
+
+    pyproject = true;
+
+    unpackPhase = ''
+      python -c "import json, tomli_w; print(tomli_w.dumps(json.load(open('$pyprojectContentsPath'))))" > pyproject.toml
+      echo 'import os.path, sys; sys.path.insert(0, os.path.expandvars("${root}"))' > _${pname}.pth
+    '';
+
+    build-system = [ hatchling ];
+  }
+  // derivationArgs
+  // {
+    # Note: Using formats.toml generates another intermediary derivation that needs to be built.
+    # We inline the same functionality for better UX.
+    nativeBuildInputs = (derivationArgs.nativeBuildInputs or [ ]) ++ [ tomli-w ];
+    pyprojectContents = builtins.toJSON pyproject;
+    passAsFile = [ "pyprojectContents" ];
+    preferLocalBuild = true;
+  }
+)

pkgs/development/interpreters/python/python-packages-base.nix

@@ -61,6 +61,8 @@ let
 
   removePythonPrefix = lib.removePrefix namePrefix;
 
+  mkPythonEditablePackage = callPackage ./editable.nix { };
+
   mkPythonMetaPackage = callPackage ./meta-package.nix { };
 
   # Convert derivation to a Python module.
@@ -99,7 +101,7 @@ in {
   inherit buildPythonPackage buildPythonApplication;
   inherit hasPythonModule requiredPythonModules makePythonPath disabled disabledIf;
   inherit toPythonModule toPythonApplication;
-  inherit mkPythonMetaPackage;
+  inherit mkPythonMetaPackage mkPythonEditablePackage;
 
   python = toPythonModule python;
 

pkgs/development/interpreters/python/tests.nix

@@ -122,6 +122,43 @@ let
     }
   );
 
+  # Test editable package support
+  editableTests = let
+    testPython = python.override {
+      self = testPython;
+      packageOverrides = pyfinal: pyprev: {
+        # An editable package with a script that loads our mutable location
+        my-editable = pyfinal.mkPythonEditablePackage {
+          pname = "my-editable";
+          version = "0.1.0";
+          root = "$NIX_BUILD_TOP/src"; # Use environment variable expansion at runtime
+          # Inject a script
+          scripts = {
+            my-script = "my_editable.main:main";
+          };
+        };
+      };
+    };
+
+
+  in {
+    editable-script = runCommand "editable-test" {
+      nativeBuildInputs = [ (testPython.withPackages (ps: [ ps.my-editable ])) ];
+    } ''
+      mkdir -p src/my_editable
+
+      cat > src/my_editable/main.py << EOF
+      def main():
+        print("hello mutable")
+      EOF
+
+      test "$(my-script)" == "hello mutable"
+      test "$(python -c 'import sys; print(sys.path[1])')" == "$NIX_BUILD_TOP/src"
+
+      touch $out
+    '';
+  };
+
   # Tests to ensure overriding works as expected.
   overrideTests = let
     extension = self: super: {
@@ -192,4 +229,4 @@ let
       '';
     };
 
-in lib.optionalAttrs (stdenv.hostPlatform == stdenv.buildPlatform ) (environmentTests // integrationTests // overrideTests // condaTests)
+in lib.optionalAttrs (stdenv.hostPlatform == stdenv.buildPlatform ) (environmentTests // integrationTests // overrideTests // condaTests // editableTests)