Add support for editable workspace dependencies

Depends on: - nix-community/pyproject.nix#145 - NixOS/nixpkgs#339228
adisbladis · Sep 7, 2024 · 88a91c5 · 88a91c5
1 parent 5b36429
commit 88a91c5
Show file tree

Hide file tree

Showing 8 changed files with 156 additions and 46 deletions.
diff --git a/.github/workflows/nix-github-actions.yml b/.github/workflows/nix-github-actions.yml
@@ -71,7 +71,7 @@ jobs:
           name: adisbladis
           authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}"
       - name: Flake show
-        run: nix flake show --no-update-lock-file --override-input uv2nix .  --reference-lock-file ./flake.lock ./templates/app
+        run: nix flake show --no-update-lock-file --inputs-from . --override-input uv2nix .  --reference-lock-file ./flake.lock ./templates/app
 
   collect:
     runs-on: ubuntu-latest

diff --git a/dev/checks.nix b/dev/checks.nix
@@ -83,7 +83,41 @@ let
     let
       mkCheck = mkCheck' sourcePreference;
     in
-    mapAttrs' (name: v: nameValuePair "${name}-pref-${sourcePreference}" v) {
+    {
+      editable-workspace =
+        let
+          workspaceRoot = ../lib/fixtures/workspace;
+          ws = uv2nix.workspace.loadWorkspace { inherit workspaceRoot; };
+          overlay = ws.mkOverlay {
+            sourcePreference = "wheel"; # We're testing editable support, wheels are fine.
+          };
+          editableOverlay = ws.mkEditableOverlay {
+            root = "$NIX_BUILD_TOP";
+          };
+
+          python = pkgs.python312.override {
+            self = python;
+            packageOverrides = lib.composeExtensions overlay editableOverlay;
+          };
+
+          pythonEnv = python.withPackages (ps: [ ps.workspace ]);
+
+        in
+        pkgs.runCommand "editable-workspace-test"
+          {
+            nativeBuildInputs = [ pythonEnv ];
+          }
+          ''
+            cp -r ${workspaceRoot}/* .
+            chmod +w .*
+            test "$(python -c 'import workspace_package; print(workspace_package.hello())')" = "Hello from workspace-package!"
+            substituteInPlace ./packages/workspace-package/src/workspace_package/__init__.py --replace-fail workspace-package mutable-package
+            test "$(python -c 'import workspace_package; print(workspace_package.hello())')" = "Hello from mutable-package!"
+            touch $out
+          '';
+
+    }
+    // mapAttrs' (name: v: nameValuePair "${name}-pref-${sourcePreference}" v) {
       trivial = mkCheck {
         root = ../lib/fixtures/trivial;
         packages = ps: [ ps."trivial" ];

diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -2,7 +2,7 @@
   description = "Uv2nix";
 
   inputs = {
-    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+    nixpkgs.url = "github:adisbladis/nixpkgs/python-editable";
 
     flake-parts.url = "github:hercules-ci/flake-parts";
     flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
@@ -16,7 +16,7 @@
     nixdoc.url = "github:nix-community/nixdoc";
     nixdoc.inputs.nixpkgs.follows = "nixpkgs";
 
-    pyproject-nix.url = "github:adisbladis/pyproject.nix";
+    pyproject-nix.url = "github:nix-community/pyproject.nix/editable";
     pyproject-nix.inputs.nixpkgs.follows = "nixpkgs";
     pyproject-nix.inputs.nix-github-actions.follows = "nix-github-actions";
     pyproject-nix.inputs.mdbook-nixdoc.follows = "mdbook-nixdoc";

diff --git a/lib/lock1.nix b/lib/lock1.nix
@@ -35,6 +35,7 @@ let
     hasPrefix
     intersectLists
     assertMsg
+    isString
     ;
 
 in
@@ -167,6 +168,13 @@ fix (self: {
     in
     reduceDependencies allDependencies;
 
+  /*
+    Returns whether a package is a locally developed package
+    .
+  */
+  isLocalPackage =
+    package: package.source ? editable || package.source ? directory || package.source ? virtual;
+
   /*
     Filter dependencies/optional-dependencies/dev-dependencies from a uv.lock package entry
     .
@@ -242,12 +250,9 @@ fix (self: {
     let
       inherit (package) source;
       isGit = source ? git;
-      isProject = source ? editable;
       isPypi = source ? registry; # From pypi registry
-      isURL = source ? url;
-      isDirectory = source ? directory; # Path to non-uv project
+      isURL = source ? url; # Path to non-uv project
       isPath = source ? path; # Path to sdist
-      isVirtual = source ? virtual;
 
       # Wheels grouped by filename
       wheels = mapAttrs (
@@ -258,20 +263,17 @@ fix (self: {
       # List of parsed wheels
       wheelFiles = map (whl: whl.file') package.wheels;
 
+      # Local projects
+      localPath =
+        "/"
+        + source.editable or source.directory or source.virtual
+          or (throw "Not a project path: ${toJSON source}");
       localProject =
         if projects ? package.name then
           projects.${package.name}
         else
           pyproject-nix.lib.project.loadUVPyproject {
-            projectRoot =
-              if isProject then
-                workspaceRoot + "/${source.editable}"
-              else if isDirectory then
-                workspaceRoot + "/${source.directory}"
-              else if isVirtual then
-                workspaceRoot + "/${source.virtual}"
-              else
-                throw "Not a project path: ${toJSON source}";
+            projectRoot = workspaceRoot + localPath;
           };
 
     in
@@ -286,7 +288,11 @@ fix (self: {
       stdenv,
       autoPatchelfHook,
       pythonManylinuxPackages,
+      mkPythonEditablePackage,
+      # Source preference (sdist or wheel)
       sourcePreference ? wsargs.sourcePreference,
+      # Editable root as a string
+      editableRoot ? null,
     }:
     let
       preferWheel =
@@ -353,8 +359,24 @@ fix (self: {
       "Package source for '${package.name}' was derived as sdist, but was present in tool.uv.no-build-package";
     assert assertMsg (format == "wheel" -> !elem package.name no-binary-package)
       "Package source for '${package.name}' was derived as wheel, but was present in tool.uv.no-binary-package";
-    if (isProject || isDirectory || isVirtual) then
-      buildPythonPackage (localProject.renderers.buildPythonPackage { inherit python environ; })
+    if (self.isLocalPackage package) then
+      # Package is not editable, render buildPythonPackage
+      if editableRoot == null then
+        buildPythonPackage (localProject.renderers.buildPythonPackage { inherit python environ; })
+      # Package is editable, render mkPythonEditablePackage
+      else
+        assert isString editableRoot;
+        mkPythonEditablePackage (
+          localProject.renderers.mkPythonEditablePackage {
+            inherit python environ;
+            # Prefer src style layout if available, otherwise use project root as editable root.
+            root =
+              if lib.pathExists (workspaceRoot + localPath + "/src") then
+                editableRoot + localPath + "/src"
+              else
+                editableRoot + localPath;
+          }
+        )
     else
       buildPythonPackage (
         {

diff --git a/lib/test_lock1.nix b/lib/test_lock1.nix
@@ -80,6 +80,17 @@ in
     };
   };
 
+  isLocalPackage = {
+    testLocal = {
+      expr = lock1.isLocalPackage { source.directory = "foo"; };
+      expected = true;
+    };
+    testNonLocal = {
+      expr = lock1.isLocalPackage { source.url = "http://foo"; };
+      expected = false;
+    };
+  };
+
   mkPackage =
     let
 

diff --git a/lib/workspace.nix b/lib/workspace.nix
@@ -31,6 +31,7 @@ let
     isAttrs
     attrValues
     assertMsg
+    nameValuePair
     ;
   inherit (builtins) readDir;
   inherit (pyproject-nix.lib.project) loadUVPyproject; # Note: Maybe we want a loader that will just "remap-all-the-things" into standard attributes?
@@ -178,6 +179,27 @@ fix (self: {
         assert all (spec: pep440.comparators.${spec.op} pythonVersion spec.version) uvLock.requires-python;
         mapAttrs (_: pkg: callPackage (mkPackage pkg) { }) resolved;
 
+      mkEditableOverlay =
+        let
+          workspaceProjects' = attrNames workspaceProjects;
+          localProjects = map (package: package.name) (filter lock1.isLocalPackage uvLock.package);
+          allLocal = unique (workspaceProjects' ++ localProjects);
+        in
+        {
+          # Editable root as a string.
+          root ? (toString workspaceRoot),
+          # Workspace members to make editable as a list of strings. Defaults to all local projects.
+          members ? allLocal,
+        }:
+        _final: prev:
+        let
+          # Filter any local packages that might be deactivated by markers or other filtration mechanisms.
+          activeMembers = filter (name: !prev ? name) members;
+        in
+        lib.listToAttrs (
+          map (name: nameValuePair name (prev.${name}.override { editableRoot = root; })) activeMembers
+        );
+
       inherit topLevelDependencies;
     };
 

diff --git a/templates/app/flake.nix b/templates/app/flake.nix
@@ -20,7 +20,6 @@
       overlay =
         let
           # Create overlay from workspace.
-
           overlay' = workspace.mkOverlay {
             # Prefer prebuilt binary wheels as a package source.
             # Sdists are less likely to "just work" because of the metadata missing from uv.lock.
@@ -46,25 +45,46 @@
       # This example is only using x86_64-linux
       pkgs = nixpkgs.legacyPackages.x86_64-linux;
 
-      # Create an overriden interpreter
-      python = pkgs.python3.override {
-        # Note the self argument.
-        # It's important so the interpreter/set is internally consistent.
-        self = python;
-        # Pass composed Python overlay to the interpreter
-        packageOverrides = overlay;
-      };
-
     in
     {
       # 'app' is the name in pyproject.toml after name normalization.
       # See https://packaging.python.org/en/latest/specifications/name-normalization/#normalization
 
-      packages.x86_64-linux.default = python.pkgs.app;
-      # TODO: A better mkShell withPackages example.
-      devShells.x86_64-linux.default = pkgs.mkShell {
-        inputsFrom = [ python.pkgs.app ];
-        packages = [ pkgs.uv ];
-      };
+      packages.x86_64-linux.default =
+        let
+          # Create an overriden interpreter
+          python = pkgs.python3.override {
+            # Note the self argument.
+            # It's important so the interpreter/set is internally consistent.
+            self = python;
+            # Pass composed Python overlay to the interpreter
+            packageOverrides = overlay;
+          };
+        in
+        python.pkgs.app;
+
+      devShells.x86_64-linux.default =
+        let
+          # Create an overriden interpreter with editable support enable for local development
+          python = pkgs.python3.override {
+            self = python;
+            # Compose an editable overlay with the initial overlay.
+            # This enables editable mode installs for all local packages.
+            packageOverrides = lib.composeExtensions overlay (
+              workspace.mkEditableOverlay {
+                # Needs to be a string, see nixpkgs documentation for mkPythonEditablePackage for more information.
+                root = "$REPO_ROOT";
+              }
+            );
+          };
+
+          pythonEnv = python.withPackages (ps: [ ps.app ]);
+        in
+        pkgs.mkShell {
+          packages = [
+            pkgs.uv
+            pythonEnv
+          ];
+        };
     };
 }