From bcb9c36df79129779c5b7c17b52683408681c6b8 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Fri, 11 Nov 2022 13:04:26 +0000 Subject: [PATCH 1/2] trying to get matplotlib working --- kernels/available/python/default.nix | 2 +- kernels/available/python/overrides.nix | 29 +++++++++++++++++++++++++- kernels/example_python.nix | 1 + kernels/example_stable_python.nix | 1 + 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/kernels/available/python/default.nix b/kernels/available/python/default.nix index 39e8e683..a755c70d 100644 --- a/kernels/available/python/default.nix +++ b/kernels/available/python/default.nix @@ -7,7 +7,7 @@ projectDir ? self + "/kernels/available/python", pyproject ? projectDir + "/pyproject.toml", poetrylock ? projectDir + "/poetry.lock", - overrides ? pkgs.poetry2nix.overrides.withDefaults (import ./overrides.nix), + overrides ? import ./overrides.nix pkgs, python ? pkgs.python3, editablePackageSources ? {}, extraPackages ? ps: [], diff --git a/kernels/available/python/overrides.nix b/kernels/available/python/overrides.nix index 0e2fe68a..dd46949a 100644 --- a/kernels/available/python/overrides.nix +++ b/kernels/available/python/overrides.nix @@ -1 +1,28 @@ -final: prev: {} +pkgs: let + addNativeBuildInputs = prev: drvName: inputs: { + "${drvName}" = prev.${drvName}.overridePythonAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or []) ++ inputs; + }); + }; + + # A fix is on the way soon, https://github.com/nix-community/poetry2nix/pull/787 + preOverlay = final: prev: { + babel = null; + Babel = null; + babel_ = prev.babel.overridePythonAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or []) ++ [final.setuptools]; + }); + pillow_ = prev.pillow; + matplotlib_ = prev.matplotlib; + }; + + postOverlay = final: prev: + {} + // { + babel = prev.babel_; + Babel = prev.babel_; + pillow = prev.pillow_; + matplotlib = prev.matplotlib_; + }; +in + (pkgs.poetry2nix.defaultPoetryOverrides.overrideOverlay preOverlay).extend postOverlay diff --git a/kernels/example_python.nix b/kernels/example_python.nix index 6af95eb9..50e0ccc2 100644 --- a/kernels/example_python.nix +++ b/kernels/example_python.nix @@ -7,4 +7,5 @@ availableKernels.python { inherit name; inherit (extraArgs) pkgs; displayName = "Example Python Kernel"; + extraPackages = ps: with ps; [requests numpy matplotlib]; } diff --git a/kernels/example_stable_python.nix b/kernels/example_stable_python.nix index 0976a10a..439547aa 100644 --- a/kernels/example_stable_python.nix +++ b/kernels/example_stable_python.nix @@ -7,4 +7,5 @@ availableKernels.python { inherit name; pkgs = extraArgs.pkgs_stable; displayName = "Example (nixpkgs stable) Python Kernel"; + extraPackages = ps: with ps; [requests numpy matplotlib]; } From e2063752fa81a4b54fe561691c1024a372d764b4 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Tue, 15 Nov 2022 11:05:54 +0000 Subject: [PATCH 2/2] Another idea how this could work --- flake.nix | 65 ++++++++++++++++++++- kernels/available/python/overrides.nix | 44 +++++--------- overrides.nix | 81 +++++++++++++++----------- 3 files changed, 127 insertions(+), 63 deletions(-) diff --git a/flake.nix b/flake.nix index 5cdaf5ec..97b92f40 100644 --- a/flake.nix +++ b/flake.nix @@ -44,6 +44,68 @@ # flake-utils.lib.system.x86_64-darwin ]; + poetryBuildSystems = final: prev: + lib.mapAttrs + (attr: systems: + builtins.foldl' + (drv: attr: + addBuildSystem { + inherit drv final attr; + }) + prev.${attr} + systems) + (lib.importJSON "${poetry2nix}/overrides/build-systems.json"); + + addBuildSystem = { + final, + drv, + attr, + extraAttrs ? [], + }: let + buildSystem = + if builtins.isAttrs attr + then let + fromIsValid = + if builtins.hasAttr "from" attr + then lib.versionAtLeast drv.version attr.from + else true; + untilIsValid = + if builtins.hasAttr "until" attr + then lib.versionOlder drv.version attr.until + else true; + intendedBuildSystem = + if attr.buildSystem == "cython" + then final.python.pythonForBuild.cython + else final.${attr.buildSystem}; + in + if fromIsValid && untilIsValid + then intendedBuildSystem + else null + else if attr == "cython" + then final.python.pythonForBuild.pkgs.cython + else final.${attr}; + in ( + # Flit only works on Python3 + if (attr == "flit-core" || attr == "flit" || attr == "hatchling") && !final.isPy3k + then drv + else if drv == null + then null + else if drv ? overridePythonAttrs == false + then drv + else + drv.overridePythonAttrs ( + old: + # We do not need the build system for wheels. + if old ? format && old.format == "wheel" + then {} + else { + nativeBuildInputs = + (old.nativeBuildInputs or []) + ++ lib.optionals (!(builtins.isNull buildSystem)) [buildSystem] + ++ map (a: final.${a}) extraAttrs; + } + ) + ); /* Takes a path to the kernels directory, `kernelsPath`, a kernel name, `fileName`, @@ -166,8 +228,9 @@ overlays = [ poetry2nix.overlay rust-overlay.overlays.default - (self: super: { + (final: prev: { npmlock2nix = pkgs.callPackage npmlock2nix {}; + inherit poetryBuildSystems; }) ]; diff --git a/kernels/available/python/overrides.nix b/kernels/available/python/overrides.nix index dd46949a..ecf2fed7 100644 --- a/kernels/available/python/overrides.nix +++ b/kernels/available/python/overrides.nix @@ -1,28 +1,16 @@ -pkgs: let - addNativeBuildInputs = prev: drvName: inputs: { - "${drvName}" = prev.${drvName}.overridePythonAttrs (old: { - nativeBuildInputs = (old.nativeBuildInputs or []) ++ inputs; - }); - }; - - # A fix is on the way soon, https://github.com/nix-community/poetry2nix/pull/787 - preOverlay = final: prev: { - babel = null; - Babel = null; - babel_ = prev.babel.overridePythonAttrs (old: { - nativeBuildInputs = (old.nativeBuildInputs or []) ++ [final.setuptools]; - }); - pillow_ = prev.pillow; - matplotlib_ = prev.matplotlib; - }; - - postOverlay = final: prev: - {} - // { - babel = prev.babel_; - Babel = prev.babel_; - pillow = prev.pillow_; - matplotlib = prev.matplotlib_; - }; -in - (pkgs.poetry2nix.defaultPoetryOverrides.overrideOverlay preOverlay).extend postOverlay +pkgs: final: prev: +(pkgs.poetryBuildSystems final prev) +// { + cffi = prev.cffi.overridePythonAttrs (old: { + nativeBuildInputs = old.nativeBuildInputs or [] ++ [pkgs.pkg-config final.setuptools]; + buildInputs = old.buildInputs or [] ++ [pkgs.libffi]; + prePatch = + (old.prePatch or "") + + pkgs.lib.optionalString pkgs.stdenv.isDarwin '' + # Remove setup.py impurities + substituteInPlace setup.py --replace "'-iwithsysroot/usr/include/ffi'" "" + substituteInPlace setup.py --replace "'/usr/include/ffi'," "" + substituteInPlace setup.py --replace '/usr/include/libffi' '${pkgs.lib.getDev pkgs.libffi}/include' + ''; + }); +} diff --git a/overrides.nix b/overrides.nix index 08138980..30801fbd 100644 --- a/overrides.nix +++ b/overrides.nix @@ -1,36 +1,49 @@ -pkgs: let - addNativeBuildInputs = prev: drvName: inputs: { - "${drvName}" = prev.${drvName}.overridePythonAttrs (old: { - nativeBuildInputs = (old.nativeBuildInputs or []) ++ inputs; - }); - }; +pkgs: final: prev: +(pkgs.poetryBuildSystems final prev) +// { + cffi = prev.cffi.overridePythonAttrs (old: { + nativeBuildInputs = old.nativeBuildInputs or [] ++ [pkgs.pkg-config final.setuptools]; + buildInputs = old.buildInputs or [] ++ [pkgs.libffi]; + prePatch = + (old.prePatch or "") + + pkgs.lib.optionalString pkgs.stdenv.isDarwin '' + # Remove setup.py impurities + substituteInPlace setup.py --replace "'-iwithsysroot/usr/include/ffi'" "" + substituteInPlace setup.py --replace "'/usr/include/ffi'," "" + substituteInPlace setup.py --replace '/usr/include/libffi' '${pkgs.lib.getDev pkgs.libffi}/include' + ''; + }); - # A fix is on the way soon, https://github.com/nix-community/poetry2nix/pull/787 - preOverlay = final: prev: { - babel = null; - Babel = null; - babel_ = prev.babel.overridePythonAttrs (old: { - nativeBuildInputs = (old.nativeBuildInputs or []) ++ [final.setuptools]; - }); - }; + nbconvert = prev.nbconvert.overridePythonAttrs (old: { + nativeBuildInputs = old.nativeBuildInputs or [] ++ [final.hatchling]; + postPatch = + pkgs.lib.optionalString (pkgs.lib.versionAtLeast final.nbconvert.version "6.5.0") '' + substituteInPlace \ + ./nbconvert/exporters/templateexporter.py \ + --replace \ + 'root_dirs.extend(jupyter_path())' \ + 'root_dirs.extend(jupyter_path() + [os.path.join("@out@", "share", "jupyter")])' \ + --subst-var out + '' + + pkgs.lib.optionalString (pkgs.lib.versionAtLeast final.nbconvert.version "7.0") '' + substituteInPlace \ + ./hatch_build.py \ + --replace \ + 'if self.target_name not in ["wheel", "sdist"]:' \ + 'if True:' + ''; + }); - postOverlay = final: prev: - {} - // { - babel = prev.babel_; - Babel = prev.babel_; - testbook = prev.testbook.overridePythonAttrs (old: { - postPatch = '' - mkdir ./tmp - ${pkgs.unzip}/bin/unzip dist/testbook-${old.version}-py3-none-any.whl -d ./tmp - sed -i -e "s|if not any(arg.startswith('--Kernel|if False and not any(arg.startswith('--Kernel|" tmp/testbook/client.py - rm dist/testbook-${old.version}-py3-none-any.whl - pushd tmp - ${pkgs.zip}/bin/zip -r ../dist/testbook-${old.version}-py3-none-any.whl ./* - popd - rm -rf tmp - ''; - }); - }; -in - (pkgs.poetry2nix.defaultPoetryOverrides.overrideOverlay preOverlay).extend postOverlay + testbook = prev.testbook.overridePythonAttrs (old: { + postPatch = '' + mkdir ./tmp + ${pkgs.unzip}/bin/unzip dist/testbook-${old.version}-py3-none-any.whl -d ./tmp + sed -i -e "s|if not any(arg.startswith('--Kernel|if False and not any(arg.startswith('--Kernel|" tmp/testbook/client.py + rm dist/testbook-${old.version}-py3-none-any.whl + pushd tmp + ${pkgs.zip}/bin/zip -r ../dist/testbook-${old.version}-py3-none-any.whl ./* + popd + rm -rf tmp + ''; + }); +}