From 02b932909de5125328cfcaadd660cc73012592ce Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 30 Oct 2023 15:09:36 +1300 Subject: [PATCH] mk-poetry-dep: Use wheel platform filtering from pyproject.nix --- mk-poetry-dep.nix | 18 ++- pep425.nix | 135 -------------------- tests/pep425/default.nix | 265 --------------------------------------- 3 files changed, 12 insertions(+), 406 deletions(-) delete mode 100644 pep425.nix delete mode 100644 tests/pep425/default.nix diff --git a/mk-poetry-dep.nix b/mk-poetry-dep.nix index 54f9ad71a..8edf95bec 100644 --- a/mk-poetry-dep.nix +++ b/mk-poetry-dep.nix @@ -23,20 +23,23 @@ , ... }: +let + selectWheel = files: lib.take 1 (builtins.filter (fileEntry: let + isWheelFilName = pyproject-nix.lib.pypa.isWheelFileName fileEntry.file; + file = pyproject-nix.lib.pypa.parseWheelFileName fileEntry.file; + in isWheelFilName && pyproject-nix.lib.pypa.isWheelFileCompatible python file) files); + +in + pythonPackages.callPackage ( { preferWheel ? preferWheels , ... }@args: let - inherit (python) stdenv; inherit (pyproject-nix.lib.pypa) normalizePackageName; - inherit (poetryLib) getManyLinuxDeps fetchFromLegacy fetchFromPypi; + inherit (poetryLib) getManyLinuxDeps; - inherit (import ./pep425.nix { - inherit lib python stdenv pyproject-nix; - }) selectWheel - ; fileCandidates = let supportedRegex = "^.*(" + builtins.concatStringsSep "|" supportedExtensions + ")"; @@ -45,6 +48,7 @@ pythonPackages.callPackage isCompatibleEgg = fname: ! lib.strings.hasSuffix ".egg" fname || lib.strings.hasSuffix "py${python.pythonVersion}.egg" fname; in builtins.filter (f: matchesVersion f.file && hasSupportedExtension f.file && isCompatibleEgg f.file) files; + isLocked = lib.length fileCandidates > 0; isSource = source != null; isGit = isSource && source.type == "git"; @@ -73,8 +77,10 @@ pythonPackages.callPackage isBdist = f: lib.strings.hasSuffix "whl" f.file; isSdist = f: ! isBdist f && ! isEgg f; isEgg = f: lib.strings.hasSuffix ".egg" f.file; + binaryDist = selectWheel fileCandidates; sourceDist = builtins.filter isSdist fileCandidates; + eggs = builtins.filter isEgg fileCandidates; # the `wheel` package cannot be built from a wheel, since that requires the wheel package # this causes a circular dependency so we special-case ignore its `preferWheel` attribute value diff --git a/pep425.nix b/pep425.nix deleted file mode 100644 index 27cb07cea..000000000 --- a/pep425.nix +++ /dev/null @@ -1,135 +0,0 @@ -{ lib, stdenv, python, pyproject-nix, isLinux ? stdenv.isLinux }: -let - inherit (lib.strings) escapeRegex hasPrefix hasSuffix hasInfix splitString removeSuffix; - targetMachine = pyproject-nix.lib.pep599.manyLinuxTargetMachines.${stdenv.targetPlatform.parsed.cpu.name}; - - pythonVer = - let - ver = builtins.splitVersion python.version; - major = builtins.elemAt ver 0; - minor = builtins.elemAt ver 1; - tags = [ "cp" "py" ]; - in - { inherit major minor tags; }; - # Builds with and without pymalloc (m) are ABI compatible since python 3.8 (bpo-36707) - abiTag = "cp${pythonVer.major}${pythonVer.minor}" - + lib.optionalString (builtins.compareVersions python.version "3.8" < 0) "m"; - - # - # Parses wheel file returning an attribute set - # - toWheelAttrs = str: - let - entries' = splitString "-" str; - el = builtins.length entries'; - entryAt = builtins.elemAt entries'; - - # Hack: Remove version "suffixes" like 2.11.4-1 - entries = - if el == 6 then [ - (entryAt 0) # name - (entryAt 1) # version - # build tag is skipped - (entryAt (el - 3)) # python version - (entryAt (el - 2)) # abi - (entryAt (el - 1)) # platform - ] else entries'; - p = removeSuffix ".whl" (builtins.elemAt entries 4); - in - { - pkgName = builtins.elemAt entries 0; - pkgVer = builtins.elemAt entries 1; - pyVer = builtins.elemAt entries 2; - abi = builtins.elemAt entries 3; - platform = p; - }; - - # - # Builds list of acceptable osx wheel files - # - # accepted versions in descending order of preference - # list of wheel files to select from - findBestMatches = versions: candidates: - let - v = lib.lists.head versions; - vs = lib.lists.tail versions; - in - if (builtins.length versions == 0) - then [ ] - else (builtins.filter (x: hasInfix v x.file) candidates) ++ (findBestMatches vs candidates); - - # x = "cpXX" | "py2" | "py3" | "py2.py3" - isPyVersionCompatible = { major, minor, tags }: x: - let - isCompat = m: - builtins.elem m.tag tags - && m.major == major - && builtins.compareVersions minor m.minor >= 0; - parseMarker = v: - let - tag = builtins.substring 0 2 v; - major = builtins.substring 2 1 v; - end = builtins.substring 3 3 v; - minor = if builtins.stringLength end > 0 then end else "0"; - in - { inherit major minor tag; }; - markers = splitString "." x; - in - lib.lists.any isCompat (map parseMarker markers); - - # - # Selects the best matching wheel file from a list of files - # - selectWheel = files: - let - filesWithoutSources = builtins.filter (x: hasSuffix ".whl" x.file) files; - isPyAbiCompatible = pyabi: x: x == "none" || hasPrefix pyabi x || hasPrefix x pyabi || ( - # The CPython stable ABI is abi3 as in the shared library suffix. - python.passthru.implementation == "cpython" && - builtins.elemAt (lib.splitString "." python.version) 0 == "3" && - x == "abi3" - ); - withPython = ver: abi: x: (isPyVersionCompatible ver x.pyVer) && (isPyAbiCompatible abi x.abi); - withPlatform = - if isLinux - then - if targetMachine != null - then - # See PEP 600 for details. - (p: - builtins.match "any|(linux|manylinux(1|2010|2014))_${escapeRegex targetMachine}|manylinux_[0-9]+_[0-9]+_${escapeRegex targetMachine}" p != null - ) - else - (p: p == "any") - else - if stdenv.isDarwin - then - if stdenv.targetPlatform.isAarch64 - then (p: p == "any" || (hasInfix "macosx" p && lib.lists.any (e: hasSuffix e p) [ "arm64" "aarch64" ])) - else (p: p == "any" || (hasInfix "macosx" p && hasSuffix "x86_64" p)) - else (p: p == "any"); - withPlatforms = x: lib.lists.any withPlatform (splitString "." x.platform); - filterWheel = x: - let - f = toWheelAttrs x.file; - in - (withPython pythonVer abiTag f) && (withPlatforms f); - filtered = builtins.filter filterWheel filesWithoutSources; - choose = files: - let - osxMatches = [ "12_0" "11_0" "10_15" "10_14" "10_12" "10_11" "10_10" "10_9" "10_8" "10_7" "any" ]; - linuxMatches = [ "manylinux1_" "manylinux2010_" "manylinux2014_" "manylinux_" "linux_" "any" ]; - chooseLinux = x: lib.take 1 (findBestMatches linuxMatches x); - chooseOSX = x: lib.take 1 (findBestMatches osxMatches x); - in - if isLinux - then chooseLinux files - else chooseOSX files; - in - if (builtins.length filtered == 0) - then [ ] - else choose filtered; -in -{ - inherit selectWheel toWheelAttrs isPyVersionCompatible; -} diff --git a/tests/pep425/default.nix b/tests/pep425/default.nix deleted file mode 100644 index 823b6132d..000000000 --- a/tests/pep425/default.nix +++ /dev/null @@ -1,265 +0,0 @@ -{ lib, pep425, pep425OSX, pep425PythonOldest }: - -lib.debug.runTests { - - # - # selectWheel - # - - testLinuxSimple = - let - cs = [ - { file = "grpcio-1.25.0-cp27-cp27m-macosx_10_10_x86_64.whl"; } - { file = "grpcio-1.25.0-cp27-cp27m-manylinux2010_x86_64.whl"; } - ]; - in - { - expr = pep425.selectWheel cs; - expected = [{ file = "grpcio-1.25.0-cp27-cp27m-manylinux2010_x86_64.whl"; }]; - }; - - testOSXSimple = - let - cs = [ - { file = "grpcio-1.25.0-cp27-cp27m-macosx_10_10_x86_64.whl"; } - { file = "grpcio-1.25.0-cp27-cp27m-manylinux2010_x86_64.whl"; } - ]; - in - { - expr = pep425OSX.selectWheel cs; - expected = [{ file = "grpcio-1.25.0-cp27-cp27m-macosx_10_10_x86_64.whl"; }]; - }; - - testLinuxPickPythonOldest = - let - cs = [ - { file = "grpcio-1.25.0-cp27-cp27m-macosx_10_10_x86_64.whl"; } - { file = "grpcio-1.25.0-cp27-cp27m-macosx_10_9_x86_64.whl"; } - { file = "grpcio-1.25.0-cp27-cp27m-manylinux1_i686.whl"; } - { file = "grpcio-1.25.0-cp27-cp27m-manylinux2010_x86_64.whl"; } - { file = "grpcio-1.25.0-cp37-cp37m-manylinux1_i686.whl"; } - { file = "grpcio-1.25.0-cp37-cp37m-manylinux2010_x86_64.whl"; } - ]; - in - { - expr = pep425PythonOldest.selectWheel cs; - expected = [{ file = "grpcio-1.25.0-cp37-cp37m-manylinux2010_x86_64.whl"; }]; - }; - - testOSXPreferNewer = - let - cs = [ - { file = "grpcio-1.25.0-cp27-cp27m-macosx_10_9_x86_64.whl"; } - { file = "grpcio-1.25.0-cp27-cp27m-macosx_10_12_x86_64.whl"; } - ]; - in - { - expr = pep425OSX.selectWheel cs; - expected = [{ file = "grpcio-1.25.0-cp27-cp27m-macosx_10_12_x86_64.whl"; }]; - }; - - testOSXNoMatch = - let - cs = [ - { file = "grpcio-1.25.0-cp27-cp27m-manylinux1_x86_64.whl"; } - { file = "grpcio-1.25.0-cp27-cp27m-manylinux2010_x86_64.whl"; } - ]; - in - { - expr = pep425OSX.selectWheel cs; - expected = [ ]; - }; - - testLinuxPreferOlder = - let - cs = [ - { file = "grpcio-1.25.0-cp27-cp27m-manylinux1_x86_64.whl"; } - { file = "grpcio-1.25.0-cp27-cp27m-manylinux2010_x86_64.whl"; } - ]; - in - { - expr = pep425.selectWheel cs; - expected = [{ file = "grpcio-1.25.0-cp27-cp27m-manylinux1_x86_64.whl"; }]; - }; - - testLinuxNoMatch = - let - cs = [ - { file = "grpcio-1.25.0-cp27-cp27m-macosx_10_9_x86_64.whl"; } - { file = "grpcio-1.25.0-cp27-cp27m-macosx_10_12_x86_64.whl"; } - ]; - in - { - expr = pep425.selectWheel cs; - expected = [ ]; - }; - - testLinuxEmptyList = { - expr = pep425.selectWheel [ ]; - expected = [ ]; - }; - - testOSXEmptyList = { - expr = pep425OSX.selectWheel [ ]; - expected = [ ]; - }; - - testLinuxCffiWhlFiles = - let - cs = [ - { file = "cffi-1.13.2-cp27-cp27m-macosx_10_6_intel.whl"; } - { file = "cffi-1.13.2-cp27-cp27m-manylinux1_i686.whl"; } - { file = "cffi-1.13.2-cp27-cp27m-manylinux1_x86_64.whl"; } - { file = "cffi-1.13.2-cp27-cp27m-win32.whl"; } - { file = "cffi-1.13.2-cp27-cp27m-win_amd64.whl"; } - { file = "cffi-1.13.2-cp27-cp27mu-manylinux1_i686.whl"; } - { file = "cffi-1.13.2-cp27-cp27mu-manylinux1_x86_64.whl"; } - { file = "cffi-1.13.2-cp34-cp34m-macosx_10_6_intel.whl"; } - { file = "cffi-1.13.2-cp34-cp34m-manylinux1_i686.whl"; } - { file = "cffi-1.13.2-cp34-cp34m-manylinux1_x86_64.whl"; } - { file = "cffi-1.13.2-cp34-cp34m-win32.whl"; } - { file = "cffi-1.13.2-cp34-cp34m-win_amd64.whl"; } - { file = "cffi-1.13.2-cp35-cp35m-macosx_10_6_intel.whl"; } - { file = "cffi-1.13.2-cp35-cp35m-manylinux1_i686.whl"; } - { file = "cffi-1.13.2-cp35-cp35m-manylinux1_x86_64.whl"; } - { file = "cffi-1.13.2-cp35-cp35m-win32.whl"; } - { file = "cffi-1.13.2-cp35-cp35m-win_amd64.whl"; } - { file = "cffi-1.13.2-cp36-cp36m-macosx_10_6_intel.whl"; } - { file = "cffi-1.13.2-cp36-cp36m-manylinux1_i686.whl"; } - { file = "cffi-1.13.2-cp36-cp36m-manylinux1_x86_64.whl"; } - { file = "cffi-1.13.2-cp36-cp36m-win32.whl"; } - { file = "cffi-1.13.2-cp36-cp36m-win_amd64.whl"; } - { file = "cffi-1.13.2-cp37-cp37m-macosx_10_6_intel.whl"; } - { file = "cffi-1.13.2-cp37-cp37m-manylinux1_i686.whl"; } - { file = "cffi-1.13.2-cp37-cp37m-manylinux1_x86_64.whl"; } - { file = "cffi-1.13.2-cp37-cp37m-win32.whl"; } - { file = "cffi-1.13.2-cp37-cp37m-win_amd64.whl"; } - { file = "cffi-1.13.2-cp38-cp38-macosx_10_9_x86_64.whl"; } - { file = "cffi-1.13.2-cp38-cp38-manylinux1_i686.whl"; } - { file = "cffi-1.13.2-cp38-cp38-manylinux1_x86_64.whl"; } - { file = "cffi-1.13.2-cp38-cp38-win32.whl"; } - { file = "cffi-1.13.2-cp38-cp38-win_amd64.whl"; } - { file = "cffi-1.13.2.tar.gz"; } - ]; - in - { - expr = pep425.selectWheel cs; - expected = [{ file = "cffi-1.13.2-cp27-cp27m-manylinux1_x86_64.whl"; }]; - }; - - testMsgPack = - let - cs = [ - { file = "msgpack-0.6.2-cp27-cp27m-manylinux1_i686.whl"; } - { file = "msgpack-0.6.2-cp27-cp27m-manylinux1_x86_64.whl"; } - { file = "msgpack-0.6.2-cp27-cp27m-win32.whl"; } - { file = "msgpack-0.6.2-cp27-cp27m-win_amd64.whl"; } - { file = "msgpack-0.6.2-cp27-cp27mu-manylinux1_i686.whl"; } - { file = "msgpack-0.6.2-cp27-cp27mu-manylinux1_x86_64.whl"; } - { file = "msgpack-0.6.2-cp35-cp35m-macosx_10_6_intel.whl"; } - { file = "msgpack-0.6.2-cp35-cp35m-manylinux1_i686.whl"; } - { file = "msgpack-0.6.2-cp35-cp35m-manylinux1_x86_64.whl"; } - { file = "msgpack-0.6.2-cp36-cp36m-macosx_10_6_intel.whl"; } - { file = "msgpack-0.6.2-cp36-cp36m-manylinux1_i686.whl"; } - { file = "msgpack-0.6.2-cp36-cp36m-manylinux1_x86_64.whl"; } - { file = "msgpack-0.6.2-cp36-cp36m-win32.whl"; } - { file = "msgpack-0.6.2-cp36-cp36m-win_amd64.whl"; } - { file = "msgpack-0.6.2-cp37-cp37m-macosx_10_14_x86_64.whl"; } - { file = "msgpack-0.6.2-cp37-cp37m-macosx_10_9_x86_64.whl"; } - { file = "msgpack-0.6.2-cp37-cp37m-manylinux1_i686.whl"; } - { file = "msgpack-0.6.2-cp37-cp37m-manylinux1_x86_64.whl"; } - { file = "msgpack-0.6.2-cp37-cp37m-win32.whl"; } - { file = "msgpack-0.6.2-cp37-cp37m-win_amd64.whl"; } - { file = "msgpack-0.6.2.tar.gz"; } - ]; - in - { - expr = pep425PythonOldest.selectWheel cs; - expected = [{ file = "msgpack-0.6.2-cp37-cp37m-manylinux1_x86_64.whl"; }]; - }; - - testNonManyLinuxWheels = - let - cs = [ - { file = "tensorboard-1.14.0-py2-none-any.whl"; } - { file = "tensorboard-1.14.0-py3-none-any.whl"; } - ]; - in - { - expr = pep425PythonOldest.selectWheel cs; - expected = [{ file = "tensorboard-1.14.0-py3-none-any.whl"; }]; - }; - - testPy2Py3Wheels = - let - cs = [ - { file = "tensorboard-1.14.0-py2.py3-none-any.whl"; } - ]; - in - { - expr = pep425PythonOldest.selectWheel cs; - expected = [{ file = "tensorboard-1.14.0-py2.py3-none-any.whl"; }]; - }; - - # - # toWheelAttrs - # - - testToWheelAttrs = - let - name = "msgpack-0.6.2-cp27-cp27m-manylinux1_i686.whl"; - in - { - expr = pep425.toWheelAttrs name; - expected = { - pkgName = "msgpack"; - pkgVer = "0.6.2"; - pyVer = "cp27"; - abi = "cp27m"; - platform = "manylinux1_i686"; - }; - }; - - testToWheelAttrsAny = - let - name = "tensorboard-1.14.0-py3-none-any.whl"; - in - { - expr = pep425.toWheelAttrs name; - expected = { - pkgName = "tensorboard"; - pkgVer = "1.14.0"; - pyVer = "py3"; - abi = "none"; - platform = "any"; - }; - }; - - # - # isPyVersionCompatible - # - - tesPyVersionCompatible = - let - f = pep425.isPyVersionCompatible; - in - { - expr = [ - (f "cp27" "cp27") - (f "cp27" "cp37") - (f "cp27" "py2") - (f "cp27" "py3") - (f "cp27" "py2.py3") - (f "cp37" "py2.py3") - ]; - - expected = [ - true - false - true - false - true - ]; - }; - -}