Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Validate and update macOS platform tag in wheel name #198

Merged
merged 36 commits into from
Feb 17, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
efa212e
initial implementation
Czaki Feb 4, 2024
c3a35e5
use Path
Czaki Feb 5, 2024
670bb76
change regexp name
Czaki Feb 5, 2024
d242da7
add set type annotation
Czaki Feb 5, 2024
93b17d9
extract logic to separate function, update function names
Czaki Feb 5, 2024
5dad39d
add basic tests
Czaki Feb 5, 2024
3570f59
add test for universal2
Czaki Feb 5, 2024
2078633
apply changes from code review
Czaki Feb 6, 2024
07c3976
fix test part 1
Czaki Feb 6, 2024
6885624
fix mypy
Czaki Feb 6, 2024
a56ec61
update changelog
Czaki Feb 6, 2024
c086486
fix test 2
Czaki Feb 6, 2024
6332cb5
verbose run
Czaki Feb 6, 2024
7726378
change name of wheel
Czaki Feb 6, 2024
0bbbbad
provide flag
Czaki Feb 6, 2024
b109f7b
improve exception
Czaki Feb 7, 2024
6ba5099
fix test
Czaki Feb 7, 2024
d0d99c6
add tests
Czaki Feb 7, 2024
0fd885d
add tests
Czaki Feb 7, 2024
2b6a3e1
fix calling by absolute path
Czaki Feb 7, 2024
fc081af
Apply suggestions from code review
Czaki Feb 7, 2024
b939a1e
add mised type
Czaki Feb 7, 2024
d65946c
remove old wheel
Czaki Feb 7, 2024
046fcab
fix updating metadata and add testing it
Czaki Feb 7, 2024
2486313
add docstrings
Czaki Feb 7, 2024
048b256
fix mypy
Czaki Feb 7, 2024
e972852
fix wrong variable name
Czaki Feb 7, 2024
89871b9
Apply suggestions from code review
Czaki Feb 8, 2024
987dc95
Apply suggestions from code review
Czaki Feb 8, 2024
5ce9344
next fixes from code review
Czaki Feb 8, 2024
2f13b20
fix pre-commit formatting
Czaki Feb 8, 2024
9b8aaca
Update delocate/tests/test_scripts.py
Czaki Feb 10, 2024
bd85986
pre-commit fix
Czaki Feb 10, 2024
222033d
fix reporting problematic libs for x86_64
Czaki Feb 10, 2024
e2979a1
add assetr for testing
Czaki Feb 10, 2024
51f03fc
add comment with explanation
Czaki Feb 10, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions delocate/cmd/delocate_wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,14 @@
" (from 'intel', 'i386', 'x86_64', 'i386,x86_64', 'universal2',"
" 'x86_64,arm64')",
)
parser.add_argument(
"--verify-name",
action="store_true",
help="Verify if platform tag in wheel name is proper",
)
parser.add_argument(
"--fix-name", action="store_true", help="Fix platform tag in wheel name"
)


def main() -> None:
Expand Down Expand Up @@ -94,6 +102,8 @@ def main() -> None:
out_wheel,
lib_sdir=args.lib_sdir,
require_archs=require_archs,
check_wheel_name=args.verify_name,
fix_wheel_name=args.fix_name,
**delocate_values(args),
)
if args.verbose and len(copied):
Expand Down
211 changes: 211 additions & 0 deletions delocate/delocating.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
from __future__ import division, print_function

import functools
import itertools
import logging
import os
import re
import shutil
import warnings
from os.path import abspath, basename, dirname, exists, realpath, relpath
Expand All @@ -26,6 +28,15 @@
Union,
)

from macholib.mach_o import ( # type: ignore[import-untyped]
CPU_TYPE_NAMES,
LC_BUILD_VERSION,
LC_VERSION_MIN_MACOSX,
)
from macholib.MachO import MachO # type: ignore[import-untyped]
from packaging.utils import parse_wheel_filename
from packaging.version import Version

from .libsana import (
_allow_all,
get_rp_stripper,
Expand All @@ -51,6 +62,8 @@
# Prefix for install_name_id of copied libraries
DLC_PREFIX = "/DLC/"

_PLATFORM_REGEXP = re.compile(r"macosx_(\d+)_(\d+)_(\w+)")


class DelocationError(Exception):
pass
Expand Down Expand Up @@ -578,6 +591,191 @@
validate_signature(lib)


def _get_macos_min_version(dylib_path: Path) -> Iterable[Tuple[str, Version]]:
"""Get the minimum macOS version from a dylib file.

Parameters
----------
dylib_path : Path
The path to the dylib file.

Returns
-------
List[Tuple[str, Version]]
A list of tuples containing the CPU type and the minimum macOS version.
"""
m = MachO(dylib_path)
res = []
for header in m.headers:
for cmd in header.commands:
if cmd[0].cmd == LC_BUILD_VERSION:
version = cmd[1].minos
elif cmd[0].cmd == LC_VERSION_MIN_MACOSX:
version = cmd[1].version
else:
continue
res.append(
(
CPU_TYPE_NAMES.get(header.header.cputype, "unknown"),
Version(f"{version >> 16 & 0xFF}.{version >> 8 & 0xFF}"),
)
)
break
return res
Czaki marked this conversation as resolved.
Show resolved Hide resolved


def _get_archs_and_version_from_wheel_name(
wheel_name: str,
) -> Dict[str, Version]:
"""
Get the architecture and minimum macOS version from the wheel name.

Parameters
----------
wheel_name : str
The name of the wheel.

Returns
-------
Dict[str, Version]
A dictionary containing the architecture and minimum macOS version
for each architecture in the wheel name.
"""
platform_tag_set = parse_wheel_filename(wheel_name)[-1]
res = {}
for platform_tag in platform_tag_set:
match = _PLATFORM_REGEXP.match(platform_tag.platform)
if match is None:
raise ValueError(f"Invalid platform tag: {platform_tag.platform}")
major, minor, arch = match.groups()
res[arch] = Version(f"{major}.{minor}")
return res
Czaki marked this conversation as resolved.
Show resolved Hide resolved


def _get_problematic_libs(
version: Version, version_lib_dict: Dict[Version, List[Path]]
) -> Set[Path]:
"""
Filter libraries that require more modern macOS
version than the provided one.

Parameters
----------
version : Version
The expected minimum macOS version
version_lib_dict : Dict[Version, List[Path]]
A dictionary containing mapping from minimum macOS version to libraries
that require that version.

Returns
-------
Set[Path]
A set of libraries that require a more modern macOS version than the
provided one.
"""
result = set()
for v, lib in version_lib_dict.items():
if v > version:
result.update(lib)
return result


def _calculate_minimum_wheel_name(
wheel_name: str, wheel_dir: Path
) -> Tuple[str, set[Path]]:

Check failure on line 685 in delocate/delocating.py

View workflow job for this annotation

GitHub Actions / mypy

"set" is not subscriptable, use "typing.Set" instead
HexDecimal marked this conversation as resolved.
Show resolved Hide resolved
"""
Update wheel name platform tag, based on the architecture
of the libraries in the wheel and actual platform tag.

Parameters
----------
wheel_name : str
The name of the wheel.
wheel_dir : Path
The directory of the unpacked wheel.

Returns
-------
str
The updated wheel name.
"""
# get platform tag from wheel name using packaging
arch_version = _get_archs_and_version_from_wheel_name(wheel_name)
# get the architecture and minimum macOS version from the libraries
# in the wheel
version_info_dict: Dict[str, Dict[Version, List[Path]]] = {}

for lib in itertools.chain(
wheel_dir.glob("**/*.dylib"), wheel_dir.glob("**/*.so")
):
for arch, version in _get_macos_min_version(lib):
version_info_dict.setdefault(arch.lower(), {}).setdefault(
version, []
).append(lib)
version_dkt = {
arch: max(version) for arch, version in version_info_dict.items()
}

problematic_libs: Set[Path] = set()

for arch, version in list(arch_version.items()):
if arch == "universal2":
if version_dkt["arm64"] == Version("11.0"):
arch_version["universal2"] = max(version, version_dkt["x86_64"])
else:
arch_version["universal2"] = max(
version, version_dkt["arm64"], version_dkt["x86_64"]
)
problematic_libs.update(
_get_problematic_libs(version, version_info_dict["arm64"])
)
problematic_libs.update(
_get_problematic_libs(version, version_info_dict["x86_64"])
)
elif arch == "universal":
arch_version["universal"] = max(
version, version_dkt["i386"], version_dkt["x86_64"]
)
problematic_libs.update(
_get_problematic_libs(version, version_info_dict["i386"])
)
problematic_libs.update(
_get_problematic_libs(version, version_info_dict["x86_64"])
)
else:
arch_version[arch] = max(version, version_dkt[arch])
problematic_libs.update(
_get_problematic_libs(version, version_info_dict[arch])
)
prefix = wheel_name.rsplit("-", 1)[0]
platform_tag = ".".join(
f"macosx_{version.major}_{version.minor}_{arch}"
for arch, version in arch_version.items()
)
return f"{prefix}-{platform_tag}.whl", problematic_libs


def _check_and_update_wheel_name(
wheel_path: str, wheel_dir: Path, check_wheel_name: bool
) -> str:
wheel_name = os.path.basename(wheel_path)

new_name, problematic_files = _calculate_minimum_wheel_name(
wheel_name, Path(wheel_dir)
)
if check_wheel_name and new_name != wheel_name:
problematic_files_str = "\n".join(str(x) for x in problematic_files)
raise DelocationError(
"Wheel name does not satisfy minimal package requirements. "
f"Provided name is {os.path.basename(wheel_name)}, "
f"but the minimal name should be {os.path.basename(new_name)}. "
f"Problematic files are:\n{problematic_files_str}."
)
if new_name != wheel_name:
wheel_path = os.path.join(os.path.dirname(wheel_path), new_name)
return wheel_path


def delocate_wheel(
in_wheel: str,
out_wheel: Optional[str] = None,
Expand All @@ -590,6 +788,8 @@
executable_path: Optional[str] = None,
ignore_missing: bool = False,
sanitize_rpaths: bool = False,
check_wheel_name: bool = False,
fix_wheel_name: bool = False,
) -> Dict[str, Dict[str, str]]:
"""Update wheel by copying required libraries to `lib_sdir` in wheel

Expand Down Expand Up @@ -637,6 +837,10 @@
Continue even if missing dependencies are detected.
sanitize_rpaths : bool, default=False, keyword-only
If True, absolute paths in rpaths of binaries are removed.
check_wheel_name : bool, default=False, keyword-only
If True, check if wheel name platform tag is proper.
fix_wheel_name : bool, default=False, keyword-only
If True, fix wheel name platform tag.

Returns
-------
Expand Down Expand Up @@ -704,6 +908,13 @@
install_id_prefix=DLC_PREFIX + relpath(lib_sdir, wheel_dir),
)
rewrite_record(wheel_dir)
if check_wheel_name or fix_wheel_name:
out_wheel_fixed = _check_and_update_wheel_name(
out_wheel, Path(wheel_dir), check_wheel_name
)
if out_wheel_fixed != out_wheel:
out_wheel = out_wheel_fixed
in_place = False
if len(copied_libs) or not in_place:
dir2zip(wheel_dir, out_wheel)
return stripped_lib_dict(copied_libs, wheel_dir + os.path.sep)
Expand Down
Binary file added delocate/tests/data/libam1_12.dylib
Binary file not shown.
Binary file added delocate/tests/data/libc_12.dylib
Binary file not shown.
4 changes: 4 additions & 0 deletions delocate/tests/data/make_libs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ void c();
int main(int, char**) { c(); return 0; }
EOF


CXX_64="$CXX -arch x86_64"
CXX_M1="$CXX -arch arm64"

Expand All @@ -45,13 +46,16 @@ fi

$CXX_64 -o liba.dylib -dynamiclib a.cc
$CXX_M1 -o libam1.dylib -dynamiclib a.cc
MACOSX_DEPLOYMENT_TARGET=12.0 $CXX_M1 -o libam1_12.dylib -dynamiclib a.cc
$CXX_64 -o a.o -c a.cc
ar rcs liba.a a.o
$CXX_64 -o libb.dylib -dynamiclib b.cc -L. -la
$CXX_64 -o libb.dylib -dynamiclib b.cc -L. -la
$CXX_64 -o libc.dylib -dynamiclib c.cc -L. -la -lb
$CXX_64 -o test-lib d.cc -L. -lc

MACOSX_DEPLOYMENT_TARGET=12.0 $CXX_64 -o libc_12.dylib -dynamiclib c.cc -L. -la -lb

# Make a dual-arch library
lipo -create liba.dylib libam1.dylib -output liba_both.dylib

Expand Down
83 changes: 83 additions & 0 deletions delocate/tests/test_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -658,3 +658,86 @@ def test_glob(
assert "FileNotFoundError:" in result.stderr

script_runner.run(["delocate-path", "*/"], check=True, cwd=tmp_path)


@pytest.mark.xfail( # type: ignore[misc]
sys.platform != "darwin", reason="Needs macOS linkage."
)
def test_delocate_wheel_fix_name(
plat_wheel: PlatWheel, script_runner: ScriptRunner, tmp_path: Path
) -> None:
zip2dir(plat_wheel.whl, tmp_path / "plat")
whl_10_6 = tmp_path / "plat-1.0-cp311-cp311-macosx_10_6_x86_64.whl"
dir2zip(tmp_path / "plat", whl_10_6)
script_runner.run(
["delocate-wheel", whl_10_6, "--fix-name"], check=True, cwd=tmp_path
)
assert (tmp_path / "plat-1.0-cp311-cp311-macosx_10_9_x86_64.whl").exists()


@pytest.mark.xfail( # type: ignore[misc]
sys.platform != "darwin", reason="Needs macOS linkage."
)
def test_delocate_wheel_verify_name(
plat_wheel: PlatWheel, script_runner: ScriptRunner, tmp_path: Path
) -> None:
zip2dir(plat_wheel.whl, tmp_path / "plat")
whl_10_6 = tmp_path / "plat-1.0-cp311-cp311-macosx_10_6_x86_64.whl"
dir2zip(tmp_path / "plat", whl_10_6)
result = script_runner.run(
["delocate-wheel", whl_10_6, "--verify-name"],
check=False,
cwd=tmp_path,
print_result=False,
)
assert result.returncode != 0
assert (
"Wheel name does not satisfy minimal package requirements"
in result.stderr
)
assert "is plat-1.0-cp311-cp311-macosx_10_6_x86_64.whl" in result.stderr
assert "be plat-1.0-cp311-cp311-macosx_10_9_x86_64.whl" in result.stderr


@pytest.mark.xfail( # type: ignore[misc]
sys.platform != "darwin", reason="Needs macOS linkage."
)
def test_delocate_wheel_verify_name_universal2_ok(
plat_wheel: PlatWheel, script_runner: ScriptRunner, tmp_path: Path
) -> None:
zip2dir(plat_wheel.whl, tmp_path / "plat")
shutil.copy(
DATA_PATH / "libam1.dylib", tmp_path / "plat/fakepkg1/libam1.dylib"
)
whl_10_9 = tmp_path / "plat-1.0-cp311-cp311-macosx_10_9_universal2.whl"
dir2zip(tmp_path / "plat", whl_10_9)
script_runner.run(
["delocate-wheel", whl_10_9, "--verify-name"], check=True, cwd=tmp_path
)


@pytest.mark.xfail( # type: ignore[misc]
sys.platform != "darwin", reason="Needs macOS linkage."
)
def test_delocate_wheel_verify_name_universal2_verify_crash(
plat_wheel: PlatWheel, script_runner: ScriptRunner, tmp_path: Path
) -> None:
zip2dir(plat_wheel.whl, tmp_path / "plat")
shutil.copy(
DATA_PATH / "libam1_12.dylib", tmp_path / "plat/fakepkg1/libam1.dylib"
)
whl_10_9 = tmp_path / "plat-1.0-cp311-cp311-macosx_10_9_universal2.whl"
dir2zip(tmp_path / "plat", whl_10_9)
result = script_runner.run(
["delocate-wheel", whl_10_9, "--verify-name"],
check=False,
cwd=tmp_path,
print_result=False,
)
assert result.returncode != 0
assert (
"Wheel name does not satisfy minimal package requirements"
in result.stderr
)
assert "is plat-1.0-cp311-cp311-macosx_10_9_universal2.whl" in result.stderr
assert "be plat-1.0-cp311-cp311-macosx_12_0_universal2.whl" in result.stderr
Loading
Loading