From e41234b828be102991ae548c3c19c27d2b95c5be Mon Sep 17 00:00:00 2001 From: Dmitrii Sutiagin Date: Thu, 7 Sep 2023 09:18:00 -0700 Subject: [PATCH] Add symlink traversal to support PDM cached envs (#237) * Add symlink traversal to support PDM cached envs Fixes #236 Co-authored-by: Dmitrii Sutiagin --- setup.cfg | 2 +- src/shiv/builder.py | 18 ++++++++++++++++-- test/test_builder.py | 12 +++++++++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 36d394a..f0e3dca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER [metadata] name = shiv -version = 1.0.3 +version = 1.0.4 description = A command line utility for building fully self contained Python zipapps. long_description = file: README.md long_description_content_type = text/markdown diff --git a/src/shiv/builder.py b/src/shiv/builder.py index e4c14aa..67a7753 100644 --- a/src/shiv/builder.py +++ b/src/shiv/builder.py @@ -12,9 +12,10 @@ import zipfile from datetime import datetime, timezone +from itertools import chain from pathlib import Path from stat import S_IFMT, S_IMODE, S_IXGRP, S_IXOTH, S_IXUSR -from typing import IO, Any, List, Optional, Tuple +from typing import Generator, IO, Any, List, Optional, Tuple from . import bootstrap from .bootstrap.environment import Environment @@ -69,6 +70,15 @@ def write_to_zipapp( archive.writestr(zinfo, data) +def rglob_follow_symlinks(path: Path, glob: str) -> Generator[Path, None, None]: + """Path.rglob extended to follow symlinks, while we wait for Python 3.13.""" + for p in path.rglob('*'): + if p.is_symlink() and p.is_dir(): + yield from chain([p], rglob_follow_symlinks(p, glob)) + else: + yield p + + def create_archive( sources: List[Path], target: Path, interpreter: str, main: str, env: Environment, compressed: bool = True ) -> None: @@ -110,7 +120,11 @@ def create_archive( # Glob is known to return results in non-deterministic order. # We need to sort them by in-archive paths to ensure # that archive contents are reproducible. - for path in sorted(source.rglob("*"), key=str): + # + # NOTE: https://github.com/linkedin/shiv/issues/236 + # this special rglob function can be replaced with "rglob('*', follow_symlinks=True)" + # when Python 3.13 becomes the lowest supported version + for path in sorted(rglob_follow_symlinks(source, "*"), key=str): # Skip compiled files and directories (as they are not required to be present in the zip). if path.suffix == ".pyc" or path.is_dir(): diff --git a/test/test_builder.py b/test/test_builder.py index 4d8a357..ec5af35 100644 --- a/test/test_builder.py +++ b/test/test_builder.py @@ -9,7 +9,7 @@ import pytest -from shiv.builder import create_archive, write_file_prefix +from shiv.builder import create_archive, rglob_follow_symlinks, write_file_prefix UGOX = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH @@ -39,6 +39,16 @@ def test_binprm_error(self): with pytest.raises(SystemExit): tmp_write_prefix(f"/{'c' * 200}/python") + def test_rglob_follow_symlinks(self, tmp_path): + real_dir = tmp_path / 'real_dir' + real_dir.mkdir() + real_file = real_dir / 'real_file' + real_file.touch() + sym_dir = tmp_path / 'sym_dir' + sym_dir.symlink_to(real_dir) + sym_file = sym_dir / real_file.name + assert sorted(rglob_follow_symlinks(tmp_path, '*'), key=str) == [real_dir, real_file, sym_dir, sym_file] + def test_create_archive(self, sp, env): with tempfile.TemporaryDirectory() as tmpdir: target = Path(tmpdir, "test.zip")