diff --git a/README.md b/README.md index 78fb70e..43f23d0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # [zipapps](https://github.com/ClericPy/zipapps) -[![PyPI](https://img.shields.io/pypi/v/zipapps?style=plastic)](https://pypi.org/project/zipapps/)[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/clericpy/zipapps/pythonpackage.yml)](https://github.com/ClericPy/zipapps/actions)![PyPI - Wheel](https://img.shields.io/pypi/wheel/zipapps?style=plastic)![PyPI - Downloads](https://img.shields.io/pypi/dm/zipapps?style=plastic)![PyPI - License](https://img.shields.io/pypi/l/zipapps?style=plastic) +[![PyPI](https://img.shields.io/pypi/v/zipapps?style=plastic)](https://pypi.org/project/zipapps/)[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/clericpy/zipapps/pythonpackage.yml)](https://github.com/ClericPy/zipapps/actions)![PyPI - Wheel](https://img.shields.io/pypi/wheel/zipapps?style=plastic)![Python Version from PEP 621 TOML](https://img.shields.io/python/required-version-toml?tomlFilePath=zipapps)![PyPI - Downloads](https://img.shields.io/pypi/dm/zipapps?style=plastic)![PyPI - License](https://img.shields.io/pypi/l/zipapps?style=plastic) [Changelogs](https://github.com/ClericPy/zipapps/blob/master/changelog.md) diff --git a/changelog.md b/changelog.md index 6f2cb1b..ebe6acf 100644 --- a/changelog.md +++ b/changelog.md @@ -1,6 +1,17 @@ # Changelogs +- 2024.08.07 + - [**Compatible WARNING**]: update `sys_paths` insert index from `-1` to `0` + - disable `--download-python`, use `python -m zipapps.download_python` instead + - add `-a/--auto` + - auto download the latest version matched the current platform: x86_64+install_only + - `python -m zipapps.download_python -a` + - add `-k/--keywords`, filt with keywords split by `,` + - `python -m zipapps.download_python -a -k 3.11` + - add `-u/--unzip`, unzip the `.tar.gz` + - `python -m zipapps.download_python -a -k 3.11 -u` + - download and unzip the `.tar.gz` - 2024.06.04 - add arg `--download-python`: interactive download standalone python interpreter (https://www.github.com/indygreg/python-build-standalone) - custom `--rm-patterns` to remove useless files . fixed #28 #29 diff --git a/test_utils.py b/test_utils.py index 179a7d7..8ea0e27 100644 --- a/test_utils.py +++ b/test_utils.py @@ -310,21 +310,19 @@ def test_includes(): def test_pip_args(): # test pip_args _clean_paths() - _, stderr_output = subprocess.Popen( - [sys.executable, "-c", "import bottle"], + stdout, _ = subprocess.Popen( + [sys.executable, "-c", "import bottle;print(bottle.__file__)"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, ).communicate() - assert ( - b"No module named" in stderr_output - ), "check init failed, bottle should not be installed" + assert b'app.pyz' not in stdout, "test pip_args failed %s" % stdout app_path = create_app(pip_args=["bottle"]) - _, stderr_output = subprocess.Popen( - [sys.executable, str(app_path), "-c", "import bottle"], + stdout, _ = subprocess.Popen( + [sys.executable, str(app_path), "-c", "import bottle;print(bottle.__file__)"], stderr=subprocess.PIPE, stdout=subprocess.PIPE, ).communicate() - assert stderr_output == b"", "test pip_args failed" + assert b'app.pyz' in stdout, "test pip_args failed %s" % stdout def test_cache_path(): @@ -626,8 +624,6 @@ def test_sys_paths(): # pip install by given --target args = [sys.executable, "-m", "pip", "install", "bottle", "-t", "./bottle_env"] subprocess.Popen(args=args).wait() - mock_requirement = Path("_requirements.txt") - mock_requirement.write_text("bottle") create_app(sys_paths="$SELF/bottle_env") output = subprocess.check_output( [sys.executable, "app.pyz", "-c", "import bottle;print(bottle.__file__)"] diff --git a/zipapps/__main__.py b/zipapps/__main__.py index 27d253d..82fe1f8 100644 --- a/zipapps/__main__.py +++ b/zipapps/__main__.py @@ -304,7 +304,7 @@ def main(): "--python-path", default="", dest="sys_paths", - help="Paths be insert to sys.path[-1] while running." + help="Paths be insert to sys.path[0] while running." " Support $TEMP/$HOME/$SELF/$PID/$CWD prefix, separated by commas.", ) parser.add_argument( @@ -395,7 +395,7 @@ def main(): "--download-python", action="store_true", dest="download_python", - help=f'Download standalone python from "{DOWNLOAD_PYTHON_URL}"', + help=f'(please use `python -m zipapps.download_python` instead). Download standalone python from "{DOWNLOAD_PYTHON_URL}"', ) parser.add_argument( "--rm-patterns", @@ -409,9 +409,7 @@ def main(): return args, pip_args = parser.parse_known_args() if args.download_python: - from .download_python import download_python - - return download_python() + raise ValueError("please use `python -m zipapps.download_python` instead") elif args.download_pip_pyz: return download_pip_pyz(args.download_pip_pyz) if args.quite_mode: diff --git a/zipapps/download_python.py b/zipapps/download_python.py index b5268cc..44c6fa1 100644 --- a/zipapps/download_python.py +++ b/zipapps/download_python.py @@ -1,5 +1,9 @@ +import argparse import http.client import json +import re +import shutil +import tarfile import time import traceback import urllib.request @@ -10,9 +14,13 @@ @dataclass class Asset(object): + # cpython-3.9.19+20240726-x86_64_v4-unknown-linux-musl-lto-full.tar.zst name: str + # keywords=['cpython', '3.9.19', 'x86_64_v4', 'unknown', 'linux', 'musl', 'noopt-full.tar.zst'] keywords: str + # https://github.com/indygreg/python-build-standalone/releases/download/20240726/cpython-3.9.19%2B20240726-x86_64_v4-unknown-linux-musl-noopt-full.tar.zst url: str + # 27204188 size: int @@ -41,10 +49,70 @@ def get_time(): return time.strftime("%H:%M:%S") +def choose_asset(assets: list, keywords, auto=False): + keywords = {k for k in keywords if k} + not_include = set() + if auto: + import platform + + system = platform.system().lower() + keywords.add(system) + if system == "windows": + keywords.add("msvc-shared") + elif system == "linux": + keywords.add("gnu") + keywords.add("unknown") + not_include.add("v2") + not_include.add("v3") + not_include.add("v4") + keywords.add("install_only") + keywords.add("tar.gz") + keywords.add("cpython") + keywords.add("x86_64") + not_include.add("stripped") + + result = [] + for asset in assets: + url = asset.url + for k in not_include: + if k in url: + break + else: + for k in keywords: + if k not in url: + break + else: + result.append(asset) + if result: + if len(result) > 1: + # cpython-3.10.14 + def sort_k(i: Asset): + # cpython-3.9.19 + m = re.search(r"-(\d+)\.(\d+)\.(\d+)", i.name) + if m: + a, b, c = int(m[1]), int(m[2]), int(m[3]) + return a * 10**3 + b * 10**2 + c * 10**1 + else: + return i.size + + result.sort(key=sort_k, reverse=True) + for i in result: + print(i.url) + print( + f"[{get_time()}] Got {len(result)} urls from github, will choose the first one and start download in 3 seconds. {result[0].name}" + ) + for _ in range(3): + print(3 - _, flush=True) + time.sleep(1) + return result[0] + else: + raise ValueError("No asset found.") + + def download_python(): - """Download python portable interpreter from https://github.com/indygreg/python-build-standalone/releases. `python -m morebuiltins.download_python` + """Download python portable interpreter from https://github.com/indygreg/python-build-standalone/releases. `python -m download_python -i` or `python -m download_python -a`(auto download the latest version matched the current platform: x86_64+install_only) or `python -m download_python -auto -k 3.11 -u` - λ python -m morebuiltins.download_python + λ python -m download_python -i [10:56:17] Checking https://api.github.com/repos/indygreg/python-build-standalone/releases/latest [10:56:19] View the rules: https://gregoryszorc.com/docs/python-build-standalone/main/running.html#obtaining-distributions @@ -90,6 +158,33 @@ def download_python(): D:\github\morebuiltins\morebuiltins\download_python\cpython-3.12.3+20240415-x86_64-pc-windows-msvc-install_only.tar.gz [10:56:44] Downloading: 39.12 / 39.12 MB | 100.00% | 11.3 MB/s | 0s [10:56:44] Download complete.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--auto", + "-a", + help="auto choose the best choice(like platform), such as: -k 3.11 --auto", + action="store_true", + ) + parser.add_argument("--target", "-t", help="target dir path", default=".") + parser.add_argument( + "--keywords", + "-k", + help="keywords to filt, split by , for many keywords.", + default="", + ) + parser.add_argument( + "-i", + "--interactive", + action="store_true", + help="interactive mode, will ask for filt keywords.", + ) + parser.add_argument( + "-u", + "--unzip", + action="store_true", + help="unzip the tar.gz file to target/short_name/. only for .tar.gz", + ) + args, _ = parser.parse_known_args() print( f"[{get_time()}] Checking https://api.github.com/repos/indygreg/python-build-standalone/releases/latest", flush=True, @@ -100,52 +195,58 @@ def download_python(): flush=True, ) print(f"[{get_time()}] Got {len(assets)} urls from github.") + keywords = args.keywords.split(",") + if args.interactive: + if args.keywords: + assets = choose_asset(assets, keywords, auto=False) - def sort_key(s): - try: - return tuple(map(int, s.split("."))) - except ValueError: - return s + def sort_key(s): + try: + return tuple(map(int, s.split("."))) + except ValueError: + return s - indexs = [4, 1, 5, 2, 3, 0, 6] - for index in indexs: - try: - to_filt = sorted( - {i.keywords[index] for i in assets}, key=sort_key, reverse=True + indexs = [4, 1, 5, 2, 3, 0, 6] + for index in indexs: + try: + to_filt = sorted( + {i.keywords[index] for i in assets}, key=sort_key, reverse=True + ) + except IndexError: + continue + if len(to_filt) == 1: + continue + choices = "\n".join((f"{idx}. {ii}" for idx, ii in enumerate(to_filt, 0))) + arg = input( + f"\n[{len(assets)}] Enter keywords (can be int index or partial match, defaults to 0):\n{choices}\n" ) - except IndexError: - continue - if len(to_filt) == 1: - continue - choices = "\n".join((f"{idx}. {ii}" for idx, ii in enumerate(to_filt, 0))) - arg = input( - f"\n[{len(assets)}] Enter keywords (can be int index or partial match, defaults to 0):\n{choices}\n" - ) - if not arg: - arg = to_filt[0] - elif arg.isdigit(): - arg = to_filt[int(arg)] - old = len(assets) - temp = [i for i in assets if arg == i.keywords[index]] - if temp: - assets = temp - else: - assets = [i for i in assets if arg in i.keywords[index]] - print( - f'[{get_time()}] Filt with keyword: "{arg}".', - old, - "=>", - len(assets), - flush=True, - ) - while len(assets) > 1: - names = "\n".join(i.name for i in assets) - arg = input(f"Enter keyword to reduce the list (partial match):\n{names}\n") - assets = [i for i in assets if arg in i.name] - if not assets: - input("No match, press enter to exit.") - return - asset = assets[0] + if not arg: + arg = to_filt[0] + elif arg.isdigit(): + arg = to_filt[int(arg)] + old = len(assets) + temp = [i for i in assets if arg == i.keywords[index]] + if temp: + assets = temp + else: + assets = [i for i in assets if arg in i.keywords[index]] + print( + f'[{get_time()}] Filt with keyword: "{arg}".', + old, + "=>", + len(assets), + flush=True, + ) + while len(assets) > 1: + names = "\n".join(i.name for i in assets) + arg = input(f"Enter keyword to reduce the list (partial match):\n{names}\n") + assets = [i for i in assets if arg in i.name] + if not assets: + input("No match, press enter to exit.") + return + asset = assets[0] + else: + asset = choose_asset(assets, keywords, args.auto) download_url = asset.url total_size = asset.size print( @@ -154,25 +255,21 @@ def sort_key(s): "MB", ) print(download_url, flush=True) - target = ( - input( - f"File path to save(defaults to `./{asset.name}`)?\nor `q` to exit.\n" - ).lower() - or asset.name - ) - if target == "q": - return - target_path = Path(target) - try: - target_path.unlink() - except FileNotFoundError: - pass + target = Path(args.target).resolve() + target.mkdir(parents=True, exist_ok=True) + target_path = target / asset.name + temp_path = target_path.with_name("python_downloading.tmp") + for _path in [target_path, temp_path]: + if _path.is_file(): + _path.unlink() print(f"[{get_time()}] Start downloading...") print(download_url) - print(target_path.absolute(), flush=True) + print(target_path.resolve(), flush=True) records = deque(maxlen=1000) + last_print = time.time() def reporthook(blocknum, blocksize, totalsize): + nonlocal last_print if totalsize < 0: totalsize = total_size _done = blocknum * blocksize @@ -206,15 +303,17 @@ def reporthook(blocknum, blocksize, totalsize): speed = f"{round(_speed / 1024, 1)} KB/s" else: speed = f"{round(_speed, 1)} B/s" - print( - f"[{get_time()}] Downloading: {done:.2f} / {total:.2f} MB | {percent:.2f}% | {speed} | {timeleft} {' ' * 10}", - end="\r", - flush=True, - ) + if time.time() - last_print > 1: + print( + f"[{get_time()}] Downloading: {done:.2f} / {total:.2f} MB | {percent:.2f}% | {speed} | {timeleft} {' ' * 10}", + end="\r", + flush=True, + ) + last_print = time.time() - temp_path = target_path.with_suffix(".tmp") for _ in range(3): try: + last_print = time.time() urllib.request.urlretrieve( download_url, temp_path.absolute().as_posix(), reporthook=reporthook ) @@ -242,8 +341,17 @@ def reporthook(blocknum, blocksize, totalsize): except FileNotFoundError: pass break - print("Press enter to exit.", flush=True) - input() + if args.interactive: + print("Press enter to exit.", flush=True) + return input() + if args.unzip and target_path.name.endswith(".tar.gz") and target_path.is_file(): + target_dir = target_path.with_name(target_path.stem.split("+", 1)[0]) + if target_dir.is_dir(): + shutil.rmtree(target_dir.as_posix()) + print(get_time(), f"start unzip file to {target_dir.as_posix()}", flush=True) + with tarfile.open(target_path.as_posix(), "r:gz") as tar: + tar.extractall(path=target_dir.as_posix()) + print(get_time(), "bye~") if __name__ == "__main__": diff --git a/zipapps/ensure_zipapps.py.template b/zipapps/ensure_zipapps.py.template index ff83838..3dee2d1 100644 --- a/zipapps/ensure_zipapps.py.template +++ b/zipapps/ensure_zipapps.py.template @@ -211,10 +211,10 @@ def prepare_path(): if ignore_system_python_path: sys.path.clear() # env of Popen is not valid for win32 platform, use os.environ instead. - _new_paths = _zipapps_python_path_list + new_sys_paths + _new_paths = new_sys_paths + _zipapps_python_path_list else: _old_path = os.environ.get('PYTHONPATH') or '' - _new_paths = _zipapps_python_path_list + [_old_path] + new_sys_paths + _new_paths = new_sys_paths + _zipapps_python_path_list + [_old_path] os.environ['PYTHONPATH'] = os.pathsep.join(_new_paths) # let the dir path first zipapps_paths = [ @@ -222,7 +222,7 @@ def prepare_path(): ] seen_path = set() result = [] - for path in zipapps_paths + sys.path + new_sys_paths: + for path in new_sys_paths + zipapps_paths + sys.path: if path not in seen_path: seen_path.add(path) result.append(path) diff --git a/zipapps/main.py b/zipapps/main.py index fa7cae9..81a17b6 100644 --- a/zipapps/main.py +++ b/zipapps/main.py @@ -15,7 +15,7 @@ from pkgutil import get_data from zipfile import ZIP_DEFLATED, ZIP_STORED, BadZipFile, ZipFile -__version__ = "2024.06.04" +__version__ = "2024.08.07" def get_pip_main(ensurepip_root=None): @@ -137,7 +137,7 @@ def __init__( :type env_paths: str, optional :param lazy_install: Install packages with pip while running, which means requirements will not be install into pyz file, defaults to False :type lazy_install: bool, optional - :param sys_paths: Paths be insert to sys.path[-1] while running. Support $TEMP/$HOME/$SELF/$PID/$CWD prefix, separated by commas, defaults to '' + :param sys_paths: Paths be insert to sys.path[0] while running. Support $TEMP/$HOME/$SELF/$PID/$CWD prefix, separated by commas, defaults to '' :type sys_paths: str, optional :param python_version_slice: Only work for lazy-install mode, then `pip` target folders differ according to sys.version_info[:_slice], defaults to 2, which means 3.8.3 equals to 3.8.4 for same version accuracy 3.8, defaults to 2 :type python_version_slice: int, optional