From 6f4ead90e72c21452e3964311b7373a6088a725e Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 16 May 2024 19:07:55 +0200 Subject: [PATCH] Implement `--base-url` CLI flag (#9) * add base_url argument * fix repodata schema * fix header * fix readme * add test --- README.md | 2 +- action.yml | 7 +++++++ conda_subchannel/cli.py | 11 ++++++++++- conda_subchannel/core.py | 33 ++++++++++++++++++++++++--------- tests/test_subchannel.py | 21 +++++++++++++++++++++ 5 files changed, 63 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 6b64f65..dd18846 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Create subsets of conda channels thanks to CEP-15 metadata ```bash $ conda install -n base conda-subchannel -$ conda subchannel --channel=conda-forge python=3.9 +$ conda subchannel --channel=conda-forge --keep-tree python=3.9 $ python -m http.serve --directory subchannel/ ``` diff --git a/action.yml b/action.yml index 6e06159..d6601e7 100644 --- a/action.yml +++ b/action.yml @@ -8,6 +8,10 @@ inputs: description: "Source repodata file to process from channel" required: false default: "repodata.json" + base-url: + description: "URL where the packages will be available. Defaults to the base URL for 'channel'" + required: false + default: "" subdirs: description: "List of platforms to support, space separated. Defaults to linux-64. Noarch is always included" required: false @@ -91,6 +95,9 @@ runs: "--repodata-fn", """${{ inputs.repodata-fn }}""".strip(), ] + base_url = """${{ inputs.base-url }}""".strip() + if base_url: + args += ["--base_url", base_url] after = """${{ inputs.after }}""".strip() if after: args += ["--after", after] diff --git a/conda_subchannel/cli.py b/conda_subchannel/cli.py index 03a1a9c..e2c09a8 100644 --- a/conda_subchannel/cli.py +++ b/conda_subchannel/cli.py @@ -12,6 +12,7 @@ from conda.base.constants import REPODATA_FN from conda.base.context import context from conda.common.io import Spinner +from conda.models.channel import Channel from .core import _fetch_channel, _reduce_index, _dump_records, _write_to_disk @@ -47,6 +48,13 @@ def configure_parser(parser: argparse.ArgumentParser): default=REPODATA_FN, help="Source repodata file to process from channel.", ) + parser.add_argument( + "--base-url", + required=False, + help="URL where the packages will be available. " + "Defaults to the base URL for '--channel'. Only needed if the user wants to mirror " + "the required packages separately." + ) parser.add_argument( "--output", default="subchannel", @@ -123,7 +131,8 @@ def execute(args: argparse.Namespace) -> int: print(" - Reduced from", total_count, "to", filtered_count, "records") with Spinner(f"Writing output to {args.output}"): - repodatas = _dump_records(records, args.channel) + base_url = args.base_url or Channel(args.channel).base_url + repodatas = _dump_records(records, base_url) _write_to_disk(args.channel, repodatas, args.output) return 0 diff --git a/conda_subchannel/core.py b/conda_subchannel/core.py index b9352fa..0d37abd 100644 --- a/conda_subchannel/core.py +++ b/conda_subchannel/core.py @@ -144,23 +144,40 @@ def _reduce_index( def _dump_records( - records: dict[tuple[str, str], PackageRecord], source_channel: Channel | str + records: dict[tuple[str, str], PackageRecord], base_url: str ) -> dict[str, dict[str, Any]]: - source_channel = Channel(source_channel) repodatas = {} + record_keys = ( + "build", + "build_number", + "constrains", + "depends", + "license", + "md5", + "name", + "sha256", + "size", + "subdir", + "timestamp", + "version", + ) for (subdir, filename), record in records.items(): if subdir not in repodatas: repodatas[subdir] = { - "metadata": { - "repodata_version": 2, - "base_url": source_channel.base_url, + "repodata_version": 2, + "info": { + "base_url": base_url, "subdir": subdir, }, "packages": {}, "packages.conda": {}, + "removed": [], } key = "packages.conda" if record.fn.endswith(".conda") else "packages" - repodatas[record.subdir][key][filename] = record.dump() + dumped = record.dump() + repodatas[record.subdir][key][filename] = { + key: dumped[key] for key in record_keys if key in dumped + } return repodatas @@ -206,9 +223,7 @@ def _write_subdir_index_md(subdir_path: Path): lastmod = datetime.fromtimestamp(stat.st_mtime) sha256 = _checksum(path, "sha256") md5 = _checksum(path, "md5") - lines.append( - f"| [{path.name}]({path.name}) | {size} | {lastmod} | `{sha256}` | `{md5}` |" - ) + lines.append(f"| [{path.name}]({path.name}) | {size} | {lastmod} | `{sha256}` | `{md5}` |") lines.append("") lines.append(f"> Last modified on {datetime.now(tz=timezone.utc)}") (subdir_path / "index.md").write_text("\n".join(lines)) diff --git a/tests/test_subchannel.py b/tests/test_subchannel.py index 3759d45..5a8d94e 100644 --- a/tests/test_subchannel.py +++ b/tests/test_subchannel.py @@ -1,3 +1,4 @@ +import json import sys from datetime import datetime, timezone @@ -210,3 +211,23 @@ def test_between_dates(conda_cli, tmp_path): timestamp_2023 <= rec.timestamp <= timestamp_2024 for rec in sd.iter_records() ) assert tested + +def test_base_url(conda_cli, tmp_path): + base_url = "https://a-redefined-base.url" + out, err, rc = conda_cli( + "subchannel", + "-c", + "conda-forge", + "--keep", + "python=3.9", + "--base-url", + base_url, + "--output", + tmp_path, + ) + print(out) + print(err, file=sys.stderr) + assert rc == 0 + + data = json.loads((tmp_path / context.subdir / "repodata.json").read_text()) + assert data["info"]["base_url"] == base_url