From 6662e549206356cfad661ff7fb9c43fd0ab565fe Mon Sep 17 00:00:00 2001 From: no92 Date: Mon, 3 Jun 2024 20:18:36 +0200 Subject: [PATCH] xbstrap: allow for specifying multiple git URLs as fallbacks --- xbstrap/schema.yml | 7 ++- xbstrap/vcs_utils.py | 116 +++++++++++++++++++++++++++---------------- 2 files changed, 80 insertions(+), 43 deletions(-) diff --git a/xbstrap/schema.yml b/xbstrap/schema.yml index fe756a2..27b2d88 100644 --- a/xbstrap/schema.yml +++ b/xbstrap/schema.yml @@ -82,7 +82,12 @@ definitions: type: string # VCS sources. 'git': - type: string + oneOf: + - type: array + items: + type: string + minItems: 1 + - type: string 'hg': type: string 'svn': diff --git a/xbstrap/vcs_utils.py b/xbstrap/vcs_utils.py index 4287297..b740664 100644 --- a/xbstrap/vcs_utils.py +++ b/xbstrap/vcs_utils.py @@ -195,10 +195,16 @@ def fetch_repo(cfg, src, subdir, *, ignore_mirror=False, bare_repo=False): xbstrap_mirror = None else: xbstrap_mirror = src.cfg.xbstrap_mirror + if xbstrap_mirror is None: - git_url = src._this_yml["git"] + if isinstance(source["git"], list): + if len(source["git"]) == 0: + raise GenericError("at least one URL is required for git sources") + git_sources = source["git"] + else: + git_sources = [source["git"]] else: - git_url = urllib.parse.urljoin(xbstrap_mirror + "/git/", src.name) + git_sources = [urllib.parse.urljoin(xbstrap_mirror + "/git/", src.name)] git = shutil.which("git") if git is None: @@ -223,7 +229,12 @@ def fetch_repo(cfg, src, subdir, *, ignore_mirror=False, bare_repo=False): else: subprocess.check_call([git, "init"] + b_args, cwd=source_dir) # We always set the remote to the true remote, not a mirror. - subprocess.check_call([git, "remote", "add", "origin", source["git"]], cwd=source_dir) + subprocess.check_call([git, "remote", "add", "origin", git_sources[0]], cwd=source_dir) + + for num, fallback in enumerate(git_sources[1:]): + subprocess.check_call( + [git, "remote", "add", f"fallback-{num}", fallback], cwd=source_dir + ) shallow = not source.get("disable_shallow_fetch", False) # We have to disable shallow fetches to get rolling versions right. @@ -234,46 +245,67 @@ def fetch_repo(cfg, src, subdir, *, ignore_mirror=False, bare_repo=False): if bare_repo: shallow = False - args = [git, "fetch"] - if "tag" in source: - if shallow: - args.append("--depth=1") - args.extend([git_url, "tag", source["tag"]]) - else: - # If a commit is specified, we need the branch's full history. - # TODO: it's unclear whether this is the best strategy: - # - for simplicity, it might be easier to always pull the full history - # - some remotes support fetching individual SHA1s. - if "commit" in source or fixed_commit is not None: - shallow = False - - # When initializing the repository, we fetch only one commit. - # For updates, we fetch all *new* commits (= default behavior of 'git fetch'). - # We do not unshallow the repository. - if init and shallow: - args.append("--depth=1") - - # For bare repos, we mirror the original repo - # (in particular, we do not distinguish local and remote branches). - if bare_repo: - args.extend( - [ - git_url, - "refs/heads/" + source["branch"] + ":" + "refs/heads/" + source["branch"], - ] - ) + fetch_succeeded = False + fetch_failed_before = False + + for git_url in git_sources: + args = [git, "fetch"] + if "tag" in source: + if shallow: + args.append("--depth=1") + args.extend([git_url, "tag", source["tag"]]) else: - args.extend( - [ - git_url, - "refs/heads/" - + source["branch"] - + ":" - + "refs/remotes/origin/" - + source["branch"], - ] - ) - subprocess.check_call(args, cwd=source_dir) + # If a commit is specified, we need the branch's full history. + # TODO: it's unclear whether this is the best strategy: + # - for simplicity, it might be easier to always pull the full history + # - some remotes support fetching individual SHA1s. + if "commit" in source or fixed_commit is not None: + shallow = False + + # When initializing the repository, we fetch only one commit. + # For updates, we fetch all *new* commits (= default behavior of 'git fetch'). + # We do not unshallow the repository. + if init and shallow: + args.append("--depth=1") + + # For bare repos, we mirror the original repo + # (in particular, we do not distinguish local and remote branches). + if bare_repo: + args.extend( + [ + git_url, + "refs/heads/" + + source["branch"] + + ":" + + "refs/heads/" + + source["branch"], + ] + ) + else: + args.extend( + [ + git_url, + "refs/heads/" + + source["branch"] + + ":" + + "refs/remotes/origin/" + + source["branch"], + ] + ) + + try: + subprocess.check_call(args, cwd=source_dir) + except subprocess.SubprocessError: + _util.log_warn(f'Fetching from git remote "{git_url}" failed') + fetch_failed_before = True + else: + if fetch_failed_before: + _util.log_warn(f'Fetching from fallback git remote "{git_url}" succeeded') + fetch_succeeded = True + break + + if not fetch_succeeded: + raise GenericError("Fetching {} failed".format(src.name)) elif "hg" in source: source_dir = os.path.join(subdir, src.name)