Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow for specifying multiple git URLs as fallbacks #84

Merged
merged 1 commit into from
Sep 10, 2024
Merged

Allow for specifying multiple git URLs as fallbacks #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion xbstrap/schema.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,12 @@ definitions:
type: string
# VCS sources.
'git':
type: string
oneOf:
- type: array
items:
type: string
minItems: 1
- type: string
'hg':
type: string
'svn':
Expand Down
116 changes: 74 additions & 42 deletions xbstrap/vcs_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,10 +195,16 @@ def fetch_repo(cfg, src, subdir, *, ignore_mirror=False, bare_repo=False):
xbstrap_mirror = None
else:
xbstrap_mirror = src.cfg.xbstrap_mirror

if xbstrap_mirror is None:
git_url = src._this_yml["git"]
if isinstance(source["git"], list):
if len(source["git"]) == 0:
raise GenericError("at least one URL is required for git sources")
git_sources = source["git"]
else:
git_sources = [source["git"]]
else:
git_url = urllib.parse.urljoin(xbstrap_mirror + "/git/", src.name)
git_sources = [urllib.parse.urljoin(xbstrap_mirror + "/git/", src.name)]

git = shutil.which("git")
if git is None:
Expand All @@ -223,7 +229,12 @@ def fetch_repo(cfg, src, subdir, *, ignore_mirror=False, bare_repo=False):
else:
subprocess.check_call([git, "init"] + b_args, cwd=source_dir)
# We always set the remote to the true remote, not a mirror.
subprocess.check_call([git, "remote", "add", "origin", source["git"]], cwd=source_dir)
subprocess.check_call([git, "remote", "add", "origin", git_sources[0]], cwd=source_dir)

for num, fallback in enumerate(git_sources[1:]):
no92 marked this conversation as resolved.
Show resolved Hide resolved
subprocess.check_call(
[git, "remote", "add", f"fallback-{num}", fallback], cwd=source_dir
)

shallow = not source.get("disable_shallow_fetch", False)
# We have to disable shallow fetches to get rolling versions right.
Expand All @@ -234,46 +245,67 @@ def fetch_repo(cfg, src, subdir, *, ignore_mirror=False, bare_repo=False):
if bare_repo:
shallow = False

args = [git, "fetch"]
if "tag" in source:
if shallow:
args.append("--depth=1")
args.extend([git_url, "tag", source["tag"]])
else:
# If a commit is specified, we need the branch's full history.
# TODO: it's unclear whether this is the best strategy:
# - for simplicity, it might be easier to always pull the full history
# - some remotes support fetching individual SHA1s.
if "commit" in source or fixed_commit is not None:
shallow = False

# When initializing the repository, we fetch only one commit.
# For updates, we fetch all *new* commits (= default behavior of 'git fetch').
# We do not unshallow the repository.
if init and shallow:
args.append("--depth=1")

# For bare repos, we mirror the original repo
# (in particular, we do not distinguish local and remote branches).
if bare_repo:
args.extend(
[
git_url,
"refs/heads/" + source["branch"] + ":" + "refs/heads/" + source["branch"],
]
)
fetch_succeeded = False
fetch_failed_before = False

for git_url in git_sources:
args = [git, "fetch"]
if "tag" in source:
if shallow:
args.append("--depth=1")
args.extend([git_url, "tag", source["tag"]])
else:
args.extend(
[
git_url,
"refs/heads/"
+ source["branch"]
+ ":"
+ "refs/remotes/origin/"
+ source["branch"],
]
)
subprocess.check_call(args, cwd=source_dir)
# If a commit is specified, we need the branch's full history.
# TODO: it's unclear whether this is the best strategy:
# - for simplicity, it might be easier to always pull the full history
# - some remotes support fetching individual SHA1s.
if "commit" in source or fixed_commit is not None:
shallow = False

# When initializing the repository, we fetch only one commit.
# For updates, we fetch all *new* commits (= default behavior of 'git fetch').
# We do not unshallow the repository.
if init and shallow:
args.append("--depth=1")

# For bare repos, we mirror the original repo
# (in particular, we do not distinguish local and remote branches).
if bare_repo:
args.extend(
[
git_url,
"refs/heads/"
+ source["branch"]
+ ":"
+ "refs/heads/"
+ source["branch"],
]
)
else:
args.extend(
[
git_url,
"refs/heads/"
+ source["branch"]
+ ":"
+ "refs/remotes/origin/"
+ source["branch"],
]
)

try:
subprocess.check_call(args, cwd=source_dir)
except subprocess.SubprocessError:
_util.log_warn(f'Fetching from git remote "{git_url}" failed')
fetch_failed_before = True
else:
if fetch_failed_before:
_util.log_warn(f'Fetching from fallback git remote "{git_url}" succeeded')
fetch_succeeded = True
break

if not fetch_succeeded:
raise GenericError("Fetching {} failed".format(src.name))
elif "hg" in source:
source_dir = os.path.join(subdir, src.name)

Expand Down
Loading