diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 806440c8..76c5b207 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,8 @@ variables: description: "Run components jobs, default: 0" CI_RUN_TEMPLATE_JOBS: description: "Run template jobs, default: 0" + CI_RUN_CACHE_JOBS: + description: "Run cache jobs, subset of CI_RUN_COMPONENT_JOBS, default: 0" CI_RUN_ISO_JOBS: description: "Run ISO jobs, default: 0" CI_RUN_INFRA_JOBS: @@ -75,7 +77,7 @@ builderv2-github: .init_cache_job: rules: - - if: $CI_RUN_CACHE_JOBS == "1" || $CI_RUN_COMPONENT_JOBS == "1" || $CI_RUN_ALL_JOBS == "1" + - if: $CI_RUN_CACHE_JOBS == "1" || $CI_RUN_COMPONENT_JOBS == "1" || $CI_RUN_ALL_JOBS == "1" || $CI_COMMIT_BRANCH == "main" when: always - when: never stage: prep diff --git a/README.md b/README.md index 0794776e..95356d12 100644 --- a/README.md +++ b/README.md @@ -687,6 +687,8 @@ Options available in `builder.yml`: - `skip-git-fetch: bool` --- When set, do not update already downloaded git repositories (those in `sources` artifacts dir). New components are still fetched (once). Useful when doing development builds from non-default branches, local modifications etc. +- `skip-files-fetch: bool` --- When set, do not fetch component files like source tarballs (those in the `distfiles` artifacts dir). Component builds *will fail* without those files. Useful to save time and space when fetching git repositories. + - `increment-devel-versions: bool` --- When set, each package built will have local build number appended to package release number. This way, it's easy to update test environment without manually forcing reinstall of packages with unchanged versions. Example versions with devel build number: - `qubes-core-dom0-4.2.12-1.13.fc37.x86_64` (`.13` is the additional version here) diff --git a/qubesbuilder/executors/local.py b/qubesbuilder/executors/local.py index 90a828e3..d8814d9f 100644 --- a/qubesbuilder/executors/local.py +++ b/qubesbuilder/executors/local.py @@ -82,29 +82,29 @@ def copy_out(self, source_path: Path, destination_dir: Path): # type: ignore def cleanup(self): try: - subprocess.run( - [ - "sudo", - "--non-interactive", - "rm", - "-rf", - "--", - self._temporary_dir, - ], - check=True, - ) - except subprocess.CalledProcessError as e: + shutil.rmtree(self._temporary_dir) + except PermissionError: + # retry with sudo try: - # retry without sudo, as local executor for many - # actions doesn't really need it subprocess.run( - ["rm", "-rf", "--", self._temporary_dir], + [ + "sudo", + "--non-interactive", + "rm", + "-rf", + "--", + self._temporary_dir, + ], check=True, ) except subprocess.CalledProcessError as e: raise ExecutorError( f"Failed to clean executor temporary directory: {str(e)}" ) + except OSError as e: + raise ExecutorError( + f"Failed to clean executor temporary directory: {str(e)}" + ) def run( # type: ignore self, diff --git a/qubesbuilder/plugins/fetch/__init__.py b/qubesbuilder/plugins/fetch/__init__.py index 3d70343e..00b8247f 100644 --- a/qubesbuilder/plugins/fetch/__init__.py +++ b/qubesbuilder/plugins/fetch/__init__.py @@ -194,14 +194,17 @@ def run(self, stage: str): distfiles_dir.mkdir(parents=True, exist_ok=True) # Download and verify files given in .qubesbuilder - for file in parameters.get("files", []): - if "url" in file: - self.download_file(file, executor, distfiles_dir) - elif "git-url" in file: - self.download_git_archive(file, executor, distfiles_dir) - else: - msg = "'files' entries must have either url or git-url entry" - raise FetchError(msg) + if not self.config.get("skip-files-fetch", False): + for file in parameters.get("files", []): + if "url" in file: + self.download_file(file, executor, distfiles_dir) + elif "git-url" in file: + self.download_git_archive(file, executor, distfiles_dir) + else: + msg = ( + "'files' entries must have either url or git-url entry" + ) + raise FetchError(msg) # # source hash and version tags determination diff --git a/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py b/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py index af8f2a9d..a194b30f 100755 --- a/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py +++ b/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # # The Qubes OS Project, http://www.qubes-os.org # @@ -44,12 +44,14 @@ def verify_git_obj(gpg_client, keyring_dir, repository_dir, obj_type, obj_path): obj_path, ] + env = os.environ.copy() + env["GNUPGHOME"] = str(keyring_dir) output = subprocess.run( command, capture_output=True, universal_newlines=True, cwd=repository_dir, - env={"GNUPGHOME": str(keyring_dir)}, + env=env, check=True, ).stderr @@ -181,7 +183,8 @@ def main(args): if repo.exists(): shutil.rmtree(repo) try: - if args.git_commit: + looks_like_commit = re.match(r"^[a-fA-F0-9]{40}$", git_branch) + if args.git_commit or looks_like_commit: # git clone can't handle commit reference, use fetch instead repo.mkdir() subprocess.run( @@ -192,6 +195,7 @@ def main(args): ) subprocess.run( ["git", "fetch"] + + (["--tags"] if looks_like_commit else []) + git_options + ["--", git_url, git_branch], capture_output=True, @@ -268,7 +272,8 @@ def main(args): else: print("--> Verifying tags...") - env = {"GNUPGHOME": str(git_keyring_dir)} + env = os.environ.copy() + env["GNUPGHOME"] = str(git_keyring_dir) git_keyring_dir.mkdir(parents=True, exist_ok=True) git_keyring_dir.chmod(0o700) # We request a list to init the keyring. It looks like it does diff --git a/tests/test_cli.py b/tests/test_cli.py index d950965d..d8fb767f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,8 +19,7 @@ HASH_RE = re.compile(r"[a-f0-9]{40}") -@pytest.fixture(scope="session") -def artifacts_dir(): +def _artifacts_dir(): if os.environ.get("BASE_ARTIFACTS_DIR"): tmpdir = tempfile.mktemp( prefix="github-", dir=os.environ.get("BASE_ARTIFACTS_DIR") @@ -33,6 +32,16 @@ def artifacts_dir(): yield artifacts_dir +@pytest.fixture +def artifacts_dir_single(): + yield from _artifacts_dir() + + +@pytest.fixture(scope="session") +def artifacts_dir(): + yield from _artifacts_dir() + + def qb_call(builder_conf, artifacts_dir, *args, **kwargs): cmd = [ str(PROJECT_PATH / "qb"), @@ -316,6 +325,60 @@ def test_common_component_fetch_inplace_updating(artifacts_dir): ) +def test_common_component_fetch_skip_files(artifacts_dir_single): + artifacts_dir = artifacts_dir_single + + result = qb_call_output( + DEFAULT_BUILDER_CONF, + artifacts_dir, + "--option", + "skip-files-fetch=true", + "package", + "fetch", + ).decode() + + infos = _get_infos(artifacts_dir) + + for component in [ + "core-qrexec", + "core-vchan-xen", + "desktop-linux-xfce4-xfwm4", + "python-qasync", + "app-linux-split-gpg", + ]: + assert ( + artifacts_dir / "sources" / component / ".qubesbuilder" + ).exists() + assert not list((artifacts_dir / "distfiles" / component).iterdir()) + assert ( + "Enough distinct tag signatures. Found 3, mandatory minimum is 3." + in result + ) + +def test_common_component_fetch_commit_fresh(artifacts_dir_single): + artifacts_dir = artifacts_dir_single + commit_sha = "0589ae8a242b3be6a1b8985c6eb8900e5236152a" + result = qb_call_output( + DEFAULT_BUILDER_CONF, + artifacts_dir, + "-c", + "core-qrexec", + "-o", + f"+components+core-qrexec:branch={commit_sha}", + "package", + "fetch", + ).decode() + + fetch_artifact = ( + artifacts_dir / + "components/core-qrexec/4.2.20-1/nodist/fetch/source.fetch.yml" + ) + assert fetch_artifact.exists() + with open(fetch_artifact) as f: + info = yaml.safe_load(f.read()) + assert info["git-commit-hash"] == commit_sha + + # # Pipeline for core-qrexec and host-fc37 #