From 88636d8c207bb349f55c8edb7b4fddb16dea0de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 20 Dec 2024 15:48:35 +0100 Subject: [PATCH 1/5] Add skip-files-fetch option It's useful to fetch just git repos, but not distfiles. It's useful when fetching all repositories, but fetching actual files only when building. Especially saves a lot of space from kernel tarballs. And also, useful when using builder just to verify sources. QubesOS/qubes-issues#9662 --- README.md | 2 ++ qubesbuilder/plugins/fetch/__init__.py | 19 ++++++----- tests/test_cli.py | 44 ++++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 0794776e..95356d12 100644 --- a/README.md +++ b/README.md @@ -687,6 +687,8 @@ Options available in `builder.yml`: - `skip-git-fetch: bool` --- When set, do not update already downloaded git repositories (those in `sources` artifacts dir). New components are still fetched (once). Useful when doing development builds from non-default branches, local modifications etc. +- `skip-files-fetch: bool` --- When set, do not fetch component files like source tarballs (those in the `distfiles` artifacts dir). Component builds *will fail* without those files. Useful to save time and space when fetching git repositories. + - `increment-devel-versions: bool` --- When set, each package built will have local build number appended to package release number. This way, it's easy to update test environment without manually forcing reinstall of packages with unchanged versions. Example versions with devel build number: - `qubes-core-dom0-4.2.12-1.13.fc37.x86_64` (`.13` is the additional version here) diff --git a/qubesbuilder/plugins/fetch/__init__.py b/qubesbuilder/plugins/fetch/__init__.py index 3d70343e..00b8247f 100644 --- a/qubesbuilder/plugins/fetch/__init__.py +++ b/qubesbuilder/plugins/fetch/__init__.py @@ -194,14 +194,17 @@ def run(self, stage: str): distfiles_dir.mkdir(parents=True, exist_ok=True) # Download and verify files given in .qubesbuilder - for file in parameters.get("files", []): - if "url" in file: - self.download_file(file, executor, distfiles_dir) - elif "git-url" in file: - self.download_git_archive(file, executor, distfiles_dir) - else: - msg = "'files' entries must have either url or git-url entry" - raise FetchError(msg) + if not self.config.get("skip-files-fetch", False): + for file in parameters.get("files", []): + if "url" in file: + self.download_file(file, executor, distfiles_dir) + elif "git-url" in file: + self.download_git_archive(file, executor, distfiles_dir) + else: + msg = ( + "'files' entries must have either url or git-url entry" + ) + raise FetchError(msg) # # source hash and version tags determination diff --git a/tests/test_cli.py b/tests/test_cli.py index d950965d..3fbf4531 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,8 +19,7 @@ HASH_RE = re.compile(r"[a-f0-9]{40}") -@pytest.fixture(scope="session") -def artifacts_dir(): +def _artifacts_dir(): if os.environ.get("BASE_ARTIFACTS_DIR"): tmpdir = tempfile.mktemp( prefix="github-", dir=os.environ.get("BASE_ARTIFACTS_DIR") @@ -33,6 +32,16 @@ def artifacts_dir(): yield artifacts_dir +@pytest.fixture +def artifacts_dir_single(): + yield from _artifacts_dir() + + +@pytest.fixture(scope="session") +def artifacts_dir(): + yield from _artifacts_dir() + + def qb_call(builder_conf, artifacts_dir, *args, **kwargs): cmd = [ str(PROJECT_PATH / "qb"), @@ -316,6 +325,37 @@ def test_common_component_fetch_inplace_updating(artifacts_dir): ) +def test_common_component_fetch_skip_files(artifacts_dir_single): + artifacts_dir = artifacts_dir_single + + result = qb_call_output( + DEFAULT_BUILDER_CONF, + artifacts_dir, + "--option", + "skip-files-fetch=true", + "package", + "fetch", + ).decode() + + infos = _get_infos(artifacts_dir) + + for component in [ + "core-qrexec", + "core-vchan-xen", + "desktop-linux-xfce4-xfwm4", + "python-qasync", + "app-linux-split-gpg", + ]: + assert ( + artifacts_dir / "sources" / component / ".qubesbuilder" + ).exists() + assert not list((artifacts_dir / "distfiles" / component).iterdir()) + assert ( + "Enough distinct tag signatures. Found 3, mandatory minimum is 3." + in result + ) + + # # Pipeline for core-qrexec and host-fc37 # From 93d397d0c003a52ffcb402d47a6d23bd86e41cc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 20 Dec 2024 21:13:59 +0100 Subject: [PATCH 2/5] get-and-verify-source: fix handling non-standard PATH Do not override the whole env, just add/replace GNUPGHOME env. This is especially for subprocess to find git or gpg using PATH variable. And also use /usr/bin/env python3 in shebang (similarly to the main qubesbuilder-cli script), do not assume /usr/bin/python3. --- .../plugins/fetch/scripts/get-and-verify-source.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py b/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py index af8f2a9d..ec23c49c 100755 --- a/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py +++ b/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # # The Qubes OS Project, http://www.qubes-os.org # @@ -44,12 +44,14 @@ def verify_git_obj(gpg_client, keyring_dir, repository_dir, obj_type, obj_path): obj_path, ] + env = os.environ.copy() + env["GNUPGHOME"] = str(keyring_dir) output = subprocess.run( command, capture_output=True, universal_newlines=True, cwd=repository_dir, - env={"GNUPGHOME": str(keyring_dir)}, + env=env, check=True, ).stderr @@ -268,7 +270,8 @@ def main(args): else: print("--> Verifying tags...") - env = {"GNUPGHOME": str(git_keyring_dir)} + env = os.environ.copy() + env["GNUPGHOME"] = str(git_keyring_dir) git_keyring_dir.mkdir(parents=True, exist_ok=True) git_keyring_dir.chmod(0o700) # We request a list to init the keyring. It looks like it does From 41fa9e883f2a20ebd97315af08e71c2001aab41e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 20 Dec 2024 22:00:35 +0100 Subject: [PATCH 3/5] executors/local: try cleanup as normal user first If there is no need to use sudo (most of the cases), don't try it. This avoids polluting logs if sudo is not allowed, or even not installed. --- qubesbuilder/executors/local.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/qubesbuilder/executors/local.py b/qubesbuilder/executors/local.py index 90a828e3..d8814d9f 100644 --- a/qubesbuilder/executors/local.py +++ b/qubesbuilder/executors/local.py @@ -82,29 +82,29 @@ def copy_out(self, source_path: Path, destination_dir: Path): # type: ignore def cleanup(self): try: - subprocess.run( - [ - "sudo", - "--non-interactive", - "rm", - "-rf", - "--", - self._temporary_dir, - ], - check=True, - ) - except subprocess.CalledProcessError as e: + shutil.rmtree(self._temporary_dir) + except PermissionError: + # retry with sudo try: - # retry without sudo, as local executor for many - # actions doesn't really need it subprocess.run( - ["rm", "-rf", "--", self._temporary_dir], + [ + "sudo", + "--non-interactive", + "rm", + "-rf", + "--", + self._temporary_dir, + ], check=True, ) except subprocess.CalledProcessError as e: raise ExecutorError( f"Failed to clean executor temporary directory: {str(e)}" ) + except OSError as e: + raise ExecutorError( + f"Failed to clean executor temporary directory: {str(e)}" + ) def run( # type: ignore self, From 11e1eec172a3f6460123197a448c11e04f9296c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Fri, 20 Dec 2024 23:48:33 +0100 Subject: [PATCH 4/5] Fix handling "branch" set to a commit hash There is --git-commit option for get-and-verify-source.py already. Use its implementation, but keep verifying tags. For that, they need to be fetched. --- .../fetch/scripts/get-and-verify-source.py | 4 +++- tests/test_cli.py | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py b/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py index ec23c49c..a194b30f 100755 --- a/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py +++ b/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py @@ -183,7 +183,8 @@ def main(args): if repo.exists(): shutil.rmtree(repo) try: - if args.git_commit: + looks_like_commit = re.match(r"^[a-fA-F0-9]{40}$", git_branch) + if args.git_commit or looks_like_commit: # git clone can't handle commit reference, use fetch instead repo.mkdir() subprocess.run( @@ -194,6 +195,7 @@ def main(args): ) subprocess.run( ["git", "fetch"] + + (["--tags"] if looks_like_commit else []) + git_options + ["--", git_url, git_branch], capture_output=True, diff --git a/tests/test_cli.py b/tests/test_cli.py index 3fbf4531..d8fb767f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -355,6 +355,29 @@ def test_common_component_fetch_skip_files(artifacts_dir_single): in result ) +def test_common_component_fetch_commit_fresh(artifacts_dir_single): + artifacts_dir = artifacts_dir_single + commit_sha = "0589ae8a242b3be6a1b8985c6eb8900e5236152a" + result = qb_call_output( + DEFAULT_BUILDER_CONF, + artifacts_dir, + "-c", + "core-qrexec", + "-o", + f"+components+core-qrexec:branch={commit_sha}", + "package", + "fetch", + ).decode() + + fetch_artifact = ( + artifacts_dir / + "components/core-qrexec/4.2.20-1/nodist/fetch/source.fetch.yml" + ) + assert fetch_artifact.exists() + with open(fetch_artifact) as f: + info = yaml.safe_load(f.read()) + assert info["git-commit-hash"] == commit_sha + # # Pipeline for core-qrexec and host-fc37 From a8d4ec80a2a9fb6c33374c37b4e003e463beaeba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?= Date: Sun, 22 Dec 2024 02:41:47 +0100 Subject: [PATCH 5/5] ci: enable cache jobs on regular "main" pipelines Scheduled pipelines on the "main" branch have cache jobs enabled, and those jobs are used by more or less all pipelines in other repos for quicker run. Do this for ordinary "main" pipelines too, otherwise all jobs won't use a cache for a week after merging anything to the builderv2 repo. --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 806440c8..76c5b207 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,8 @@ variables: description: "Run components jobs, default: 0" CI_RUN_TEMPLATE_JOBS: description: "Run template jobs, default: 0" + CI_RUN_CACHE_JOBS: + description: "Run cache jobs, subset of CI_RUN_COMPONENT_JOBS, default: 0" CI_RUN_ISO_JOBS: description: "Run ISO jobs, default: 0" CI_RUN_INFRA_JOBS: @@ -75,7 +77,7 @@ builderv2-github: .init_cache_job: rules: - - if: $CI_RUN_CACHE_JOBS == "1" || $CI_RUN_COMPONENT_JOBS == "1" || $CI_RUN_ALL_JOBS == "1" + - if: $CI_RUN_CACHE_JOBS == "1" || $CI_RUN_COMPONENT_JOBS == "1" || $CI_RUN_ALL_JOBS == "1" || $CI_COMMIT_BRANCH == "main" when: always - when: never stage: prep