Merge remote-tracking branch 'origin/pr/166'

* origin/pr/166:
  ci: enable cache jobs on regular "main" pipelines
  Fix handling "branch" set to a commit hash
  executors/local: try cleanup as normal user first
  get-and-verify-source: fix handling non-standard PATH
  Add skip-files-fetch option

QubesOS/qubes-issues#9662

.gitlab-ci.yml

@@ -20,6 +20,8 @@ variables:
     description: "Run components jobs, default: 0"
   CI_RUN_TEMPLATE_JOBS:
     description: "Run template jobs, default: 0"
+  CI_RUN_CACHE_JOBS:
+    description: "Run cache jobs, subset of CI_RUN_COMPONENT_JOBS, default: 0"
   CI_RUN_ISO_JOBS:
     description: "Run ISO jobs, default: 0"
   CI_RUN_INFRA_JOBS:
@@ -75,7 +77,7 @@ builderv2-github:
 
 .init_cache_job:
   rules:
-    - if: $CI_RUN_CACHE_JOBS == "1" || $CI_RUN_COMPONENT_JOBS == "1" || $CI_RUN_ALL_JOBS == "1"
+    - if: $CI_RUN_CACHE_JOBS == "1" || $CI_RUN_COMPONENT_JOBS == "1" || $CI_RUN_ALL_JOBS == "1" || $CI_COMMIT_BRANCH == "main"
       when: always
     - when: never
   stage: prep

README.md

@@ -687,6 +687,8 @@ Options available in `builder.yml`:
 
 - `skip-git-fetch: bool` --- When set, do not update already downloaded git repositories (those in `sources` artifacts dir). New components are still fetched (once). Useful when doing development builds from non-default branches, local modifications etc.
 
+- `skip-files-fetch: bool` --- When set, do not fetch component files like source tarballs (those in the `distfiles` artifacts dir). Component builds *will fail* without those files. Useful to save time and space when fetching git repositories.
+
 - `increment-devel-versions: bool`  --- When set, each package built will have local build number appended to package release number. This way, it's easy to update test environment without manually forcing reinstall of packages with unchanged versions. Example versions with devel build number:
 
   - `qubes-core-dom0-4.2.12-1.13.fc37.x86_64` (`.13` is the additional version here)

qubesbuilder/executors/local.py

@@ -82,29 +82,29 @@ def copy_out(self, source_path: Path, destination_dir: Path):  # type: ignore
 
     def cleanup(self):
         try:
-            subprocess.run(
-                [
-                    "sudo",
-                    "--non-interactive",
-                    "rm",
-                    "-rf",
-                    "--",
-                    self._temporary_dir,
-                ],
-                check=True,
-            )
-        except subprocess.CalledProcessError as e:
+            shutil.rmtree(self._temporary_dir)
+        except PermissionError:
+            # retry with sudo
             try:
-                # retry without sudo, as local executor for many
-                # actions doesn't really need it
                 subprocess.run(
-                    ["rm", "-rf", "--", self._temporary_dir],
+                    [
+                        "sudo",
+                        "--non-interactive",
+                        "rm",
+                        "-rf",
+                        "--",
+                        self._temporary_dir,
+                    ],
                     check=True,
                 )
             except subprocess.CalledProcessError as e:
                 raise ExecutorError(
                     f"Failed to clean executor temporary directory: {str(e)}"
                 )
+        except OSError as e:
+            raise ExecutorError(
+                f"Failed to clean executor temporary directory: {str(e)}"
+            )
 
     def run(  # type: ignore
         self,

qubesbuilder/plugins/fetch/__init__.py

@@ -194,14 +194,17 @@ def run(self, stage: str):
         distfiles_dir.mkdir(parents=True, exist_ok=True)
 
         # Download and verify files given in .qubesbuilder
-        for file in parameters.get("files", []):
-            if "url" in file:
-                self.download_file(file, executor, distfiles_dir)
-            elif "git-url" in file:
-                self.download_git_archive(file, executor, distfiles_dir)
-            else:
-                msg = "'files' entries must have either url or git-url entry"
-                raise FetchError(msg)
+        if not self.config.get("skip-files-fetch", False):
+            for file in parameters.get("files", []):
+                if "url" in file:
+                    self.download_file(file, executor, distfiles_dir)
+                elif "git-url" in file:
+                    self.download_git_archive(file, executor, distfiles_dir)
+                else:
+                    msg = (
+                        "'files' entries must have either url or git-url entry"
+                    )
+                    raise FetchError(msg)
 
         #
         # source hash and version tags determination

qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 #
 # The Qubes OS Project, http://www.qubes-os.org
 #
@@ -44,12 +44,14 @@ def verify_git_obj(gpg_client, keyring_dir, repository_dir, obj_type, obj_path):
             obj_path,
         ]
 
+        env = os.environ.copy()
+        env["GNUPGHOME"] = str(keyring_dir)
         output = subprocess.run(
             command,
             capture_output=True,
             universal_newlines=True,
             cwd=repository_dir,
-            env={"GNUPGHOME": str(keyring_dir)},
+            env=env,
             check=True,
         ).stderr
 
@@ -181,7 +183,8 @@ def main(args):
         if repo.exists():
             shutil.rmtree(repo)
         try:
-            if args.git_commit:
+            looks_like_commit = re.match(r"^[a-fA-F0-9]{40}$", git_branch)
+            if args.git_commit or looks_like_commit:
                 # git clone can't handle commit reference, use fetch instead
                 repo.mkdir()
                 subprocess.run(
@@ -192,6 +195,7 @@ def main(args):
                 )
                 subprocess.run(
                     ["git", "fetch"]
+                    + (["--tags"] if looks_like_commit else [])
                     + git_options
                     + ["--", git_url, git_branch],
                     capture_output=True,
@@ -268,7 +272,8 @@ def main(args):
         else:
             print("--> Verifying tags...")
 
-        env = {"GNUPGHOME": str(git_keyring_dir)}
+        env = os.environ.copy()
+        env["GNUPGHOME"] = str(git_keyring_dir)
         git_keyring_dir.mkdir(parents=True, exist_ok=True)
         git_keyring_dir.chmod(0o700)
         # We request a list to init the keyring. It looks like it does

tests/test_cli.py

@@ -19,8 +19,7 @@
 HASH_RE = re.compile(r"[a-f0-9]{40}")
 
 
-@pytest.fixture(scope="session")
-def artifacts_dir():
+def _artifacts_dir():
     if os.environ.get("BASE_ARTIFACTS_DIR"):
         tmpdir = tempfile.mktemp(
             prefix="github-", dir=os.environ.get("BASE_ARTIFACTS_DIR")
@@ -33,6 +32,16 @@ def artifacts_dir():
     yield artifacts_dir
 
 
+@pytest.fixture
+def artifacts_dir_single():
+    yield from _artifacts_dir()
+
+
+@pytest.fixture(scope="session")
+def artifacts_dir():
+    yield from _artifacts_dir()
+
+
 def qb_call(builder_conf, artifacts_dir, *args, **kwargs):
     cmd = [
         str(PROJECT_PATH / "qb"),
@@ -316,6 +325,60 @@ def test_common_component_fetch_inplace_updating(artifacts_dir):
         )
 
 
+def test_common_component_fetch_skip_files(artifacts_dir_single):
+    artifacts_dir = artifacts_dir_single
+
+    result = qb_call_output(
+        DEFAULT_BUILDER_CONF,
+        artifacts_dir,
+        "--option",
+        "skip-files-fetch=true",
+        "package",
+        "fetch",
+    ).decode()
+
+    infos = _get_infos(artifacts_dir)
+
+    for component in [
+        "core-qrexec",
+        "core-vchan-xen",
+        "desktop-linux-xfce4-xfwm4",
+        "python-qasync",
+        "app-linux-split-gpg",
+    ]:
+        assert (
+            artifacts_dir / "sources" / component / ".qubesbuilder"
+        ).exists()
+        assert not list((artifacts_dir / "distfiles" / component).iterdir())
+    assert (
+        "Enough distinct tag signatures. Found 3, mandatory minimum is 3."
+        in result
+    )
+
+def test_common_component_fetch_commit_fresh(artifacts_dir_single):
+    artifacts_dir = artifacts_dir_single
+    commit_sha = "0589ae8a242b3be6a1b8985c6eb8900e5236152a"
+    result = qb_call_output(
+        DEFAULT_BUILDER_CONF,
+        artifacts_dir,
+        "-c",
+        "core-qrexec",
+        "-o",
+        f"+components+core-qrexec:branch={commit_sha}",
+        "package",
+        "fetch",
+    ).decode()
+
+    fetch_artifact = (
+        artifacts_dir /
+        "components/core-qrexec/4.2.20-1/nodist/fetch/source.fetch.yml"
+    )
+    assert fetch_artifact.exists()
+    with open(fetch_artifact) as f:
+        info = yaml.safe_load(f.read())
+    assert info["git-commit-hash"] == commit_sha
+
+
 #
 # Pipeline for core-qrexec and host-fc37
 #