[CI] Cross-compile the mac builds from linux

.github/workflows/ci.yml

@@ -125,9 +125,25 @@ jobs:
           mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
           artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
 
+  macossdk:
+    if: ${{ fromJSON(needs.decision.outputs.matrix)['macossdk'] && !cancelled() }}
+    needs: [decision]
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.decision.outputs.matrix)['macossdk'] }}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/task
+        with:
+          name: ${{ matrix.task }}
+          mounts: ${{ toJSON(fromJSON(needs.decision.outputs.mounts)[matrix.task]) }}
+          artifacts: ${{ toJSON(fromJSON(needs.decision.outputs.artifacts)[matrix.task]) }}
+
   build:
     if: ${{ fromJSON(needs.decision.outputs.matrix)['build'] && !cancelled() }}
-    needs: [decision, docker]
+    needs: [decision, docker, macossdk]
     strategy:
       fail-fast: false
       matrix:

CI/docker.py

@@ -98,6 +98,7 @@ def sources_list(snapshot, sections):
                     [
                         "clang-19",
                         "lld-19",
+                        "llvm-19",
                         "git",
                         "make",
                         "patch",

CI/osx.py

@@ -5,7 +5,9 @@
 import hashlib
 
 from tasks import (
+    Task,
     TaskEnvironment,
+    Tool,
     bash_command,
 )
 
@@ -56,3 +58,30 @@ def prepare_params(self, params):
         env.setdefault("PIP_DISABLE_PIP_VERSION_CHECK", "1")
         params["command"].insert(0, "export PATH=$PATH:/opt/homebrew/bin")
         return super(OsxArm64, self).prepare_params(params)
+
+
+class MacosSDK(Task, metaclass=Tool):
+    PREFIX = "macossdk"
+    SDK_VERSION = "14.2"
+    XCODE_VERSION = "15.2"
+
+    def __init__(self, name):
+        dev = f"/Applications/Xcode_{self.XCODE_VERSION}.app/Contents/Developer"
+        sdkroot = f"{dev}/Platforms/MacOSX.platform/Developer/SDKs/MacOSX{self.SDK_VERSION}.sdk"
+        Task.__init__(
+            self,
+            description=f"macossdk {self.SDK_VERSION}",
+            task_env=TaskEnvironment.by_name("arm64-osx.build"),
+            command=[
+                f"cp -RH {sdkroot} MacOSX{self.SDK_VERSION}.sdk",
+                f"gtar --zstd -cf MacOSX{self.SDK_VERSION}.sdk.tar.zst MacOSX{self.SDK_VERSION}.sdk",
+            ],
+            artifact=f"MacOSX{self.SDK_VERSION}.sdk.tar.zst",
+            index=f"macossdk.{self.SDK_VERSION}",
+        )
+
+    def mount(self):
+        return {f"directory:MacOSX{self.SDK_VERSION}.sdk": self}
+
+    def install(self):
+        return [f"export SDKROOT=$(realpath $PWD/MacOSX{self.SDK_VERSION}.sdk)"]

CI/tools.py

@@ -358,10 +358,7 @@ def __init__(self, os_and_variant):
         env = TaskEnvironment.by_name(
             "{}.build".format(os.replace("arm64-linux", "linux"))
         )
-        if os.startswith("mingw"):
-            build_env = TaskEnvironment.by_name("linux.build")
-        else:
-            build_env = env
+        build_env = TaskEnvironment.by_name("linux.build")
 
         artifact = "git-cinnabar"
         if os.startswith("mingw"):
@@ -427,8 +424,7 @@ def prefix(p, s):
         elif variant:
             raise Exception("Unknown variant: {}".format(variant))
 
-        if "osx" not in os:
-            environ["CC"] = "clang-19"
+        environ["CC"] = "clang-19"
 
         if os.startswith("mingw"):
             cpu = msys.msys_cpu(env.cpu)
@@ -441,46 +437,45 @@ def prefix(p, s):
             rust_target = "x86_64-unknown-linux-gnu"
         elif os == "arm64-linux":
             rust_target = "aarch64-unknown-linux-gnu"
-        if "osx" not in os:
-            for target in dict.fromkeys(
-                ["x86_64-unknown-linux-gnu", rust_target]
-            ).keys():
-                arch = {
-                    "x86_64": "amd64",
-                    "aarch64": "arm64",
-                }[target.partition("-")[0]]
-                multiarch = target.replace("unknown-", "")
-                TARGET = target.replace("-", "_").upper()
-                environ[f"CARGO_TARGET_{TARGET}_LINKER"] = environ["CC"]
-                link_args = [
-                    f"--target={target}",
-                    "-fuse-ld=lld-19",
-                ]
-                if "linux" in os:
-                    link_args.append(f"--sysroot=/sysroot-{arch}")
-                if os.startswith("mingw"):
-                    link_args.append(f"-L/usr/lib/gcc/{cpu}-w64-mingw32/10-win32")
-                    link_args.append("-Wl,-Xlink,-Brepro")
-                environ[f"CARGO_TARGET_{TARGET}_RUSTFLAGS"] = " ".join(
-                    f"-C link-arg={arg}" for arg in link_args
-                )
-                rustflags = environ.pop("RUSTFLAGS", None)
-                if rustflags:
-                    environ[f"CARGO_TARGET_{TARGET}_RUSTFLAGS"] += f" {rustflags}"
-                if "linux" in os:
-                    environ[f"CFLAGS_{target.replace('-', '_')}"] = (
-                        f"--sysroot=/sysroot-{arch}"
-                    )
+
+        for target in dict.fromkeys(["x86_64-unknown-linux-gnu", rust_target]).keys():
+            arch = {
+                "x86_64": "amd64",
+                "aarch64": "arm64",
+            }[target.partition("-")[0]]
+            multiarch = target.replace("unknown-", "")
+            TARGET = target.replace("-", "_").upper()
+            environ[f"CARGO_TARGET_{TARGET}_LINKER"] = environ["CC"]
+            link_args = [
+                f"--target={target}",
+                "-fuse-ld=lld",
+            ]
             if "linux" in os:
-                environ["PKG_CONFIG_PATH"] = ""
-                environ["PKG_CONFIG_SYSROOT_DIR"] = f"/sysroot-{arch}"
-                environ["PKG_CONFIG_LIBDIR"] = ":".join(
-                    (
-                        f"/sysroot-{arch}/usr/lib/pkgconfig",
-                        f"/sysroot-{arch}/usr/lib/{multiarch}/pkgconfig",
-                        f"/sysroot-{arch}/usr/share/pkgconfig",
-                    )
+                link_args.append(f"--sysroot=/sysroot-{arch}")
+            if os.startswith("mingw"):
+                link_args.append(f"-L/usr/lib/gcc/{cpu}-w64-mingw32/10-win32")
+                link_args.append("-Wl,-Xlink,-Brepro")
+            environ[f"CARGO_TARGET_{TARGET}_RUSTFLAGS"] = " ".join(
+                f"-C link-arg={arg}" for arg in link_args
+            )
+            environ["AR"] = "llvm-ar-19"
+            rustflags = environ.pop("RUSTFLAGS", None)
+            if rustflags:
+                environ[f"CARGO_TARGET_{TARGET}_RUSTFLAGS"] += f" {rustflags}"
+            if "linux" in os:
+                environ[f"CFLAGS_{target.replace('-', '_')}"] = (
+                    f"--sysroot=/sysroot-{arch}"
+                )
+        if "linux" in os:
+            environ["PKG_CONFIG_PATH"] = ""
+            environ["PKG_CONFIG_SYSROOT_DIR"] = f"/sysroot-{arch}"
+            environ["PKG_CONFIG_LIBDIR"] = ":".join(
+                (
+                    f"/sysroot-{arch}/usr/lib/pkgconfig",
+                    f"/sysroot-{arch}/usr/lib/{multiarch}/pkgconfig",
+                    f"/sysroot-{arch}/usr/share/pkgconfig",
                 )
+            )
         if variant == "asan":
             environ["RUSTC_BOOTSTRAP"] = "1"
         if rust_version:
@@ -502,6 +497,12 @@ def prefix(p, s):
             environ.setdefault("MACOSX_DEPLOYMENT_TARGET", "10.7")
         if os.startswith("arm64-osx"):
             environ.setdefault("MACOSX_DEPLOYMENT_TARGET", "11.0")
+        sdk_install = []
+        kwargs = {}
+        if "osx" in os:
+            sdk = Tool.by_name("macossdk.arm64-osx")
+            sdk_install = sdk.install()
+            kwargs.setdefault("mounts", []).append(sdk.mount())
 
         cpu = "arm64" if os == "arm64-linux" else env.cpu
         Task.__init__(
@@ -510,6 +511,7 @@ def prefix(p, s):
             description="build {} {}{}".format(env.os, cpu, prefix(" ", desc_variant)),
             index="build.{}.{}.{}{}".format(hash, env.os, cpu, prefix(".", variant)),
             command=Task.checkout(commit=head)
+            + sdk_install
             + rust_install
             + [
                 "(cd repo ; CI/cargo.sh build {})".format(" ".join(cargo_flags)),
@@ -521,6 +523,7 @@ def prefix(p, s):
             ],
             artifacts=artifacts,
             env=environ,
+            **kwargs,
         )
 
     def mount(self):