Forward cherry pick the prebuilts feature to main.

Sources/Basics/Concurrency/SendableBox.swift

@@ -16,24 +16,24 @@ import struct Foundation.Date
 /// an `async` closure. This type serves as a replacement for `ThreadSafeBox`
 /// implemented with Swift Concurrency primitives.
 public actor SendableBox<Value: Sendable> {
-    init(_ value: Value? = nil) {
+    public init(_ value: Value) {
         self.value = value
     }
 
-    var value: Value?
+    public var value: Value
+
+    public func set(_ value: Value) {
+        self.value = value
+    }
 }
 
 extension SendableBox where Value == Int {
     func increment() {
-        if let value {
-            self.value = value + 1
-        }
+        self.value = value + 1
     }
 
     func decrement() {
-        if let value {
-            self.value = value - 1
-        }
+        self.value = value - 1
     }
 }
 

Sources/Basics/FileSystem/FileSystem+Extensions.swift

@@ -681,3 +681,17 @@ extension FileSystem {
         }
     }
 }
+
+extension FileSystem {
+    /// Do a deep enumeration, passing each file to block
+    public func enumerate(directory: AbsolutePath, block: (AbsolutePath) throws -> ()) throws {
+        for file in try getDirectoryContents(directory) {
+            let path = directory.appending(file)
+            if isDirectory(path) {
+                try enumerate(directory: path, block: block)
+            } else {
+                try block(path)
+            }
+        }
+    }
+}

Sources/Build/BuildPlan/BuildPlan+Test.swift

@@ -27,6 +27,7 @@ import struct PackageGraph.ResolvedProduct
 import struct PackageGraph.ResolvedModule
 
 import struct PackageModel.Sources
+import enum PackageModel.BuildSettings
 import class PackageModel.SwiftModule
 import class PackageModel.Module
 import struct SPMBuildCore.BuildParameters
@@ -80,17 +81,28 @@ extension BuildPlan {
                 let discoveryMainFile = discoveryDerivedDir.appending(component: TestDiscoveryTool.mainFileName)
 
                 var discoveryPaths: [AbsolutePath] = []
+                var discoveryBuildSettings: BuildSettings.AssignmentTable = .init()
                 discoveryPaths.append(discoveryMainFile)
                 for testTarget in testProduct.modules {
                     let path = discoveryDerivedDir.appending(components: testTarget.name + ".swift")
                     discoveryPaths.append(path)
+                    // Add in the include path from the test targets to ensure this module builds
+                    if let flags = testTarget.underlying.buildSettings.assignments[.OTHER_SWIFT_FLAGS] {
+                        for assignment in flags {
+                            let values = assignment.values.filter({ $0.hasPrefix("-I") })
+                            if !values.isEmpty {
+                                discoveryBuildSettings.add(.init(values: values, conditions: []), for: .OTHER_SWIFT_FLAGS)
+                            }
+                        }
+                    }
                 }
 
                 let discoveryTarget = SwiftModule(
                     name: discoveryTargetName,
                     dependencies: testProduct.underlying.modules.map { .module($0, conditions: []) },
                     packageAccess: true, // test target is allowed access to package decls by default
-                    testDiscoverySrc: Sources(paths: discoveryPaths, root: discoveryDerivedDir)
+                    testDiscoverySrc: Sources(paths: discoveryPaths, root: discoveryDerivedDir),
+                    buildSettings: discoveryBuildSettings
                 )
                 let discoveryResolvedModule = ResolvedModule(
                     packageIdentity: testProduct.packageIdentity,
@@ -127,13 +139,28 @@ extension BuildPlan {
                 let entryPointMainFile = entryPointDerivedDir.appending(component: entryPointMainFileName)
                 let entryPointSources = Sources(paths: [entryPointMainFile], root: entryPointDerivedDir)
 
+                var entryPointBuildSettings: BuildSettings.AssignmentTable = .init()
+                for testTarget in testProduct.modules {
+                    // Add in the include path from the test targets to ensure this module builds
+                    if let flags = testTarget.underlying.buildSettings.assignments[.OTHER_SWIFT_FLAGS] {
+                        for assignment in flags {
+                            let values = assignment.values.filter({ $0.hasPrefix("-I") })
+                            if !values.isEmpty {
+                                entryPointBuildSettings.add(.init(values: values, conditions: []), for: .OTHER_SWIFT_FLAGS)
+                            }
+                        }
+                    }
+                }
+
                 let entryPointTarget = SwiftModule(
                     name: testProduct.name,
                     type: .library,
                     dependencies: testProduct.underlying.modules.map { .module($0, conditions: []) } + swiftTargetDependencies,
                     packageAccess: true, // test target is allowed access to package decls
-                    testEntryPointSources: entryPointSources
+                    testEntryPointSources: entryPointSources,
+                    buildSettings: entryPointBuildSettings
                 )
+
                 let entryPointResolvedTarget = ResolvedModule(
                     packageIdentity: testProduct.packageIdentity,
                     underlying: entryPointTarget,
@@ -249,7 +276,8 @@ private extension PackageModel.SwiftModule {
         type: PackageModel.Module.Kind? = nil,
         dependencies: [PackageModel.Module.Dependency],
         packageAccess: Bool,
-        testEntryPointSources sources: Sources
+        testEntryPointSources sources: Sources,
+        buildSettings: BuildSettings.AssignmentTable = .init()
     ) {
         self.init(
             name: name,
@@ -258,6 +286,7 @@ private extension PackageModel.SwiftModule {
             sources: sources,
             dependencies: dependencies,
             packageAccess: packageAccess,
+            buildSettings: buildSettings,
             usesUnsafeFlags: false
         )
     }

Sources/CoreCommands/Options.swift

@@ -199,6 +199,19 @@ public struct CachingOptions: ParsableArguments {
           inversion: .prefixedEnableDisable,
           help: "Whether to use prebuilt swift-syntax libraries for macros.")
     public var usePrebuilts: Bool = false
+
+    /// Hidden option to override the prebuilts download location for testing
+    @Option(
+        name: .customLong("experimental-prebuilts-download-url"),
+        help: .hidden
+    )
+    public var prebuiltsDownloadURL: String?
+
+    @Option(
+        name: .customLong("experimental-prebuilts-root-cert"),
+        help: .hidden
+    )
+    public var prebuiltsRootCertPath: String?
 }
 
 public struct LoggingOptions: ParsableArguments {

Sources/CoreCommands/SwiftCommandState.swift

@@ -508,6 +508,8 @@ public final class SwiftCommandState {
                 },
                 manifestImportRestrictions: .none,
                 usePrebuilts: self.options.caching.usePrebuilts,
+                prebuiltsDownloadURL: options.caching.prebuiltsDownloadURL,
+                prebuiltsRootCertPath: options.caching.prebuiltsRootCertPath,
                 pruneDependencies: self.options.resolver.pruneDependencies,
                 traitConfiguration: traitConfiguration
             ),

Sources/PackageGraph/ModulesGraph+Loading.swift

@@ -223,7 +223,12 @@ extension ModulesGraph {
         )
 
         let rootPackages = resolvedPackages.filter { root.manifests.values.contains($0.manifest) }
-        checkAllDependenciesAreUsed(packages: resolvedPackages, rootPackages, observabilityScope: observabilityScope)
+        checkAllDependenciesAreUsed(
+            packages: resolvedPackages,
+            rootPackages,
+            prebuilts: prebuilts,
+            observabilityScope: observabilityScope
+        )
 
         return try ModulesGraph(
             rootPackages: rootPackages,
@@ -238,6 +243,7 @@ extension ModulesGraph {
 private func checkAllDependenciesAreUsed(
     packages: IdentifiableSet<ResolvedPackage>,
     _ rootPackages: [ResolvedPackage],
+    prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
     observabilityScope: ObservabilityScope
 ) {
     for package in rootPackages {
@@ -315,9 +321,10 @@ private func checkAllDependenciesAreUsed(
                 let usedByPackage = productDependencies.contains { $0.name == product.name }
                 // We check if any of the products of this dependency is guarded by a trait.
                 let traitGuarded = traitGuardedProductDependencies.contains(product.name)
+                // Consider prebuilts as used
+                let prebuilt = prebuilts[dependency.identity]?.keys.contains(product.name) ?? false
 
-                // If the product is either used directly or guarded by a trait we consider it as used
-                return usedByPackage || traitGuarded
+                return usedByPackage || traitGuarded || prebuilt
             }
 
             if !dependencyIsUsed && !observabilityScope.errorsReportedInAnyScope {
@@ -733,6 +740,24 @@ private func createResolvedPackages(
 
             // Establish product dependencies.
             for case .product(let productRef, let conditions) in moduleBuilder.module.dependencies {
+                if let package = productRef.package, prebuilts[.plain(package)]?[productRef.name] != nil {
+                    // See if we're using a prebuilt instead
+                    if moduleBuilder.module.type == .macro {
+                        continue
+                    } else if moduleBuilder.module.type == .test {
+                        // use prebuilt if this is a test that depends a macro target
+                        // these are guaranteed built for host
+                        if moduleBuilder.module.dependencies.contains(where: { dep in
+                            guard let module = dep.module else {
+                                return false
+                            }
+                            return module.type == .macro
+                        }) {
+                            continue
+                        }
+                    }
+                }
+
                 // Find the product in this package's dependency products.
                 // Look it up by ID if module aliasing is used, otherwise by name.
                 let product = lookupByProductIDs ? productDependencyMap[productRef.identity] :

Sources/PackageLoading/PackageBuilder.swift

@@ -281,8 +281,8 @@ public struct BinaryArtifact {
 
 /// A structure representing a prebuilt library to be used instead of a source dependency
 public struct PrebuiltLibrary {
-    /// The package reference.
-    public let packageRef: PackageReference
+    /// The package identity.
+    public let identity: PackageIdentity
 
     /// The name of the binary target the artifact corresponds to.
     public let libraryName: String
@@ -296,8 +296,8 @@ public struct PrebuiltLibrary {
     /// The C modules that need their includes directory added to the include path
     public let cModules: [String]
 
-    public init(packageRef: PackageReference, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
-        self.packageRef = packageRef
+    public init(identity: PackageIdentity, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
+        self.identity = identity
         self.libraryName = libraryName
         self.path = path
         self.products = products
@@ -1297,31 +1297,34 @@ public final class PackageBuilder {
             table.add(assignment, for: .SWIFT_ACTIVE_COMPILATION_CONDITIONS)
         }
 
-        // Add in flags for prebuilts
-        let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) {
-            guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1,
-                  let package = package,
-                  let prebuilt = prebuilts[.plain(package)]?[name]
-            else {
-                return
-            }
+        // Add in flags for prebuilts if the target is a macro or a macro test.
+        // Currently we only support prebuilts for macros.
+        if target.type == .macro || target.isMacroTest(in: manifest) {
+            let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) {
+                guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1,
+                      let package = package,
+                      let prebuilt = prebuilts[.plain(package)]?[name]
+                else {
+                    return
+                }
 
-            $0[prebuilt.libraryName] = prebuilt
-        }
+                $0[prebuilt.libraryName] = prebuilt
+            }
 
-        for prebuilt in prebuiltLibraries.values {
-            let lib = prebuilt.path.appending(components: ["lib", "lib\(prebuilt.libraryName).a"]).pathString
-            var ldFlagsAssignment = BuildSettings.Assignment()
-            ldFlagsAssignment.values = [lib]
-            table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS)
+            for prebuilt in prebuiltLibraries.values {
+                let lib = prebuilt.path.appending(components: ["lib", "lib\(prebuilt.libraryName).a"]).pathString
+                var ldFlagsAssignment = BuildSettings.Assignment()
+                ldFlagsAssignment.values = [lib]
+                table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS)
 
-            var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")]
-            for cModule in prebuilt.cModules {
-                includeDirs.append(prebuilt.path.appending(components: "include", cModule))
+                var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")]
+                for cModule in prebuilt.cModules {
+                    includeDirs.append(prebuilt.path.appending(components: "include", cModule))
+                }
+                var includeAssignment = BuildSettings.Assignment()
+                includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" })
+                table.add(includeAssignment, for: .OTHER_SWIFT_FLAGS)
             }
-            var includeAssignment = BuildSettings.Assignment()
-            includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" })
-            table.add(includeAssignment, for: .OTHER_SWIFT_FLAGS)
         }
 
         return table
@@ -1908,4 +1911,26 @@ extension TargetDescription {
     fileprivate var usesUnsafeFlags: Bool {
         settings.filter(\.kind.isUnsafeFlags).isEmpty == false
     }
+
+    fileprivate func isMacroTest(in manifest: Manifest) -> Bool {
+        guard self.type == .test else { return false }
+
+        return self.dependencies.contains(where: {
+            let name: String
+            switch $0 {
+            case .byName(name: let n, condition: _):
+                name = n
+            case .target(name: let n, condition: _):
+                name = n
+            default:
+                return false
+            }
+
+            guard let target = manifest.targetMap[name] else {
+                return false
+            }
+
+            return target.type == .macro
+        })
+    }
 }

Sources/PackageModel/Module/SwiftModule.swift

@@ -28,7 +28,12 @@ public final class SwiftModule: Module {
         [defaultTestEntryPointName, "LinuxMain.swift"]
     }
 
-    public init(name: String, dependencies: [Module.Dependency], packageAccess: Bool, testDiscoverySrc: Sources) {
+    public init(
+        name: String,
+        dependencies: [Module.Dependency],
+        packageAccess: Bool,
+        testDiscoverySrc: Sources,
+        buildSettings: BuildSettings.AssignmentTable = .init()) {
         self.declaredSwiftVersions = []
 
         super.init(
@@ -38,7 +43,7 @@ public final class SwiftModule: Module {
             sources: testDiscoverySrc,
             dependencies: dependencies,
             packageAccess: packageAccess,
-            buildSettings: .init(),
+            buildSettings: buildSettings,
             buildSettingsDescription: [],
             pluginUsages: [],
             usesUnsafeFlags: false

Sources/Workspace/CMakeLists.txt

@@ -14,6 +14,13 @@ add_library(Workspace
   ManagedArtifact.swift
   ManagedDependency.swift
   ManagedPrebuilt.swift
+  ManifestSigning/Base64URL.swift
+  ManifestSigning/CertificatePolicy.swift
+  ManifestSigning/embedded_resources.swift
+  ManifestSigning/ManifestSigning.swift
+  ManifestSigning/Signature.swift
+  ManifestSigning/Utilities.swift
+  ManifestSigning/X509Extensions.swift
   PackageContainer/FileSystemPackageContainer.swift
   PackageContainer/RegistryPackageContainer.swift
   PackageContainer/SourceControlPackageContainer.swift