diff --git a/Sources/Basics/Concurrency/SendableBox.swift b/Sources/Basics/Concurrency/SendableBox.swift index 0af62417dbb..a5ec3673be0 100644 --- a/Sources/Basics/Concurrency/SendableBox.swift +++ b/Sources/Basics/Concurrency/SendableBox.swift @@ -16,24 +16,24 @@ import struct Foundation.Date /// an `async` closure. This type serves as a replacement for `ThreadSafeBox` /// implemented with Swift Concurrency primitives. public actor SendableBox { - init(_ value: Value? = nil) { + public init(_ value: Value) { self.value = value } - var value: Value? + public var value: Value + + public func set(_ value: Value) { + self.value = value + } } extension SendableBox where Value == Int { func increment() { - if let value { - self.value = value + 1 - } + self.value = value + 1 } func decrement() { - if let value { - self.value = value - 1 - } + self.value = value - 1 } } diff --git a/Sources/Basics/FileSystem/FileSystem+Extensions.swift b/Sources/Basics/FileSystem/FileSystem+Extensions.swift index 0b43b6c7d75..5f9674b2ef5 100644 --- a/Sources/Basics/FileSystem/FileSystem+Extensions.swift +++ b/Sources/Basics/FileSystem/FileSystem+Extensions.swift @@ -681,3 +681,17 @@ extension FileSystem { } } } + +extension FileSystem { + /// Do a deep enumeration, passing each file to block + public func enumerate(directory: AbsolutePath, block: (AbsolutePath) throws -> ()) throws { + for file in try getDirectoryContents(directory) { + let path = directory.appending(file) + if isDirectory(path) { + try enumerate(directory: path, block: block) + } else { + try block(path) + } + } + } +} diff --git a/Sources/Build/BuildPlan/BuildPlan+Test.swift b/Sources/Build/BuildPlan/BuildPlan+Test.swift index b8e3fc9638a..8bb0042761f 100644 --- a/Sources/Build/BuildPlan/BuildPlan+Test.swift +++ b/Sources/Build/BuildPlan/BuildPlan+Test.swift @@ -27,6 +27,7 @@ import struct PackageGraph.ResolvedProduct import struct PackageGraph.ResolvedModule import struct PackageModel.Sources +import enum PackageModel.BuildSettings import class PackageModel.SwiftModule import class PackageModel.Module import struct SPMBuildCore.BuildParameters @@ -80,17 +81,28 @@ extension BuildPlan { let discoveryMainFile = discoveryDerivedDir.appending(component: TestDiscoveryTool.mainFileName) var discoveryPaths: [AbsolutePath] = [] + var discoveryBuildSettings: BuildSettings.AssignmentTable = .init() discoveryPaths.append(discoveryMainFile) for testTarget in testProduct.modules { let path = discoveryDerivedDir.appending(components: testTarget.name + ".swift") discoveryPaths.append(path) + // Add in the include path from the test targets to ensure this module builds + if let flags = testTarget.underlying.buildSettings.assignments[.OTHER_SWIFT_FLAGS] { + for assignment in flags { + let values = assignment.values.filter({ $0.hasPrefix("-I") }) + if !values.isEmpty { + discoveryBuildSettings.add(.init(values: values, conditions: []), for: .OTHER_SWIFT_FLAGS) + } + } + } } let discoveryTarget = SwiftModule( name: discoveryTargetName, dependencies: testProduct.underlying.modules.map { .module($0, conditions: []) }, packageAccess: true, // test target is allowed access to package decls by default - testDiscoverySrc: Sources(paths: discoveryPaths, root: discoveryDerivedDir) + testDiscoverySrc: Sources(paths: discoveryPaths, root: discoveryDerivedDir), + buildSettings: discoveryBuildSettings ) let discoveryResolvedModule = ResolvedModule( packageIdentity: testProduct.packageIdentity, @@ -127,13 +139,28 @@ extension BuildPlan { let entryPointMainFile = entryPointDerivedDir.appending(component: entryPointMainFileName) let entryPointSources = Sources(paths: [entryPointMainFile], root: entryPointDerivedDir) + var entryPointBuildSettings: BuildSettings.AssignmentTable = .init() + for testTarget in testProduct.modules { + // Add in the include path from the test targets to ensure this module builds + if let flags = testTarget.underlying.buildSettings.assignments[.OTHER_SWIFT_FLAGS] { + for assignment in flags { + let values = assignment.values.filter({ $0.hasPrefix("-I") }) + if !values.isEmpty { + entryPointBuildSettings.add(.init(values: values, conditions: []), for: .OTHER_SWIFT_FLAGS) + } + } + } + } + let entryPointTarget = SwiftModule( name: testProduct.name, type: .library, dependencies: testProduct.underlying.modules.map { .module($0, conditions: []) } + swiftTargetDependencies, packageAccess: true, // test target is allowed access to package decls - testEntryPointSources: entryPointSources + testEntryPointSources: entryPointSources, + buildSettings: entryPointBuildSettings ) + let entryPointResolvedTarget = ResolvedModule( packageIdentity: testProduct.packageIdentity, underlying: entryPointTarget, @@ -249,7 +276,8 @@ private extension PackageModel.SwiftModule { type: PackageModel.Module.Kind? = nil, dependencies: [PackageModel.Module.Dependency], packageAccess: Bool, - testEntryPointSources sources: Sources + testEntryPointSources sources: Sources, + buildSettings: BuildSettings.AssignmentTable = .init() ) { self.init( name: name, @@ -258,6 +286,7 @@ private extension PackageModel.SwiftModule { sources: sources, dependencies: dependencies, packageAccess: packageAccess, + buildSettings: buildSettings, usesUnsafeFlags: false ) } diff --git a/Sources/CoreCommands/Options.swift b/Sources/CoreCommands/Options.swift index d12fe55f2fd..c892694f383 100644 --- a/Sources/CoreCommands/Options.swift +++ b/Sources/CoreCommands/Options.swift @@ -199,6 +199,19 @@ public struct CachingOptions: ParsableArguments { inversion: .prefixedEnableDisable, help: "Whether to use prebuilt swift-syntax libraries for macros.") public var usePrebuilts: Bool = false + + /// Hidden option to override the prebuilts download location for testing + @Option( + name: .customLong("experimental-prebuilts-download-url"), + help: .hidden + ) + public var prebuiltsDownloadURL: String? + + @Option( + name: .customLong("experimental-prebuilts-root-cert"), + help: .hidden + ) + public var prebuiltsRootCertPath: String? } public struct LoggingOptions: ParsableArguments { diff --git a/Sources/CoreCommands/SwiftCommandState.swift b/Sources/CoreCommands/SwiftCommandState.swift index 57551331b75..4fea92bedb8 100644 --- a/Sources/CoreCommands/SwiftCommandState.swift +++ b/Sources/CoreCommands/SwiftCommandState.swift @@ -508,6 +508,8 @@ public final class SwiftCommandState { }, manifestImportRestrictions: .none, usePrebuilts: self.options.caching.usePrebuilts, + prebuiltsDownloadURL: options.caching.prebuiltsDownloadURL, + prebuiltsRootCertPath: options.caching.prebuiltsRootCertPath, pruneDependencies: self.options.resolver.pruneDependencies, traitConfiguration: traitConfiguration ), diff --git a/Sources/PackageGraph/ModulesGraph+Loading.swift b/Sources/PackageGraph/ModulesGraph+Loading.swift index 50020c2f602..fb4c3a1bc7c 100644 --- a/Sources/PackageGraph/ModulesGraph+Loading.swift +++ b/Sources/PackageGraph/ModulesGraph+Loading.swift @@ -223,7 +223,12 @@ extension ModulesGraph { ) let rootPackages = resolvedPackages.filter { root.manifests.values.contains($0.manifest) } - checkAllDependenciesAreUsed(packages: resolvedPackages, rootPackages, observabilityScope: observabilityScope) + checkAllDependenciesAreUsed( + packages: resolvedPackages, + rootPackages, + prebuilts: prebuilts, + observabilityScope: observabilityScope + ) return try ModulesGraph( rootPackages: rootPackages, @@ -238,6 +243,7 @@ extension ModulesGraph { private func checkAllDependenciesAreUsed( packages: IdentifiableSet, _ rootPackages: [ResolvedPackage], + prebuilts: [PackageIdentity: [String: PrebuiltLibrary]], observabilityScope: ObservabilityScope ) { for package in rootPackages { @@ -315,9 +321,10 @@ private func checkAllDependenciesAreUsed( let usedByPackage = productDependencies.contains { $0.name == product.name } // We check if any of the products of this dependency is guarded by a trait. let traitGuarded = traitGuardedProductDependencies.contains(product.name) + // Consider prebuilts as used + let prebuilt = prebuilts[dependency.identity]?.keys.contains(product.name) ?? false - // If the product is either used directly or guarded by a trait we consider it as used - return usedByPackage || traitGuarded + return usedByPackage || traitGuarded || prebuilt } if !dependencyIsUsed && !observabilityScope.errorsReportedInAnyScope { @@ -733,6 +740,24 @@ private func createResolvedPackages( // Establish product dependencies. for case .product(let productRef, let conditions) in moduleBuilder.module.dependencies { + if let package = productRef.package, prebuilts[.plain(package)]?[productRef.name] != nil { + // See if we're using a prebuilt instead + if moduleBuilder.module.type == .macro { + continue + } else if moduleBuilder.module.type == .test { + // use prebuilt if this is a test that depends a macro target + // these are guaranteed built for host + if moduleBuilder.module.dependencies.contains(where: { dep in + guard let module = dep.module else { + return false + } + return module.type == .macro + }) { + continue + } + } + } + // Find the product in this package's dependency products. // Look it up by ID if module aliasing is used, otherwise by name. let product = lookupByProductIDs ? productDependencyMap[productRef.identity] : diff --git a/Sources/PackageLoading/PackageBuilder.swift b/Sources/PackageLoading/PackageBuilder.swift index df8641cf00c..112e59a77f6 100644 --- a/Sources/PackageLoading/PackageBuilder.swift +++ b/Sources/PackageLoading/PackageBuilder.swift @@ -281,8 +281,8 @@ public struct BinaryArtifact { /// A structure representing a prebuilt library to be used instead of a source dependency public struct PrebuiltLibrary { - /// The package reference. - public let packageRef: PackageReference + /// The package identity. + public let identity: PackageIdentity /// The name of the binary target the artifact corresponds to. public let libraryName: String @@ -296,8 +296,8 @@ public struct PrebuiltLibrary { /// The C modules that need their includes directory added to the include path public let cModules: [String] - public init(packageRef: PackageReference, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) { - self.packageRef = packageRef + public init(identity: PackageIdentity, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) { + self.identity = identity self.libraryName = libraryName self.path = path self.products = products @@ -1297,31 +1297,34 @@ public final class PackageBuilder { table.add(assignment, for: .SWIFT_ACTIVE_COMPILATION_CONDITIONS) } - // Add in flags for prebuilts - let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) { - guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1, - let package = package, - let prebuilt = prebuilts[.plain(package)]?[name] - else { - return - } + // Add in flags for prebuilts if the target is a macro or a macro test. + // Currently we only support prebuilts for macros. + if target.type == .macro || target.isMacroTest(in: manifest) { + let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) { + guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1, + let package = package, + let prebuilt = prebuilts[.plain(package)]?[name] + else { + return + } - $0[prebuilt.libraryName] = prebuilt - } + $0[prebuilt.libraryName] = prebuilt + } - for prebuilt in prebuiltLibraries.values { - let lib = prebuilt.path.appending(components: ["lib", "lib\(prebuilt.libraryName).a"]).pathString - var ldFlagsAssignment = BuildSettings.Assignment() - ldFlagsAssignment.values = [lib] - table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS) + for prebuilt in prebuiltLibraries.values { + let lib = prebuilt.path.appending(components: ["lib", "lib\(prebuilt.libraryName).a"]).pathString + var ldFlagsAssignment = BuildSettings.Assignment() + ldFlagsAssignment.values = [lib] + table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS) - var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")] - for cModule in prebuilt.cModules { - includeDirs.append(prebuilt.path.appending(components: "include", cModule)) + var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")] + for cModule in prebuilt.cModules { + includeDirs.append(prebuilt.path.appending(components: "include", cModule)) + } + var includeAssignment = BuildSettings.Assignment() + includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" }) + table.add(includeAssignment, for: .OTHER_SWIFT_FLAGS) } - var includeAssignment = BuildSettings.Assignment() - includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" }) - table.add(includeAssignment, for: .OTHER_SWIFT_FLAGS) } return table @@ -1908,4 +1911,26 @@ extension TargetDescription { fileprivate var usesUnsafeFlags: Bool { settings.filter(\.kind.isUnsafeFlags).isEmpty == false } + + fileprivate func isMacroTest(in manifest: Manifest) -> Bool { + guard self.type == .test else { return false } + + return self.dependencies.contains(where: { + let name: String + switch $0 { + case .byName(name: let n, condition: _): + name = n + case .target(name: let n, condition: _): + name = n + default: + return false + } + + guard let target = manifest.targetMap[name] else { + return false + } + + return target.type == .macro + }) + } } diff --git a/Sources/PackageModel/Module/SwiftModule.swift b/Sources/PackageModel/Module/SwiftModule.swift index 222c62c59ad..93b083b1ebf 100644 --- a/Sources/PackageModel/Module/SwiftModule.swift +++ b/Sources/PackageModel/Module/SwiftModule.swift @@ -28,7 +28,12 @@ public final class SwiftModule: Module { [defaultTestEntryPointName, "LinuxMain.swift"] } - public init(name: String, dependencies: [Module.Dependency], packageAccess: Bool, testDiscoverySrc: Sources) { + public init( + name: String, + dependencies: [Module.Dependency], + packageAccess: Bool, + testDiscoverySrc: Sources, + buildSettings: BuildSettings.AssignmentTable = .init()) { self.declaredSwiftVersions = [] super.init( @@ -38,7 +43,7 @@ public final class SwiftModule: Module { sources: testDiscoverySrc, dependencies: dependencies, packageAccess: packageAccess, - buildSettings: .init(), + buildSettings: buildSettings, buildSettingsDescription: [], pluginUsages: [], usesUnsafeFlags: false diff --git a/Sources/Workspace/CMakeLists.txt b/Sources/Workspace/CMakeLists.txt index c079fab70b9..d0b033eb1cf 100644 --- a/Sources/Workspace/CMakeLists.txt +++ b/Sources/Workspace/CMakeLists.txt @@ -14,6 +14,13 @@ add_library(Workspace ManagedArtifact.swift ManagedDependency.swift ManagedPrebuilt.swift + ManifestSigning/Base64URL.swift + ManifestSigning/CertificatePolicy.swift + ManifestSigning/embedded_resources.swift + ManifestSigning/ManifestSigning.swift + ManifestSigning/Signature.swift + ManifestSigning/Utilities.swift + ManifestSigning/X509Extensions.swift PackageContainer/FileSystemPackageContainer.swift PackageContainer/RegistryPackageContainer.swift PackageContainer/SourceControlPackageContainer.swift diff --git a/Sources/Workspace/ManagedPrebuilt.swift b/Sources/Workspace/ManagedPrebuilt.swift index 9c8aafc2343..b4ce241331e 100644 --- a/Sources/Workspace/ManagedPrebuilt.swift +++ b/Sources/Workspace/ManagedPrebuilt.swift @@ -10,21 +10,26 @@ // //===----------------------------------------------------------------------===// +import struct TSCUtility.Version +import struct TSCBasic.StringError + import Basics import PackageModel -import TSCBasic extension Workspace { /// A downloaded prebuilt managed by the workspace. public struct ManagedPrebuilt { - /// The package reference. - public let packageRef: PackageReference + /// The package identity + public let identity: PackageIdentity + + /// The package version + public let version: Version /// The name of the binary target the artifact corresponds to. public let libraryName: String /// The path to the extracted prebuilt artifacts - public let path: Basics.AbsolutePath + public let path: AbsolutePath /// The products in the library public let products: [String] @@ -36,7 +41,7 @@ extension Workspace { extension Workspace.ManagedPrebuilt: CustomStringConvertible { public var description: String { - return "" + return "" } } @@ -46,60 +51,60 @@ extension Workspace { /// A collection of managed artifacts which have been downloaded. public final class ManagedPrebuilts { /// A mapping from package identity, to target name, to ManagedArtifact. - private var artifactMap: [PackageIdentity: [String: ManagedPrebuilt]] + private var prebuiltMap: [PackageIdentity: [String: ManagedPrebuilt]] - internal var artifacts: AnyCollection { - AnyCollection(self.artifactMap.values.lazy.flatMap{ $0.values }) + internal var prebuilts: AnyCollection { + AnyCollection(self.prebuiltMap.values.lazy.flatMap{ $0.values }) } init() { - self.artifactMap = [:] + self.prebuiltMap = [:] } - init(_ artifacts: [ManagedPrebuilt]) throws { - let artifactsByPackagePath = Dictionary(grouping: artifacts, by: { $0.packageRef.identity }) - self.artifactMap = try artifactsByPackagePath.mapValues{ artifacts in - try Dictionary(artifacts.map { ($0.libraryName, $0) }, uniquingKeysWith: { _, _ in + init(_ prebuilts: [ManagedPrebuilt]) throws { + let prebuiltsByPackagePath = Dictionary(grouping: prebuilts, by: { $0.identity }) + self.prebuiltMap = try prebuiltsByPackagePath.mapValues{ prebuilt in + try Dictionary(prebuilt.map { ($0.libraryName, $0) }, uniquingKeysWith: { _, _ in // should be unique - throw StringError("binary artifact already exists in managed artifacts") + throw StringError("prebuilt already exists in managed prebuilts") }) } } public subscript(packageIdentity packageIdentity: PackageIdentity, targetName targetName: String) -> ManagedPrebuilt? { - self.artifactMap[packageIdentity]?[targetName] + self.prebuiltMap[packageIdentity]?[targetName] } - public func add(_ artifact: ManagedPrebuilt) { - self.artifactMap[artifact.packageRef.identity, default: [:]][artifact.libraryName] = artifact + public func add(_ prebuilt: ManagedPrebuilt) { + self.prebuiltMap[prebuilt.identity, default: [:]][prebuilt.libraryName] = prebuilt } public func remove(packageIdentity: PackageIdentity, targetName: String) { - self.artifactMap[packageIdentity]?[targetName] = nil + self.prebuiltMap[packageIdentity]?[targetName] = nil } } } extension Workspace.ManagedPrebuilts: Collection { public var startIndex: AnyIndex { - self.artifacts.startIndex + self.prebuilts.startIndex } public var endIndex: AnyIndex { - self.artifacts.endIndex + self.prebuilts.endIndex } public subscript(index: AnyIndex) -> Workspace.ManagedPrebuilt { - self.artifacts[index] + self.prebuilts[index] } public func index(after index: AnyIndex) -> AnyIndex { - self.artifacts.index(after: index) + self.prebuilts.index(after: index) } } extension Workspace.ManagedPrebuilts: CustomStringConvertible { public var description: String { - "" + "" } } diff --git a/Sources/Workspace/ManifestSigning/Base64URL.swift b/Sources/Workspace/ManifestSigning/Base64URL.swift new file mode 100644 index 00000000000..bae48bb4c62 --- /dev/null +++ b/Sources/Workspace/ManifestSigning/Base64URL.swift @@ -0,0 +1,76 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2021-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// +// This source file is part of the Vapor open source project +// +// Copyright (c) 2017-2020 Vapor project authors +// Licensed under MIT +// +// See LICENSE for license information +// +// SPDX-License-Identifier: MIT +// +//===----------------------------------------------------------------------===// + +import Foundation + +// Source: https://github.com/vapor/jwt-kit/blob/master/Sources/JWTKit/Utilities/Base64URL.swift + +extension DataProtocol { + func base64URLDecodedBytes() -> Data? { + var data = Data(self) + data.base64URLUnescape() + return Data(base64Encoded: data) + } + + func base64URLEncodedBytes() -> Data { + var data = Data(self).base64EncodedData() + data.base64URLEscape() + return data + } +} + +extension Data { + /// Converts base64-url encoded data to a base64 encoded data. + /// + /// https://tools.ietf.org/html/rfc4648#page-7 + mutating func base64URLUnescape() { + for i in 0 ..< self.count { + switch self[i] { + case 0x2D: self[self.index(self.startIndex, offsetBy: i)] = 0x2B + case 0x5F: self[self.index(self.startIndex, offsetBy: i)] = 0x2F + default: break + } + } + /// https://stackoverflow.com/questions/43499651/decode-base64url-to-base64-swift + let padding = count % 4 + if padding > 0 { + self += Data(repeating: 0x3D, count: 4 - padding) + } + } + + /// Converts base64 encoded data to a base64-url encoded data. + /// + /// https://tools.ietf.org/html/rfc4648#page-7 + mutating func base64URLEscape() { + for i in 0 ..< self.count { + switch self[i] { + case 0x2B: self[self.index(self.startIndex, offsetBy: i)] = 0x2D + case 0x2F: self[self.index(self.startIndex, offsetBy: i)] = 0x5F + default: break + } + } + self = split(separator: 0x3D).first ?? .init() + } +} diff --git a/Sources/Workspace/ManifestSigning/CertificatePolicy.swift b/Sources/Workspace/ManifestSigning/CertificatePolicy.swift new file mode 100644 index 00000000000..f8c26593da0 --- /dev/null +++ b/Sources/Workspace/ManifestSigning/CertificatePolicy.swift @@ -0,0 +1,618 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2021-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Dispatch +import Foundation +import Basics +import struct TSCBasic.ByteString + +#if USE_IMPL_ONLY_IMPORTS +@_implementationOnly import SwiftASN1 +@_implementationOnly import X509 +#else +import SwiftASN1 +import X509 +#endif + +public enum CertificatePolicyKey: Hashable, CustomStringConvertible { + case `default`(subjectUserID: String? = nil, subjectOrganizationalUnit: String? = nil) + case appleSwiftPackageCollection(subjectUserID: String? = nil, subjectOrganizationalUnit: String? = nil) + + @available(*, deprecated, message: "use `appleSwiftPackageCollection` instead") + case appleDistribution(subjectUserID: String? = nil, subjectOrganizationalUnit: String? = nil) + + /// For testing only + case custom + + public var description: String { + switch self { + case .default(let userID, let organizationalUnit): + return "Default certificate policy\(userID.map { " (userID: \($0))" } ?? "")\(organizationalUnit.map { " (organizationalUnit: \($0))" } ?? "")" + case .appleSwiftPackageCollection(let userID, let organizationalUnit): + return "Swift Package Collection certificate policy\(userID.map { " (userID: \($0))" } ?? "")\(organizationalUnit.map { " (organizationalUnit: \($0))" } ?? "")" + case .appleDistribution(let userID, let organizationalUnit): + return "Distribution certificate policy\(userID.map { " (userID: \($0))" } ?? "")\(organizationalUnit.map { " (organizationalUnit: \($0))" } ?? "")" + case .custom: + return "Custom certificate policy" + } + } + + public static let `default` = CertificatePolicyKey.default() + public static let appleSwiftPackageCollection = CertificatePolicyKey.appleSwiftPackageCollection() + @available(*, deprecated, message: "use `appleSwiftPackageCollection` instead") + public static let appleDistribution = CertificatePolicyKey.appleDistribution() +} + +// MARK: - Certificate policies + +protocol CertificatePolicy { + /// Validates the given certificate chain. + /// + /// - Parameters: + /// - certChain: The certificate being verified must be the first element of the array, with its issuer the next + /// element and so on, and the root CA certificate is last. + /// - validationTime: Overrides the timestamp used for checking certificate expiry (e.g., for testing). + /// By default the current time is used. + func validate(certChain: [Certificate], validationTime: Date) async throws +} + +extension CertificatePolicy { + /// Validates the given certificate chain. + /// + /// - Parameters: + /// - certChain: The certificate being verified must be the first element of the array, with its issuer the next + /// element and so on, and the root CA certificate is last. + func validate(certChain: [Certificate]) async throws { + try await self.validate(certChain: certChain, validationTime: Date()) + } + + func verify( + certChain: [Certificate], + trustedRoots: [Certificate]?, + @PolicyBuilder policies: () -> some VerifierPolicy, + observabilityScope: ObservabilityScope + ) async throws { + guard !certChain.isEmpty else { + throw CertificatePolicyError.emptyCertChain + } + + let policies = policies() + + var trustStore = CertificateStores.defaultTrustRoots + if let trustedRoots { + trustStore.append(contentsOf: trustedRoots) + } + + guard !trustStore.isEmpty else { + throw CertificatePolicyError.noTrustedRootCertsConfigured + } + + var verifier = Verifier(rootCertificates: CertificateStore(trustStore)) { + policies + } + let result = await verifier.validate( + leafCertificate: certChain[0], + intermediates: CertificateStore(certChain) + ) + + switch result { + case .validCertificate: + return + case .couldNotValidate(let failures): + observabilityScope.emit(error: "Failed to validate certificate chain \(certChain): \(failures)") + throw CertificatePolicyError.invalidCertChain + } + } +} + +enum CertificatePolicyError: Error, Equatable { + case noTrustedRootCertsConfigured + case emptyCertChain + case invalidCertChain +} + +/// Default policy for validating certificates used to sign package collections. +/// +/// Certificates must satisfy these conditions: +/// - The timestamp at which signing/verification is done must fall within the signing certificate’s validity period. +/// - The certificate’s “Extended Key Usage” extension must include “Code Signing”. +/// - The certificate must use either 256-bit EC (recommended) or 2048-bit RSA key. +/// - The certificate must not be revoked. The certificate authority must support OCSP. +/// - The certificate chain is valid and root certificate must be trusted. +struct DefaultCertificatePolicy: CertificatePolicy { + let trustedRoots: [Certificate] + let expectedSubjectUserID: String? + let expectedSubjectOrganizationalUnit: String? + + private let httpClient: HTTPClient + private let observabilityScope: ObservabilityScope + + /// Initializes a `DefaultCertificatePolicy`. + /// + /// - Parameters: + /// - trustedRootCertsDir: Users may specify root certificates in addition to SwiftPM's default trust + /// store by placing them in this directory. + /// - additionalTrustedRootCerts: Root certificates to be trusted in addition to those in `trustedRootCertsDir`. + /// The difference between this and `trustedRootCertsDir` is that the latter is + /// user configured and dynamic, while this is configured by SwiftPM and static. + /// - expectedSubjectUserID: The subject user ID that must match if specified. + /// - expectedSubjectOrganizationalUnit: The subject organizational unit name that must match if specified. + init( + trustedRootCertsDir: AbsolutePath?, + fileSystem: FileSystem?, + additionalTrustedRootCerts: [Certificate]?, + expectedSubjectUserID: String? = nil, + expectedSubjectOrganizationalUnit: String? = nil, + observabilityScope: ObservabilityScope + ) { + var trustedRoots = [Certificate]() + if let trustedRootCertsDir, let fileSystem { + trustedRoots + .append(contentsOf: Self.loadCerts(at: trustedRootCertsDir, fileSystem: fileSystem, observabilityScope: observabilityScope)) + } + if let additionalTrustedRootCerts { + trustedRoots.append(contentsOf: additionalTrustedRootCerts) + } + self.trustedRoots = trustedRoots + self.expectedSubjectUserID = expectedSubjectUserID + self.expectedSubjectOrganizationalUnit = expectedSubjectOrganizationalUnit + self.httpClient = HTTPClient.makeDefault() + self.observabilityScope = observabilityScope + } + + func validate(certChain: [Certificate], validationTime: Date) async throws { + guard !certChain.isEmpty else { + throw CertificatePolicyError.emptyCertChain + } + + try await self.verify( + certChain: certChain, + trustedRoots: self.trustedRoots, + policies: { + _ADPCertificatePolicy() // included for testing + // Check if subject name matches + _SubjectNamePolicy( + expectedUserID: self.expectedSubjectUserID, + expectedOrganizationalUnit: self.expectedSubjectOrganizationalUnit + ) + // Must be a code signing certificate + _CodeSigningPolicy() + // Basic validations including expiry check + RFC5280Policy(validationTime: validationTime) + // Must support OCSP + _OCSPVerifierPolicy( + httpClient: self.httpClient, + validationTime: validationTime + ) + }, + observabilityScope: self.observabilityScope + ) + } +} + +/// Policy for validating developer.apple.com Swift Package Collection certificates. +/// +/// This has the same requirements as `DefaultCertificatePolicy` plus additional +/// marker extensions for Swift Package Collection certifiicates. +struct ADPSwiftPackageCollectionCertificatePolicy: CertificatePolicy { + let trustedRoots: [Certificate] + let expectedSubjectUserID: String? + let expectedSubjectOrganizationalUnit: String? + + private let httpClient: HTTPClient + private let observabilityScope: ObservabilityScope + + /// Initializes a `ADPSwiftPackageCollectionCertificatePolicy`. + /// + /// - Parameters: + /// - trustedRootCertsDir: Users may specify root certificates in addition to SwiftPM's default trust + /// store by placing them in this directory. + /// - additionalTrustedRootCerts: Root certificates to be trusted in addition to those in `trustedRootCertsDir`. + /// The difference between this and `trustedRootCertsDir` is that the latter is + /// user configured and dynamic, while this is configured by SwiftPM and static. + /// - expectedSubjectUserID: The subject user ID that must match if specified. + /// - expectedSubjectOrganizationalUnit: The subject organizational unit name that must match if specified. + init( + trustedRootCertsDir: AbsolutePath?, + fileSystem: FileSystem?, + additionalTrustedRootCerts: [Certificate]?, + expectedSubjectUserID: String? = nil, + expectedSubjectOrganizationalUnit: String? = nil, + observabilityScope: ObservabilityScope + ) { + var trustedRoots = [Certificate]() + if let trustedRootCertsDir, let fileSystem { + trustedRoots + .append(contentsOf: Self.loadCerts(at: trustedRootCertsDir, fileSystem: fileSystem, observabilityScope: observabilityScope)) + } + if let additionalTrustedRootCerts { + trustedRoots.append(contentsOf: additionalTrustedRootCerts) + } + self.trustedRoots = trustedRoots + self.expectedSubjectUserID = expectedSubjectUserID + self.expectedSubjectOrganizationalUnit = expectedSubjectOrganizationalUnit + self.httpClient = HTTPClient.makeDefault() + self.observabilityScope = observabilityScope + } + + func validate(certChain: [Certificate], validationTime: Date) async throws { + guard !certChain.isEmpty else { + throw CertificatePolicyError.emptyCertChain + } + + try await self.verify( + certChain: certChain, + trustedRoots: self.trustedRoots, + policies: { + // Check for specific markers + _ADPSwiftPackageCertificatePolicy() + _ADPCertificatePolicy() // included for testing + // Check if subject name matches + _SubjectNamePolicy( + expectedUserID: self.expectedSubjectUserID, + expectedOrganizationalUnit: self.expectedSubjectOrganizationalUnit + ) + // Must be a code signing certificate + _CodeSigningPolicy() + // Basic validations including expiry check + RFC5280Policy(validationTime: validationTime) + // Must support OCSP + _OCSPVerifierPolicy( + httpClient: self.httpClient, + validationTime: validationTime + ) + }, + observabilityScope: self.observabilityScope + ) + } +} + +/// Policy for validating developer.apple.com Apple Distribution certificates. +/// +/// This has the same requirements as `DefaultCertificatePolicy` plus additional +/// marker extensions for Apple Distribution certifiicates. +struct ADPAppleDistributionCertificatePolicy: CertificatePolicy { + let trustedRoots: [Certificate] + let expectedSubjectUserID: String? + let expectedSubjectOrganizationalUnit: String? + + private let httpClient: HTTPClient + private let observabilityScope: ObservabilityScope + + /// Initializes a `ADPAppleDistributionCertificatePolicy`. + /// + /// - Parameters: + /// - trustedRootCertsDir: Users may specify root certificates in addition to SwiftPM's default trust + /// store by placing them in this directory. + /// - additionalTrustedRootCerts: Root certificates to be trusted in addition to those in `trustedRootCertsDir`. + /// The difference between this and `trustedRootCertsDir` is that the latter is + /// user configured and dynamic, while this is configured by SwiftPM and static. + /// - expectedSubjectUserID: The subject user ID that must match if specified. + /// - expectedSubjectOrganizationalUnit: The subject organizational unit name that must match if specified. + init( + trustedRootCertsDir: AbsolutePath?, + fileSystem: FileSystem?, + additionalTrustedRootCerts: [Certificate]?, + expectedSubjectUserID: String? = nil, + expectedSubjectOrganizationalUnit: String? = nil, + observabilityScope: ObservabilityScope + ) { + var trustedRoots = [Certificate]() + if let trustedRootCertsDir, let fileSystem { + trustedRoots + .append(contentsOf: Self.loadCerts(at: trustedRootCertsDir, fileSystem: fileSystem, observabilityScope: observabilityScope)) + } + if let additionalTrustedRootCerts { + trustedRoots.append(contentsOf: additionalTrustedRootCerts) + } + self.trustedRoots = trustedRoots + self.expectedSubjectUserID = expectedSubjectUserID + self.expectedSubjectOrganizationalUnit = expectedSubjectOrganizationalUnit + self.httpClient = HTTPClient.makeDefault() + self.observabilityScope = observabilityScope + } + + func validate(certChain: [Certificate], validationTime: Date) async throws { + guard !certChain.isEmpty else { + throw CertificatePolicyError.emptyCertChain + } + + try await self.verify( + certChain: certChain, + trustedRoots: self.trustedRoots, + policies: { + // Check for specific markers + _ADPAppleDistributionCertificatePolicy() + _ADPCertificatePolicy() // included for testing + // Check if subject name matches + _SubjectNamePolicy( + expectedUserID: self.expectedSubjectUserID, + expectedOrganizationalUnit: self.expectedSubjectOrganizationalUnit + ) + // Must be a code signing certificate + _CodeSigningPolicy() + // Basic validations including expiry check + RFC5280Policy(validationTime: validationTime) + // Must support OCSP + _OCSPVerifierPolicy( + httpClient: self.httpClient, + validationTime: validationTime + ) + }, + observabilityScope: self.observabilityScope + ) + } +} + +// MARK: - Verifier policies + +/// Policy for code signing certificates. +struct _CodeSigningPolicy: VerifierPolicy { + let verifyingCriticalExtensions: [ASN1ObjectIdentifier] = [ + ASN1ObjectIdentifier.X509ExtensionID.keyUsage, + ASN1ObjectIdentifier.X509ExtensionID.extendedKeyUsage, + ] + + func chainMeetsPolicyRequirements(chain: UnverifiedCertificateChain) async -> PolicyEvaluationResult { + let isCodeSigning = ( + try? chain.leaf.extensions.extendedKeyUsage?.contains(ExtendedKeyUsage.Usage.codeSigning) + ) ?? false + guard isCodeSigning else { + return .failsToMeetPolicy(reason: "Certificate \(chain.leaf) does not have code signing extended key usage") + } + return .meetsPolicy + } +} + +/// Policy for revocation check via OCSP. +struct _OCSPVerifierPolicy: VerifierPolicy { + let verifyingCriticalExtensions: [ASN1ObjectIdentifier] = [] + + private static let cacheTTL: DispatchTimeInterval = .seconds(5 * 60) + private let cache = ThreadSafeKeyValueStore< + UnverifiedCertificateChain, + (result: PolicyEvaluationResult, expires: DispatchTime) + >() + + private var underlying: OCSPVerifierPolicy<_OCSPRequester> + + init(httpClient: HTTPClient, validationTime: Date) { + self.underlying = OCSPVerifierPolicy( + failureMode: .soft, + requester: _OCSPRequester(httpClient: httpClient), + validationTime: validationTime + ) + } + + mutating func chainMeetsPolicyRequirements(chain: UnverifiedCertificateChain) async -> PolicyEvaluationResult { + // Look for cached result + if let cached = self.cache[chain], cached.expires < .now() { + return cached.result + } + + // This makes HTTP requests + let result = await self.underlying.chainMeetsPolicyRequirements(chain: chain) + + // Save result to cache + self.cache[chain] = (result: result, expires: .now() + Self.cacheTTL) + return result + } +} + +private struct _OCSPRequester: OCSPRequester { + let httpClient: HTTPClient + + func query(request: [UInt8], uri: String) async -> OCSPRequesterQueryResult { + guard let url = URL(string: uri), let host = url.host else { + return .terminalError(SwiftOCSPRequesterError.invalidURL(uri)) + } + + do { + let response = try await self.httpClient.post( + url, + body: Data(request), + headers: [ + "Content-Type": "application/ocsp-request", + "Host": host, + ] + ) + + guard response.statusCode == 200 else { + throw SwiftOCSPRequesterError.invalidResponse(statusCode: response.statusCode) + } + guard let responseBody = response.body else { + throw SwiftOCSPRequesterError.emptyResponse + } + return .response(Array(responseBody)) + } catch { + return .nonTerminalError(error) + } + } +} + +enum SwiftOCSPRequesterError: Error { + case invalidURL(String) + case emptyResponse + case invalidResponse(statusCode: Int) +} + +/// Policy for matching subject name. +struct _SubjectNamePolicy: VerifierPolicy { + let verifyingCriticalExtensions: [ASN1ObjectIdentifier] = [] + + let expectedUserID: String? + let expectedOrganizationalUnit: String? + + func chainMeetsPolicyRequirements(chain: UnverifiedCertificateChain) async -> PolicyEvaluationResult { + if let expectedUserID { + let userID = chain.leaf.subject.userID + guard userID == expectedUserID else { + return .failsToMeetPolicy( + reason: "Subject user ID '\(String(describing: userID))' does not match expected '\(expectedUserID)'" + ) + } + } + + if let expectedOrganizationalUnit { + let organizationUnit = chain.leaf.subject.organizationalUnitName + guard organizationUnit == expectedOrganizationalUnit else { + return .failsToMeetPolicy( + reason: "Subject organizational unit name '\(String(describing: organizationUnit))' does not match expected '\(expectedOrganizationalUnit)'" + ) + } + } + + return .meetsPolicy + } +} + +/// Policy for ADP certificates. +struct _ADPCertificatePolicy: VerifierPolicy { + /// Include custom marker extensions (which can be critical) so they would not + /// be considered unhandled and cause certificate chain validation to fail. + let verifyingCriticalExtensions: [ASN1ObjectIdentifier] = + ASN1ObjectIdentifier.NameAttributes.adpAppleDevelopmentMarkers // included for testing + + func chainMeetsPolicyRequirements(chain: UnverifiedCertificateChain) async -> PolicyEvaluationResult { + // Not policing anything here. This policy is mainly for + // listing marker extensions to prevent chain validation + // from failing prematurely. + .meetsPolicy + } +} + +/// Policy for ADP Swift Package (Collection) certificates. +struct _ADPSwiftPackageCertificatePolicy: VerifierPolicy { + /// Include custom marker extensions (which can be critical) so they would not + /// be considered unhandled and cause certificate chain validation to fail. + let verifyingCriticalExtensions: [ASN1ObjectIdentifier] = [ + ASN1ObjectIdentifier.NameAttributes.adpSwiftPackageCollectionMarker, + ASN1ObjectIdentifier.NameAttributes.adpSwiftPackageMarker, + ] + + // developer.apple.com cert chain is always 3-long + private static let expectedCertChainLength = 3 + + func chainMeetsPolicyRequirements(chain: UnverifiedCertificateChain) async -> PolicyEvaluationResult { + guard chain.count == Self.expectedCertChainLength else { + return .failsToMeetPolicy( + reason: "Certificate chain should have length \(Self.expectedCertChainLength) but it's \(chain.count)" + ) + } + + // Package collection can be signed with "Swift Package Collection" + // or "Swift Package" certificate + guard chain.leaf.hasExtension(oid: ASN1ObjectIdentifier.NameAttributes.adpSwiftPackageCollectionMarker) + || chain.leaf.hasExtension(oid: ASN1ObjectIdentifier.NameAttributes.adpSwiftPackageMarker) + else { + return .failsToMeetPolicy(reason: "Leaf certificate missing marker OID") + } + + for marker in ASN1ObjectIdentifier.NameAttributes.wwdrIntermediateMarkers { + if chain[1].hasExtension(oid: marker) { + return .meetsPolicy + } + } + return .failsToMeetPolicy(reason: "Intermediate missing marker OID") + } +} + +/// Policy for ADP Apple Distribution certificates. +struct _ADPAppleDistributionCertificatePolicy: VerifierPolicy { + /// Include custom marker extensions (which can be critical) so they would not + /// be considered unhandled and cause certificate chain validation to fail. + let verifyingCriticalExtensions: [ASN1ObjectIdentifier] = + ASN1ObjectIdentifier.NameAttributes.adpAppleDistributionMarkers + + // developer.apple.com cert chain is always 3-long + private static let expectedCertChainLength = 3 + + func chainMeetsPolicyRequirements(chain: UnverifiedCertificateChain) async -> PolicyEvaluationResult { + guard chain.count == Self.expectedCertChainLength else { + return .failsToMeetPolicy( + reason: "Certificate chain should have length \(Self.expectedCertChainLength) but it's \(chain.count)" + ) + } + + var hasMarker = false + for marker in ASN1ObjectIdentifier.NameAttributes.adpAppleDistributionMarkers { + if chain.leaf.hasExtension(oid: marker) { + hasMarker = true + break + } + } + guard hasMarker else { + return .failsToMeetPolicy(reason: "Leaf certificate missing marker OID") + } + + for marker in ASN1ObjectIdentifier.NameAttributes.wwdrIntermediateMarkers { + if chain[1].hasExtension(oid: marker) { + return .meetsPolicy + } + } + return .failsToMeetPolicy(reason: "Intermediate missing marker OID") + } +} + +// MARK: - Default trust store + +enum Certificates { + static let appleRootsRaw = [ + PackageResources.AppleComputerRootCertificate_cer, + PackageResources.AppleIncRootCertificate_cer, + PackageResources.AppleRootCA_G2_cer, + PackageResources.AppleRootCA_G3_cer, + ] + + static let appleRoots = Self.appleRootsRaw.compactMap { + try? Certificate(derEncoded: $0) + } +} + +enum CertificateStores { + static let defaultTrustRoots = Certificates.appleRoots +} + +// MARK: - Utils + +extension CertificatePolicy { + fileprivate static func loadCerts(at directory: AbsolutePath, fileSystem: FileSystem, observabilityScope: ObservabilityScope) -> [Certificate] { + var certs = [Certificate]() + do { + try fileSystem.enumerate(directory: directory) { file in + do { + let certData = try fileSystem.readFileContents(file) + certs.append(try Certificate(derEncoded: Array(certData.contents))) + } catch { + observabilityScope.emit( + warning: "The certificate \(file.pathString) is invalid", + underlyingError: error + ) + } + } + } catch { + observabilityScope.emit( + warning: "Failed enumerating certificates directory", + underlyingError: error + ) + } + return certs + } +} + +extension HTTPClient { + fileprivate static func makeDefault() -> HTTPClient { + var httpClientConfig = HTTPClientConfiguration() + httpClientConfig.requestTimeout = .seconds(1) + return HTTPClient(configuration: httpClientConfig) + } +} diff --git a/Sources/Workspace/ManifestSigning/ManifestSigning.swift b/Sources/Workspace/ManifestSigning/ManifestSigning.swift new file mode 100644 index 00000000000..4f3bb2808c7 --- /dev/null +++ b/Sources/Workspace/ManifestSigning/ManifestSigning.swift @@ -0,0 +1,390 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2021-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + + +import Basics +import Dispatch +import Foundation +import SwiftASN1 + +#if USE_IMPL_ONLY_IMPORTS +@_implementationOnly import _CryptoExtras +@_implementationOnly import Crypto +@_implementationOnly import X509 +#else +import _CryptoExtras +import Crypto +import X509 +#endif + +public struct ManifestSignature: Equatable, Codable { + /// The signature + public let signature: String + + /// Details about the certificate used to generate the signature + public let certificate: Certificate + + public init(signature: String, certificate: Certificate) { + self.signature = signature + self.certificate = certificate + } + + public struct Certificate: Equatable, Codable { + /// Subject of the certificate + public let subject: Name + + /// Issuer of the certificate + public let issuer: Name + + /// Creates a `Certificate` + public init(subject: Name, issuer: Name) { + self.subject = subject + self.issuer = issuer + } + + /// Generic certificate name (e.g., subject, issuer) + public struct Name: Equatable, Codable { + /// User ID + public let userID: String? + + /// Common name + public let commonName: String? + + /// Organizational unit + public let organizationalUnit: String? + + /// Organization + public let organization: String? + + /// Creates a `Name` + public init(userID: String?, + commonName: String?, + organizationalUnit: String?, + organization: String?) { + self.userID = userID + self.commonName = commonName + self.organizationalUnit = organizationalUnit + self.organization = organization + } + } + } +} + +public protocol ManifestSigner { + /// Signs package collection using the given certificate and key. + /// + /// - Parameters: + /// - collection: The package collection to be signed + /// - certChainPaths: Paths to all DER-encoded certificates in the chain. The certificate used for signing + /// must be the first in the array. + /// - privateKeyPEM: Data of the private key (*.pem) of the certificate + /// - certPolicyKey: The key of the `CertificatePolicy` to use for validating certificates + func sign( + manifest: some Encodable, + certChainPaths: [AbsolutePath], + privateKeyPEM: Data, + fileSystem: FileSystem, + certPolicyKey: CertificatePolicyKey + ) async throws -> ManifestSignature +} + +extension ManifestSigner { + /// Signs package collection using the given certificate and key. + /// + /// - Parameters: + /// - collection: The package collection to be signed + /// - certChainPaths: Paths to all DER-encoded certificates in the chain. The certificate used for signing + /// must be the first in the array. + /// - certPrivateKeyPath: Path to the private key (*.pem) of the certificate + /// - certPolicyKey: The key of the `CertificatePolicy` to use for validating certificates + public func sign( + manifest: some Encodable, + certChainPaths: [AbsolutePath], + certPrivateKeyPath: AbsolutePath, + fileSystem: FileSystem, + certPolicyKey: CertificatePolicyKey = .default + ) async throws -> ManifestSignature { + let privateKey: Data = try fileSystem.readFileContents(certPrivateKeyPath) + return try await self.sign( + manifest: manifest, + certChainPaths: certChainPaths, + privateKeyPEM: privateKey, + fileSystem: fileSystem, + certPolicyKey: certPolicyKey + ) + } +} + +public protocol ManifestSignatureValidator { + /// Validates a signed package collection. + /// + /// - Parameters: + /// - signedCollection: The signed package collection + /// - certPolicyKey: The key of the `CertificatePolicy` to use for validating certificates + func validate( + manifest: any Encodable, + signature: ManifestSignature, + fileSystem: FileSystem, + certPolicyKey: CertificatePolicyKey + ) async throws +} + +// MARK: - Implementation + +public actor ManifestSigning: ManifestSigner, ManifestSignatureValidator { + private static let minimumRSAKeySizeInBits = 2048 + + /// Path of the optional directory containing root certificates to be trusted. + private let trustedRootCertsDir: AbsolutePath? + /// Root certificates to be trusted in additional to those found in `trustedRootCertsDir` + private let additionalTrustedRootCerts: [Certificate]? + + /// Internal cache/storage of `CertificatePolicy`s + private let certPolicies: [CertificatePolicyKey: CertificatePolicy] + + private let encoder: JSONEncoder + private let decoder: JSONDecoder + + private let observabilityScope: ObservabilityScope + + public init( + trustedRootCertsDir: AbsolutePath? = nil, + additionalTrustedRootCerts: [String]? = nil, + observabilityScope: ObservabilityScope + ) { + self.trustedRootCertsDir = trustedRootCertsDir + self.additionalTrustedRootCerts = additionalTrustedRootCerts.map { $0.compactMap { + guard let data = Data(base64Encoded: $0) else { + observabilityScope.emit(error: "The certificate \($0) is not in valid base64 encoding") + return nil + } + do { + return try Certificate(derEncoded: Array(data)) + } catch { + observabilityScope.emit( + error: "The certificate \($0) is not in valid DER format", + underlyingError: error + ) + return nil + } + } } + + self.certPolicies = [:] + self.encoder = JSONEncoder.makeWithDefaults() + self.decoder = JSONDecoder.makeWithDefaults() + self.observabilityScope = observabilityScope + } + + init(certPolicy: CertificatePolicy, observabilityScope: ObservabilityScope) { + // These should be set through the given CertificatePolicy + self.trustedRootCertsDir = nil + self.additionalTrustedRootCerts = nil + + self.certPolicies = [CertificatePolicyKey.custom: certPolicy] + self.encoder = JSONEncoder.makeWithDefaults() + self.decoder = JSONDecoder.makeWithDefaults() + self.observabilityScope = observabilityScope + } + + private func getCertificatePolicy(key: CertificatePolicyKey, fileSystem: FileSystem) throws -> CertificatePolicy { + switch key { + case .default(let subjectUserID, let subjectOrganizationalUnit): + // Create new instance each time since contents of trustedRootCertsDir might change + return DefaultCertificatePolicy( + trustedRootCertsDir: self.trustedRootCertsDir, + fileSystem: fileSystem, + additionalTrustedRootCerts: self.additionalTrustedRootCerts, + expectedSubjectUserID: subjectUserID, + expectedSubjectOrganizationalUnit: subjectOrganizationalUnit, + observabilityScope: self.observabilityScope + ) + case .appleSwiftPackageCollection(let subjectUserID, let subjectOrganizationalUnit): + // Create new instance each time since contents of trustedRootCertsDir might change + return ADPSwiftPackageCollectionCertificatePolicy( + trustedRootCertsDir: self.trustedRootCertsDir, + fileSystem: fileSystem, + additionalTrustedRootCerts: self.additionalTrustedRootCerts, + expectedSubjectUserID: subjectUserID, + expectedSubjectOrganizationalUnit: subjectOrganizationalUnit, + observabilityScope: self.observabilityScope + ) + case .appleDistribution(let subjectUserID, let subjectOrganizationalUnit): + // Create new instance each time since contents of trustedRootCertsDir might change + return ADPAppleDistributionCertificatePolicy( + trustedRootCertsDir: self.trustedRootCertsDir, + fileSystem: fileSystem, + additionalTrustedRootCerts: self.additionalTrustedRootCerts, + expectedSubjectUserID: subjectUserID, + expectedSubjectOrganizationalUnit: subjectOrganizationalUnit, + observabilityScope: self.observabilityScope + ) + case .custom: + // Custom `CertificatePolicy` can be set using the internal initializer only + guard let certPolicy = self.certPolicies[key] else { + throw ManifestSigningError.certPolicyNotFound + } + return certPolicy + } + } + + public func sign( + manifest: some Encodable, + certChainPaths: [AbsolutePath], + privateKeyPEM: Data, + fileSystem: FileSystem, + certPolicyKey: CertificatePolicyKey = .default + ) async throws -> ManifestSignature { + let certChainData: [Data] = try certChainPaths.map { try fileSystem.readFileContents($0) } + // Check that the certificate is valid + let certChain = try await self.validateCertChain(certChainData, certPolicyKey: certPolicyKey, fileSystem: fileSystem) + + let privateKeyPEMString = String(decoding: privateKeyPEM, as: UTF8.self) + + let signatureAlgorithm: Signature.Algorithm + let signatureProvider: (Data) throws -> Data + // Determine key type + do { + let privateKey = try P256.Signing.PrivateKey(pemRepresentation: privateKeyPEMString) + signatureAlgorithm = .ES256 + signatureProvider = { + try privateKey.signature(for: SHA256.hash(data: $0)).rawRepresentation + } + } catch { + do { + let privateKey = try _RSA.Signing.PrivateKey(pemRepresentation: privateKeyPEMString) + + guard privateKey.keySizeInBits >= Self.minimumRSAKeySizeInBits else { + throw ManifestSigningError.invalidKeySize(minimumBits: Self.minimumRSAKeySizeInBits) + } + + signatureAlgorithm = .RS256 + signatureProvider = { + try privateKey.signature(for: SHA256.hash(data: $0), padding: Signature.rsaSigningPadding) + .rawRepresentation + } + } catch let error as ManifestSigningError { + throw error + } catch { + throw ManifestSigningError.unsupportedKeyType + } + } + + // Generate signature + let signatureData = try Signature.generate( + payload: manifest, + certChainData: certChainData, + jsonEncoder: self.encoder, + signatureAlgorithm: signatureAlgorithm, + signatureProvider: signatureProvider + ) + + guard let signature = String(bytes: signatureData, encoding: .utf8) else { + throw ManifestSigningError.invalidSignature + } + + let certificate = certChain.first! // !-safe because certChain cannot be empty at this point + return ManifestSignature( + signature: signature, + certificate: ManifestSignature.Certificate( + subject: ManifestSignature.Certificate.Name(from: certificate.subject), + issuer: ManifestSignature.Certificate.Name(from: certificate.issuer) + ) + ) + } + + public func validate( + manifest: any Encodable, + signature: ManifestSignature, + fileSystem: FileSystem, + certPolicyKey: CertificatePolicyKey = .default + ) async throws { + let signatureBytes = Data(signature.signature.utf8).copyBytes() + + // Parse the signature + let certChainValidate = { certChainData in + try await self.validateCertChain(certChainData, certPolicyKey: certPolicyKey, fileSystem: fileSystem) + } + let signature = try await Signature.parse( + signatureBytes, + certChainValidate: certChainValidate, + jsonDecoder: self.decoder + ) + + // Verify the collection embedded in the signature is the same as received + // i.e., the signature is associated with the given collection and not another + guard try self.encoder.encode(manifest) == signature.payload else { + throw ManifestSigningError.invalidSignature + } + } + + private func validateCertChain( + _ certChainData: [Data], + certPolicyKey: CertificatePolicyKey, + fileSystem: FileSystem + ) async throws -> [Certificate] { + guard !certChainData.isEmpty else { + throw ManifestSigningError.emptyCertChain + } + + do { + let certChain = try certChainData.map { try Certificate(derEncoded: Array($0)) } + let certPolicy = try self.getCertificatePolicy(key: certPolicyKey, fileSystem: fileSystem) + + do { + try await certPolicy.validate(certChain: certChain) + return certChain + } catch { + self.observabilityScope.emit( + error: "\(certPolicyKey): The certificate chain is invalid", + underlyingError: error + ) + + if CertificatePolicyError.noTrustedRootCertsConfigured == error as? CertificatePolicyError { + throw ManifestSigningError.noTrustedRootCertsConfigured + } else { + throw ManifestSigningError.invalidCertChain + } + } + } catch let error as ManifestSigningError { + throw error + } catch { + self.observabilityScope.emit( + error: "An error has occurred while validating certificate chain", + underlyingError: error + ) + throw ManifestSigningError.invalidCertChain + } + } +} + +public enum ManifestSigningError: Error, Equatable { + case certPolicyNotFound + case emptyCertChain + case noTrustedRootCertsConfigured + case invalidCertChain + + case invalidSignature + + case unsupportedKeyType + case invalidKeySize(minimumBits: Int) +} + +extension ManifestSignature.Certificate.Name { + fileprivate init(from name: DistinguishedName) { + self.init( + userID: name.userID, + commonName: name.commonName, + organizationalUnit: name.organizationalUnitName, + organization: name.organizationName + ) + } +} diff --git a/Sources/Workspace/ManifestSigning/Signature.swift b/Sources/Workspace/ManifestSigning/Signature.swift new file mode 100644 index 00000000000..76e27d45d10 --- /dev/null +++ b/Sources/Workspace/ManifestSigning/Signature.swift @@ -0,0 +1,187 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2021-2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// +// This source file is part of the Vapor open source project +// +// Copyright (c) 2017-2020 Vapor project authors +// Licensed under MIT +// +// See LICENSE for license information +// +// SPDX-License-Identifier: MIT +// +//===----------------------------------------------------------------------===// + +import Foundation + +#if USE_IMPL_ONLY_IMPORTS +@_implementationOnly import _CryptoExtras +@_implementationOnly import Crypto +@_implementationOnly import X509 +#else +import _CryptoExtras +import Crypto +import X509 +#endif + +// The logic in this source file loosely follows https://www.rfc-editor.org/rfc/rfc7515.html +// for JSON Web Signature (JWS). + +struct Signature { + let header: Header + let payload: Data + let signature: Data +} + +extension Signature { + enum Algorithm: String, Codable { + case RS256 // RSASSA-PKCS1-v1_5 using SHA-256 + case ES256 // ECDSA using P-256 and SHA-256 + } + + struct Header: Equatable, Codable { + // https://www.rfc-editor.org/rfc/rfc7515.html#section-4.1.1 + let algorithm: Algorithm + + /// Base64 encoded certificate chain + let certChain: [String] + + enum CodingKeys: String, CodingKey { + case algorithm = "alg" + case certChain = "x5c" + } + } +} + +// Reference: https://github.com/vapor/jwt-kit/blob/master/Sources/JWTKit/JWTSerializer.swift +extension Signature { + static let rsaSigningPadding = _RSA.Signing.Padding.insecurePKCS1v1_5 + + static func generate( + payload: some Encodable, + certChainData: [Data], + jsonEncoder: JSONEncoder, + signatureAlgorithm: Signature.Algorithm, + signatureProvider: @escaping (Data) throws -> Data + ) throws -> Data { + let header = Signature.Header( + algorithm: signatureAlgorithm, + certChain: certChainData.map { $0.base64EncodedString() } + ) + let headerData = try jsonEncoder.encode(header) + let encodedHeader = headerData.base64URLEncodedBytes() + + let payloadData = try jsonEncoder.encode(payload) + let encodedPayload = payloadData.base64URLEncodedBytes() + + // https://www.rfc-editor.org/rfc/rfc7515.html#section-5.1 + // Signing input: BASE64URL(header) + '.' + BASE64URL(payload) + let signatureData = try signatureProvider(encodedHeader + .period + encodedPayload) + let encodedSignature = signatureData.base64URLEncodedBytes() + + // Result: header.payload.signature + let bytes = encodedHeader + + .period + + encodedPayload + + .period + + encodedSignature + return bytes + } +} + +// Reference: https://github.com/vapor/jwt-kit/blob/master/Sources/JWTKit/JWTParser.swift +extension Signature { + typealias CertChainValidate = ([Data]) async throws -> [Certificate] + + static func parse( + _ signature: String, + certChainValidate: CertChainValidate, + jsonDecoder: JSONDecoder + ) async throws -> Signature { + let bytes = Array(signature.utf8) + return try await Self.parse(bytes, certChainValidate: certChainValidate, jsonDecoder: jsonDecoder) + } + + static func parse( + _ signature: some DataProtocol, + certChainValidate: CertChainValidate, + jsonDecoder: JSONDecoder + ) async throws -> Signature { + let parts = signature.copyBytes().split(separator: .period) + guard parts.count == 3 else { + throw SignatureError.malformedSignature + } + + let encodedHeader = parts[0] + let encodedPayload = parts[1] + let encodedSignature = parts[2] + + guard let headerBytes = encodedHeader.base64URLDecodedBytes(), + let header = try? jsonDecoder.decode(Header.self, from: headerBytes) + else { + throw SignatureError.malformedSignature + } + + // Signature header contains the certificate and public key for verification + let certChainData = header.certChain.compactMap { Data(base64Encoded: $0) } + // Make sure we restore all certs successfully + guard certChainData.count == header.certChain.count else { + throw SignatureError.malformedSignature + } + + let certChain = try await certChainValidate(certChainData) + + guard let payloadBytes = encodedPayload.base64URLDecodedBytes(), + let signatureBytes = encodedSignature.base64URLDecodedBytes() + else { + throw SignatureError.malformedSignature + } + + // Extract public key from the certificate + let certificate = certChain.first! // !-safe because certChain is not empty at this point + // Verify the key was used to generate the signature + let message = Data(encodedHeader) + .period + Data(encodedPayload) + let digest = SHA256.hash(data: message) + + switch header.algorithm { + case .ES256: + guard let publicKey = P256.Signing.PublicKey(certificate.publicKey) else { + throw SignatureError.invalidPublicKey + } + guard try publicKey.isValidSignature(.init(rawRepresentation: signatureBytes), for: digest) + else { + throw SignatureError.invalidSignature + } + case .RS256: + guard let publicKey = _RSA.Signing.PublicKey(certificate.publicKey) else { + throw SignatureError.invalidPublicKey + } + guard publicKey.isValidSignature( + .init(rawRepresentation: signatureBytes), + for: digest, + padding: .insecurePKCS1v1_5 + ) else { + throw SignatureError.invalidSignature + } + } + + return Signature(header: header, payload: payloadBytes, signature: signatureBytes) + } +} + +enum SignatureError: Error { + case malformedSignature + case invalidSignature + case invalidPublicKey +} diff --git a/Sources/Workspace/ManifestSigning/Utilities.swift b/Sources/Workspace/ManifestSigning/Utilities.swift new file mode 100644 index 00000000000..625b7930d45 --- /dev/null +++ b/Sources/Workspace/ManifestSigning/Utilities.swift @@ -0,0 +1,39 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation + +extension DataProtocol { + func copyBytes() -> [UInt8] { + [UInt8](unsafeUninitializedCapacity: self.count) { buffer, initializedCount in + self.copyBytes(to: buffer) + initializedCount = self.count + } + } +} + +extension UInt8 { + static var period: UInt8 { + UInt8(ascii: ".") + } +} + +/// Cannot use `extension Data` if `period` is going to be used with +/// `+` operator via leading-dot syntax, for example: `Data(...) + .period` +/// because `+` is declared as `(Self, Other) -> Self` where +/// `Other: RangeReplaceableCollection, Other.Element == Self.Element` +/// which means that `.period` couldn't get `Data` inferred from the first argument. +extension RangeReplaceableCollection where Self == Data { + static var period: Data { + Data([UInt8.period]) + } +} diff --git a/Sources/Workspace/ManifestSigning/X509Extensions.swift b/Sources/Workspace/ManifestSigning/X509Extensions.swift new file mode 100644 index 00000000000..bef5d7ca1ce --- /dev/null +++ b/Sources/Workspace/ManifestSigning/X509Extensions.swift @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#if USE_IMPL_ONLY_IMPORTS +@_implementationOnly import SwiftASN1 +@_implementationOnly import X509 +#else +import SwiftASN1 +import X509 +#endif + +extension Certificate { + func hasExtension(oid: ASN1ObjectIdentifier) -> Bool { + self.extensions[oid: oid] != nil + } +} + +extension ASN1ObjectIdentifier.NameAttributes { + static let userID: ASN1ObjectIdentifier = [0, 9, 2342, 19_200_300, 100, 1, 1] + + // Marker OIDs for ADP certificates + static let adpSwiftPackageMarker: ASN1ObjectIdentifier = [1, 2, 840, 113_635, 100, 6, 1, 35] + static let adpSwiftPackageCollectionMarker: ASN1ObjectIdentifier = [1, 2, 840, 113_635, 100, 6, 1, 35] + static let adpAppleDevelopmentMarkers: [ASN1ObjectIdentifier] = [ + [1, 2, 840, 113_635, 100, 6, 1, 2], + [1, 2, 840, 113_635, 100, 6, 1, 12], + ] + static let adpAppleDistributionMarkers: [ASN1ObjectIdentifier] = [ + [1, 2, 840, 113_635, 100, 6, 1, 4], + [1, 2, 840, 113_635, 100, 6, 1, 7], + ] + static let wwdrIntermediateMarkers: [ASN1ObjectIdentifier] = [ + [1, 2, 840, 113_635, 100, 6, 2, 1], + [1, 2, 840, 113_635, 100, 6, 2, 15], + ] +} + +extension DistinguishedName { + var userID: String? { + self.stringAttribute(oid: ASN1ObjectIdentifier.NameAttributes.userID) + } + + var commonName: String? { + self.stringAttribute(oid: ASN1ObjectIdentifier.NameAttributes.commonName) + } + + var organizationalUnitName: String? { + self.stringAttribute(oid: ASN1ObjectIdentifier.NameAttributes.organizationalUnitName) + } + + var organizationName: String? { + self.stringAttribute(oid: ASN1ObjectIdentifier.NameAttributes.organizationName) + } + + private func stringAttribute(oid: ASN1ObjectIdentifier) -> String? { + for relativeDistinguishedName in self { + for attribute in relativeDistinguishedName where attribute.type == oid { + return attribute.value.description + } + } + return nil + } +} diff --git a/Sources/Workspace/ManifestSigning/embedded_resources.swift b/Sources/Workspace/ManifestSigning/embedded_resources.swift new file mode 100644 index 00000000000..a1dbeb69b2f --- /dev/null +++ b/Sources/Workspace/ManifestSigning/embedded_resources.swift @@ -0,0 +1,13 @@ +struct PackageResources { +static let AppleWWDRCAG2_cer: [UInt8] = [48,130,2,247,48,130,2,124,160,3,2,1,2,2,8,111,239,216,245,233,163,167,238,48,10,6,8,42,134,72,206,61,4,3,2,48,103,49,27,48,25,6,3,85,4,3,12,18,65,112,112,108,101,32,82,111,111,116,32,67,65,32,45,32,71,51,49,38,48,36,6,3,85,4,11,12,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,6,19,2,85,83,48,30,23,13,49,52,48,53,48,54,50,51,52,51,50,52,90,23,13,50,57,48,53,48,54,50,51,52,51,50,52,90,48,129,128,49,52,48,50,6,3,85,4,3,12,43,65,112,112,108,101,32,87,111,114,108,100,119,105,100,101,32,68,101,118,101,108,111,112,101,114,32,82,101,108,97,116,105,111,110,115,32,67,65,32,45,32,71,50,49,38,48,36,6,3,85,4,11,12,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,6,19,2,85,83,48,89,48,19,6,7,42,134,72,206,61,2,1,6,8,42,134,72,206,61,3,1,7,3,66,0,4,221,240,183,6,75,207,221,115,4,19,196,67,137,3,147,128,251,94,246,116,173,66,97,82,88,2,38,83,25,200,99,34,7,9,82,97,202,196,217,87,239,109,38,104,139,116,145,140,196,249,128,104,40,252,9,104,240,16,218,233,208,46,201,26,163,129,247,48,129,244,48,70,6,8,43,6,1,5,5,7,1,1,4,58,48,56,48,54,6,8,43,6,1,5,5,7,48,1,134,42,104,116,116,112,58,47,47,111,99,115,112,46,97,112,112,108,101,46,99,111,109,47,111,99,115,112,48,52,45,97,112,112,108,101,114,111,111,116,99,97,103,51,48,29,6,3,85,29,14,4,22,4,20,132,182,132,204,58,134,98,114,22,89,148,232,26,163,189,72,223,58,223,11,48,15,6,3,85,29,19,1,1,255,4,5,48,3,1,1,255,48,31,6,3,85,29,35,4,24,48,22,128,20,187,176,222,161,88,51,136,154,164,138,153,222,190,189,235,175,218,203,36,171,48,55,6,3,85,29,31,4,48,48,46,48,44,160,42,160,40,134,38,104,116,116,112,58,47,47,99,114,108,46,97,112,112,108,101,46,99,111,109,47,97,112,112,108,101,114,111,111,116,99,97,103,51,46,99,114,108,48,14,6,3,85,29,15,1,1,255,4,4,3,2,1,6,48,16,6,10,42,134,72,134,247,99,100,6,2,15,4,2,5,0,48,10,6,8,42,134,72,206,61,4,3,2,3,105,0,48,102,2,49,0,217,177,199,49,198,35,246,79,77,208,217,71,197,125,24,105,17,135,113,39,211,104,173,224,155,101,154,190,108,223,63,70,166,144,9,190,110,161,59,44,176,81,137,65,60,135,210,191,2,49,0,252,100,82,137,75,60,93,191,107,28,194,137,152,47,114,174,181,155,170,26,11,35,84,119,84,232,21,67,162,197,18,218,151,80,115,124,94,208,110,178,92,98,80,215,81,241,58,90] +static let AppleWWDRCAG3_cer: [UInt8] = [48,130,4,81,48,130,3,57,160,3,2,1,2,2,16,124,175,105,10,37,183,57,254,123,155,68,122,193,120,197,238,48,13,6,9,42,134,72,134,247,13,1,1,11,5,0,48,98,49,11,48,9,6,3,85,4,6,19,2,85,83,49,19,48,17,6,3,85,4,10,19,10,65,112,112,108,101,32,73,110,99,46,49,38,48,36,6,3,85,4,11,19,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,22,48,20,6,3,85,4,3,19,13,65,112,112,108,101,32,82,111,111,116,32,67,65,48,30,23,13,50,48,48,50,49,57,49,56,49,51,52,55,90,23,13,51,48,48,50,50,48,48,48,48,48,48,48,90,48,117,49,68,48,66,6,3,85,4,3,12,59,65,112,112,108,101,32,87,111,114,108,100,119,105,100,101,32,68,101,118,101,108,111,112,101,114,32,82,101,108,97,116,105,111,110,115,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,11,48,9,6,3,85,4,11,12,2,71,51,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,6,19,2,85,83,48,130,1,34,48,13,6,9,42,134,72,134,247,13,1,1,1,5,0,3,130,1,15,0,48,130,1,10,2,130,1,1,0,216,245,137,252,168,89,11,135,199,76,145,46,45,86,144,211,120,29,164,48,233,165,72,239,11,67,191,45,132,251,36,93,69,27,229,235,89,54,18,92,84,100,158,108,98,45,180,198,151,202,204,60,246,214,6,145,252,229,2,166,28,106,180,121,213,103,203,172,233,63,7,17,225,132,188,71,29,102,142,57,162,232,180,73,237,58,210,225,16,96,122,142,147,140,202,192,218,12,192,131,208,227,249,145,214,167,140,200,193,115,226,174,70,209,98,157,146,168,144,96,55,125,104,150,205,141,224,252,155,250,120,187,227,123,175,45,23,221,91,254,128,89,35,117,23,184,18,193,237,27,229,52,206,254,34,181,158,73,176,229,85,191,143,132,162,60,168,186,197,134,149,22,228,0,201,159,3,208,204,179,58,22,123,214,5,92,157,235,71,174,13,181,45,140,14,105,77,64,208,75,76,59,4,158,141,33,177,172,46,67,84,48,206,193,138,134,148,152,75,223,106,13,63,254,251,28,174,151,23,194,120,10,48,224,95,31,84,59,73,183,37,0,26,55,130,210,86,151,165,82,73,124,96,56,155,239,2,3,1,0,1,163,129,239,48,129,236,48,18,6,3,85,29,19,1,1,255,4,8,48,6,1,1,255,2,1,0,48,31,6,3,85,29,35,4,24,48,22,128,20,43,208,105,71,148,118,9,254,244,107,141,46,64,166,247,71,77,127,8,94,48,68,6,8,43,6,1,5,5,7,1,1,4,56,48,54,48,52,6,8,43,6,1,5,5,7,48,1,134,40,104,116,116,112,58,47,47,111,99,115,112,46,97,112,112,108,101,46,99,111,109,47,111,99,115,112,48,51,45,97,112,112,108,101,114,111,111,116,99,97,48,46,6,3,85,29,31,4,39,48,37,48,35,160,33,160,31,134,29,104,116,116,112,58,47,47,99,114,108,46,97,112,112,108,101,46,99,111,109,47,114,111,111,116,46,99,114,108,48,29,6,3,85,29,14,4,22,4,20,9,254,192,21,144,249,175,100,10,146,18,185,38,40,99,12,151,236,167,178,48,14,6,3,85,29,15,1,1,255,4,4,3,2,1,6,48,16,6,10,42,134,72,134,247,99,100,6,2,1,4,2,5,0,48,13,6,9,42,134,72,134,247,13,1,1,11,5,0,3,130,1,1,0,173,101,19,232,246,224,129,119,68,2,71,66,190,95,165,57,32,234,98,169,253,197,144,201,115,19,213,158,155,208,170,15,141,141,203,237,1,207,108,40,64,91,199,85,36,65,248,252,207,193,181,35,233,220,236,241,111,202,128,29,119,194,196,97,73,37,103,175,15,202,57,37,173,211,227,122,204,51,40,13,14,46,161,87,64,115,250,230,92,174,6,81,41,237,227,133,12,79,97,220,50,22,139,119,208,68,202,93,114,3,49,70,156,174,155,64,26,250,244,224,211,62,250,47,140,102,159,151,196,84,89,239,210,72,244,7,153,73,96,89,25,199,221,148,209,192,193,108,127,120,33,239,12,235,59,108,153,130,75,82,96,56,181,57,130,109,236,82,49,83,190,15,145,76,73,73,116,143,166,81,203,132,71,78,29,117,39,110,189,249,210,92,243,127,194,108,11,9,54,226,100,228,194,55,3,20,25,213,234,106,148,170,169,219,254,246,154,8,103,139,239,43,184,170,23,73,131,175,207,203,188,233,207,234,149,113,176,180,69,162,204,229,135,170,10,195,65,58,121,92,218,80,52,157,149,59] +static let AppleWWDRCAG5_cer: [UInt8] = [48,130,4,85,48,130,3,61,160,3,2,1,2,2,20,59,126,128,10,238,211,2,161,230,236,219,151,217,202,172,40,156,241,105,148,48,13,6,9,42,134,72,134,247,13,1,1,11,5,0,48,98,49,11,48,9,6,3,85,4,6,19,2,85,83,49,19,48,17,6,3,85,4,10,19,10,65,112,112,108,101,32,73,110,99,46,49,38,48,36,6,3,85,4,11,19,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,22,48,20,6,3,85,4,3,19,13,65,112,112,108,101,32,82,111,111,116,32,67,65,48,30,23,13,50,48,49,50,49,54,49,57,51,56,53,54,90,23,13,51,48,49,50,49,48,48,48,48,48,48,48,90,48,117,49,68,48,66,6,3,85,4,3,12,59,65,112,112,108,101,32,87,111,114,108,100,119,105,100,101,32,68,101,118,101,108,111,112,101,114,32,82,101,108,97,116,105,111,110,115,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,11,48,9,6,3,85,4,11,12,2,71,53,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,6,19,2,85,83,48,130,1,34,48,13,6,9,42,134,72,134,247,13,1,1,1,5,0,3,130,1,15,0,48,130,1,10,2,130,1,1,0,159,93,218,31,251,250,188,208,75,34,60,199,214,38,250,23,247,244,159,30,62,100,17,102,154,239,94,190,171,52,189,170,44,33,110,214,30,148,33,133,44,116,193,14,16,97,170,60,99,74,126,74,227,200,117,214,207,194,156,8,72,123,106,185,41,177,201,102,253,200,138,22,209,178,4,60,130,186,128,108,226,255,10,114,235,172,254,59,111,72,190,158,139,17,77,149,125,178,123,144,47,2,28,78,252,73,190,215,111,181,79,190,120,198,218,222,178,205,201,217,102,17,59,90,233,9,69,115,137,80,212,78,226,104,5,126,218,117,156,75,1,174,218,227,207,109,116,77,222,83,208,213,26,56,130,58,208,157,227,93,41,123,177,129,234,41,39,11,251,96,124,134,18,251,240,98,79,6,231,76,209,31,217,109,158,237,115,218,120,33,13,157,112,217,70,87,144,95,103,132,193,115,119,10,168,98,98,182,174,112,134,187,117,35,137,173,121,246,230,164,212,253,131,38,50,10,59,36,146,91,32,161,156,209,47,128,24,155,221,60,11,231,181,253,236,128,87,15,23,158,63,43,135,39,187,247,2,3,1,0,1,163,129,239,48,129,236,48,18,6,3,85,29,19,1,1,255,4,8,48,6,1,1,255,2,1,0,48,31,6,3,85,29,35,4,24,48,22,128,20,43,208,105,71,148,118,9,254,244,107,141,46,64,166,247,71,77,127,8,94,48,68,6,8,43,6,1,5,5,7,1,1,4,56,48,54,48,52,6,8,43,6,1,5,5,7,48,1,134,40,104,116,116,112,58,47,47,111,99,115,112,46,97,112,112,108,101,46,99,111,109,47,111,99,115,112,48,51,45,97,112,112,108,101,114,111,111,116,99,97,48,46,6,3,85,29,31,4,39,48,37,48,35,160,33,160,31,134,29,104,116,116,112,58,47,47,99,114,108,46,97,112,112,108,101,46,99,111,109,47,114,111,111,116,46,99,114,108,48,29,6,3,85,29,14,4,22,4,20,25,139,151,141,74,91,97,120,87,244,165,92,53,18,138,57,8,227,176,117,48,14,6,3,85,29,15,1,1,255,4,4,3,2,1,6,48,16,6,10,42,134,72,134,247,99,100,6,2,1,4,2,5,0,48,13,6,9,42,134,72,134,247,13,1,1,11,5,0,3,130,1,1,0,90,196,53,162,217,230,10,158,82,162,0,130,34,119,205,114,47,144,249,34,175,20,37,204,170,10,66,96,146,194,189,117,29,235,76,202,21,229,126,107,64,16,94,251,91,69,116,13,240,122,125,136,145,206,222,99,139,66,137,210,24,223,65,179,88,158,6,8,135,72,221,64,175,35,213,237,235,44,96,229,235,164,127,28,233,156,131,214,148,95,76,98,57,138,78,174,19,141,125,67,184,220,94,172,190,24,64,242,230,15,44,93,19,147,241,123,101,168,194,104,72,222,255,180,150,155,12,251,50,54,158,158,13,149,95,56,10,139,150,210,18,108,183,139,11,142,215,62,236,111,1,170,206,77,69,96,219,132,113,102,98,229,25,101,48,205,138,255,69,83,251,110,251,136,119,67,195,72,99,249,247,81,179,114,94,121,226,86,207,188,187,132,185,56,90,120,174,17,72,82,187,13,53,139,148,222,204,116,154,104,245,73,96,148,172,242,168,179,21,134,144,208,204,48,67,134,211,229,112,2,81,180,129,192,168,151,212,234,234,176,140,9,64,239,56,147,178,205,34,3,235,103,67,101,200,69] +static let AppleRootCA_G2_cer: [UInt8] = [48,130,5,146,48,130,3,122,160,3,2,1,2,2,8,1,224,229,181,131,103,163,224,48,13,6,9,42,134,72,134,247,13,1,1,12,5,0,48,103,49,27,48,25,6,3,85,4,3,12,18,65,112,112,108,101,32,82,111,111,116,32,67,65,32,45,32,71,50,49,38,48,36,6,3,85,4,11,12,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,6,19,2,85,83,48,30,23,13,49,52,48,52,51,48,49,56,49,48,48,57,90,23,13,51,57,48,52,51,48,49,56,49,48,48,57,90,48,103,49,27,48,25,6,3,85,4,3,12,18,65,112,112,108,101,32,82,111,111,116,32,67,65,32,45,32,71,50,49,38,48,36,6,3,85,4,11,12,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,6,19,2,85,83,48,130,2,34,48,13,6,9,42,134,72,134,247,13,1,1,1,5,0,3,130,2,15,0,48,130,2,10,2,130,2,1,0,216,17,18,72,72,218,41,138,73,197,28,199,236,110,51,109,254,77,251,224,28,222,172,94,226,54,167,36,249,127,80,107,76,206,185,48,84,39,229,179,214,237,37,230,48,182,5,55,94,20,34,17,197,232,170,27,210,251,178,210,9,149,56,164,239,42,73,140,93,62,113,102,3,56,251,22,245,133,136,228,90,146,12,4,50,242,200,64,251,82,95,159,246,192,241,227,186,69,160,80,213,18,139,242,221,222,145,134,35,240,245,182,114,46,1,218,11,246,46,57,8,95,25,161,99,65,11,28,167,148,193,134,196,83,47,118,246,10,215,12,209,131,63,26,83,25,243,87,213,39,127,252,19,184,248,146,141,252,211,40,67,60,181,104,0,37,93,39,98,211,221,85,221,68,32,144,131,53,147,197,191,184,25,251,107,227,220,8,66,230,175,109,250,158,64,202,78,133,133,120,73,177,215,195,193,48,57,50,171,126,95,170,211,139,111,159,45,26,33,104,112,103,179,163,241,152,65,109,145,124,248,215,219,168,231,95,33,26,140,51,191,49,116,183,184,209,244,224,34,244,191,114,52,223,247,129,77,113,125,81,161,226,179,240,211,40,22,115,111,205,204,173,55,125,78,235,173,64,225,63,129,253,247,61,10,62,162,241,189,49,150,41,89,220,194,25,128,140,91,116,198,44,211,16,83,38,29,20,79,196,212,129,102,60,135,103,51,39,20,8,233,180,119,132,52,82,143,137,248,104,152,23,191,195,187,170,19,147,31,93,84,47,168,199,124,251,13,20,190,21,61,36,52,242,154,220,117,65,102,34,180,1,214,11,175,144,158,12,234,98,248,155,89,60,8,226,150,52,228,99,222,188,55,212,235,12,136,3,67,11,80,175,160,52,221,80,77,21,251,90,36,216,12,250,12,99,158,31,3,177,225,238,225,170,67,244,102,101,40,55,2,49,239,1,199,30,209,204,159,109,202,84,58,64,219,206,207,79,70,139,74,101,154,106,198,104,108,215,204,153,27,71,176,114,195,119,143,196,247,97,156,116,31,206,253,107,161,194,156,148,130,171,148,162,231,189,27,186,185,112,57,149,23,197,41,243,57,88,52,245,196,164,198,123,96,185,102,67,80,63,110,97,252,14,249,134,170,96,12,67,75,149,2,3,1,0,1,163,66,48,64,48,29,6,3,85,29,14,4,22,4,20,196,153,19,108,24,3,194,123,192,163,160,13,127,114,128,122,28,119,38,141,48,15,6,3,85,29,19,1,1,255,4,5,48,3,1,1,255,48,14,6,3,85,29,15,1,1,255,4,4,3,2,1,6,48,13,6,9,42,134,72,134,247,13,1,1,12,5,0,3,130,2,1,0,81,166,243,226,244,184,61,147,191,45,206,15,187,91,225,85,20,78,78,209,229,206,121,93,129,127,254,182,240,135,51,248,239,148,229,126,220,106,121,167,28,190,240,148,183,166,209,48,156,200,13,10,117,158,125,146,149,126,24,157,126,194,113,105,124,20,234,207,131,14,228,20,66,158,116,14,16,205,171,26,186,17,97,129,120,216,241,181,69,64,120,171,168,192,206,251,125,99,55,104,246,231,251,175,198,195,75,236,31,54,38,19,84,134,148,114,178,234,2,237,139,109,228,12,166,144,192,87,117,207,140,66,125,92,230,49,125,243,201,178,146,105,70,14,136,248,227,45,66,178,56,168,166,25,141,241,159,205,238,106,101,188,26,176,37,189,167,41,253,244,62,162,117,73,191,158,219,201,247,167,30,99,153,225,92,70,255,146,5,140,250,30,32,249,134,148,86,37,229,180,87,56,157,235,136,100,20,33,73,33,57,191,98,102,169,177,162,202,111,63,33,96,197,137,212,69,54,200,152,124,189,246,254,153,73,128,59,44,210,166,167,136,3,4,49,25,183,182,58,97,69,250,201,242,35,200,99,115,191,86,137,49,176,217,124,98,167,123,21,168,136,138,171,56,64,194,204,18,255,21,227,240,55,223,55,114,203,204,152,230,191,162,188,250,38,138,113,86,215,231,36,27,72,68,62,158,252,159,201,204,26,236,67,60,1,188,52,120,200,105,245,198,230,86,236,6,9,54,144,235,20,74,27,94,201,136,35,218,3,48,145,11,184,54,62,249,231,181,40,111,190,63,236,60,143,101,29,229,192,30,135,164,170,186,152,253,146,227,108,38,119,221,6,180,100,6,135,244,78,214,186,74,170,22,168,244,5,103,102,150,186,226,85,121,195,44,93,73,143,128,73,43,138,18,199,118,128,81,223,186,189,101,93,62,55,71,99,49,233,229,244,197,63,75,173,4,138,122,113,44,175,9,67,55,15,168,227,50,79,244,69,182,109,151,54,236,132,245,10,1,234,23,187,133,141,66,147,112,195,80,229,20,139,191,63,195,65,15,221,34,4,35,8,138,186,109,113,68,171,115,9,58,201,249,82,128,9,223,186,233,230,22,202,46,46,76,178,211,220,229,4,84,178,212,52,128,50,181,188,15,23,225] +static let AppleComputerRootCertificate_cer: [UInt8] = [48,130,5,186,48,130,4,162,160,3,2,1,2,2,1,1,48,13,6,9,42,134,72,134,247,13,1,1,5,5,0,48,129,134,49,11,48,9,6,3,85,4,6,19,2,85,83,49,29,48,27,6,3,85,4,10,19,20,65,112,112,108,101,32,67,111,109,112,117,116,101,114,44,32,73,110,99,46,49,45,48,43,6,3,85,4,11,19,36,65,112,112,108,101,32,67,111,109,112,117,116,101,114,32,67,101,114,116,105,102,105,99,97,116,101,32,65,117,116,104,111,114,105,116,121,49,41,48,39,6,3,85,4,3,19,32,65,112,112,108,101,32,82,111,111,116,32,67,101,114,116,105,102,105,99,97,116,101,32,65,117,116,104,111,114,105,116,121,48,30,23,13,48,53,48,50,49,48,48,48,49,56,49,52,90,23,13,50,53,48,50,49,48,48,48,49,56,49,52,90,48,129,134,49,11,48,9,6,3,85,4,6,19,2,85,83,49,29,48,27,6,3,85,4,10,19,20,65,112,112,108,101,32,67,111,109,112,117,116,101,114,44,32,73,110,99,46,49,45,48,43,6,3,85,4,11,19,36,65,112,112,108,101,32,67,111,109,112,117,116,101,114,32,67,101,114,116,105,102,105,99,97,116,101,32,65,117,116,104,111,114,105,116,121,49,41,48,39,6,3,85,4,3,19,32,65,112,112,108,101,32,82,111,111,116,32,67,101,114,116,105,102,105,99,97,116,101,32,65,117,116,104,111,114,105,116,121,48,130,1,34,48,13,6,9,42,134,72,134,247,13,1,1,1,5,0,3,130,1,15,0,48,130,1,10,2,130,1,1,0,228,145,169,9,31,145,219,30,71,80,235,5,237,94,121,132,45,235,54,162,87,76,85,236,139,25,137,222,249,75,108,245,7,171,34,48,2,232,24,62,248,80,9,211,127,65,168,152,249,209,202,102,156,36,107,17,208,163,187,228,27,42,195,31,149,158,122,12,164,71,139,91,212,22,55,51,203,196,15,77,206,20,105,209,201,25,114,245,93,14,213,127,95,155,242,37,3,186,85,143,77,93,13,241,100,53,35,21,75,21,89,29,179,148,247,246,156,158,207,80,186,193,88,80,103,143,8,180,32,247,203,172,44,32,111,112,182,63,1,48,140,183,67,207,15,157,61,243,43,73,40,26,200,254,206,181,185,14,217,94,28,214,203,61,181,58,173,244,15,14,0,146,11,177,33,22,46,116,213,60,13,219,98,22,171,163,113,146,71,83,85,193,175,47,65,179,248,251,227,112,205,230,163,76,69,126,31,76,107,80,150,65,137,196,116,98,11,16,131,65,135,51,138,129,177,48,88,236,90,4,50,140,104,179,143,29,222,101,115,255,103,94,101,188,73,216,118,159,51,20,101,161,119,148,201,45,2,3,1,0,1,163,130,2,47,48,130,2,43,48,14,6,3,85,29,15,1,1,255,4,4,3,2,1,6,48,15,6,3,85,29,19,1,1,255,4,5,48,3,1,1,255,48,29,6,3,85,29,14,4,22,4,20,43,208,105,71,148,118,9,254,244,107,141,46,64,166,247,71,77,127,8,94,48,31,6,3,85,29,35,4,24,48,22,128,20,43,208,105,71,148,118,9,254,244,107,141,46,64,166,247,71,77,127,8,94,48,130,1,41,6,3,85,29,32,4,130,1,32,48,130,1,28,48,130,1,24,6,9,42,134,72,134,247,99,100,5,1,48,130,1,9,48,65,6,8,43,6,1,5,5,7,2,1,22,53,104,116,116,112,115,58,47,47,119,119,119,46,97,112,112,108,101,46,99,111,109,47,99,101,114,116,105,102,105,99,97,116,101,97,117,116,104,111,114,105,116,121,47,116,101,114,109,115,46,104,116,109,108,48,129,195,6,8,43,6,1,5,5,7,2,2,48,129,182,26,129,179,82,101,108,105,97,110,99,101,32,111,110,32,116,104,105,115,32,99,101,114,116,105,102,105,99,97,116,101,32,98,121,32,97,110,121,32,112,97,114,116,121,32,97,115,115,117,109,101,115,32,97,99,99,101,112,116,97,110,99,101,32,111,102,32,116,104,101,32,116,104,101,110,32,97,112,112,108,105,99,97,98,108,101,32,115,116,97,110,100,97,114,100,32,116,101,114,109,115,32,97,110,100,32,99,111,110,100,105,116,105,111,110,115,32,111,102,32,117,115,101,44,32,99,101,114,116,105,102,105,99,97,116,101,32,112,111,108,105,99,121,32,97,110,100,32,99,101,114,116,105,102,105,99,97,116,105,111,110,32,112,114,97,99,116,105,99,101,32,115,116,97,116,101,109,101,110,116,115,46,48,68,6,3,85,29,31,4,61,48,59,48,57,160,55,160,53,134,51,104,116,116,112,115,58,47,47,119,119,119,46,97,112,112,108,101,46,99,111,109,47,99,101,114,116,105,102,105,99,97,116,101,97,117,116,104,111,114,105,116,121,47,114,111,111,116,46,99,114,108,48,85,6,8,43,6,1,5,5,7,1,1,4,73,48,71,48,69,6,8,43,6,1,5,5,7,48,2,134,57,104,116,116,112,115,58,47,47,119,119,119,46,97,112,112,108,101,46,99,111,109,47,99,101,114,116,105,102,105,99,97,116,101,97,117,116,104,111,114,105,116,121,47,99,97,115,105,103,110,101,114,115,46,104,116,109,108,48,13,6,9,42,134,72,134,247,13,1,1,5,5,0,3,130,1,1,0,157,218,45,40,88,47,125,118,4,185,4,211,62,206,183,102,99,78,143,47,212,254,75,173,114,189,163,57,198,82,77,5,152,82,245,137,81,1,36,121,190,26,50,247,229,68,139,75,68,7,57,130,214,90,202,180,32,94,217,174,21,93,29,140,29,50,191,56,49,98,72,93,199,225,144,177,248,36,64,248,95,88,155,81,93,87,157,193,229,255,60,204,114,33,110,196,233,233,161,119,215,44,23,38,195,63,235,154,232,11,3,186,233,179,74,114,235,51,9,91,173,230,98,49,106,232,175,47,213,175,30,87,118,143,127,55,45,46,2,92,221,99,201,242,113,184,38,64,223,21,141,117,68,63,121,189,230,29,153,225,67,44,62,173,111,190,185,164,254,14,53,25,81,99,177,195,222,181,146,62,81,120,1,115,138,164,35,202,164,136,241,30,92,31,65,22,45,126,149,10,170,233,137,65,152,27,26,221,203,32,191,71,94,12,38,197,85,53,77,198,48,139,153,103,20,199,9,31,186,71,199,218,1,9,135,36,66,149,189,19,96,25,10,239,234,127,14,110,205,193,68,67,58,74,213,227] +static let AppleWWDRCAG7_cer: [UInt8] = [48,130,4,85,48,130,3,61,160,3,2,1,2,2,20,52,24,88,255,1,254,6,63,142,241,159,31,233,60,1,180,193,70,255,201,48,13,6,9,42,134,72,134,247,13,1,1,5,5,0,48,98,49,11,48,9,6,3,85,4,6,19,2,85,83,49,19,48,17,6,3,85,4,10,19,10,65,112,112,108,101,32,73,110,99,46,49,38,48,36,6,3,85,4,11,19,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,22,48,20,6,3,85,4,3,19,13,65,112,112,108,101,32,82,111,111,116,32,67,65,48,30,23,13,50,50,49,49,49,55,50,48,52,48,53,51,90,23,13,50,51,49,49,49,55,50,48,52,48,53,50,90,48,117,49,11,48,9,6,3,85,4,6,19,2,85,83,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,11,12,2,71,55,49,68,48,66,6,3,85,4,3,12,59,65,112,112,108,101,32,87,111,114,108,100,119,105,100,101,32,68,101,118,101,108,111,112,101,114,32,82,101,108,97,116,105,111,110,115,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,48,130,1,34,48,13,6,9,42,134,72,134,247,13,1,1,1,5,0,3,130,1,15,0,48,130,1,10,2,130,1,1,0,172,174,209,211,182,138,177,27,40,149,173,222,81,81,76,63,51,225,78,92,175,179,252,2,210,101,79,42,212,116,104,140,117,23,101,207,40,228,72,9,152,113,82,50,44,22,78,120,142,146,86,100,169,221,139,205,226,212,199,59,162,18,69,55,2,25,98,100,47,127,97,198,211,89,34,191,171,249,20,163,237,182,158,153,186,47,241,177,220,48,66,79,182,182,178,198,116,170,98,86,187,237,68,54,15,209,229,32,116,147,87,58,93,158,220,1,252,120,111,105,22,53,195,110,1,194,158,114,212,113,54,177,118,9,122,190,13,42,151,241,176,94,11,27,18,173,43,171,7,223,99,45,245,136,233,50,18,162,85,88,67,118,60,61,23,128,31,41,219,177,169,159,171,150,5,157,21,220,221,219,78,15,231,19,124,90,43,46,2,65,246,238,122,106,182,115,96,247,151,25,20,139,149,51,56,147,218,0,37,54,174,85,90,124,159,249,244,98,4,7,116,233,29,167,149,87,35,172,43,36,218,100,79,40,114,233,150,176,8,60,136,27,20,99,67,215,137,174,152,193,242,45,142,227,81,37,2,3,1,0,1,163,129,239,48,129,236,48,18,6,3,85,29,19,1,1,255,4,8,48,6,1,1,255,2,1,0,48,31,6,3,85,29,35,4,24,48,22,128,20,43,208,105,71,148,118,9,254,244,107,141,46,64,166,247,71,77,127,8,94,48,68,6,8,43,6,1,5,5,7,1,1,4,56,48,54,48,52,6,8,43,6,1,5,5,7,48,1,134,40,104,116,116,112,58,47,47,111,99,115,112,46,97,112,112,108,101,46,99,111,109,47,111,99,115,112,48,51,45,97,112,112,108,101,114,111,111,116,99,97,48,46,6,3,85,29,31,4,39,48,37,48,35,160,33,160,31,134,29,104,116,116,112,58,47,47,99,114,108,46,97,112,112,108,101,46,99,111,109,47,114,111,111,116,46,99,114,108,48,29,6,3,85,29,14,4,22,4,20,93,66,16,108,27,187,199,82,151,78,68,189,19,39,185,58,18,119,131,43,48,14,6,3,85,29,15,1,1,255,4,4,3,2,1,6,48,16,6,10,42,134,72,134,247,99,100,6,2,1,4,2,5,0,48,13,6,9,42,134,72,134,247,13,1,1,5,5,0,3,130,1,1,0,82,163,8,41,19,101,183,181,31,230,52,3,237,246,24,119,188,61,202,39,181,112,236,236,108,51,75,246,153,37,148,7,158,33,51,3,106,245,125,10,46,43,223,8,169,130,222,74,127,225,231,11,75,5,184,39,152,73,221,193,12,134,215,129,100,174,130,156,196,69,241,234,143,28,34,48,227,162,37,97,68,137,254,133,122,86,240,114,146,124,12,68,84,36,183,181,164,159,38,240,50,249,87,157,106,121,108,64,237,15,52,6,6,66,255,216,14,49,101,109,230,46,1,217,6,191,197,202,13,174,71,212,5,23,136,60,233,156,228,25,100,109,138,148,173,17,220,1,246,229,120,175,173,232,112,215,13,93,129,62,70,42,44,174,36,161,199,210,224,125,191,29,245,54,107,197,222,169,10,154,128,177,17,94,178,100,126,179,193,87,204,194,108,89,153,183,244,176,221,49,218,72,214,106,129,238,178,139,80,53,22,230,231,92,113,98,110,176,242,81,226,168,109,248,42,203,104,115,198,166,196,26,67,73,1,62,169,169,0,138,134,108,131,136,210,92,112,87,122,218,248,118,204,25,213,176] +static let AppleIncRootCertificate_cer: [UInt8] = [48,130,4,187,48,130,3,163,160,3,2,1,2,2,1,2,48,13,6,9,42,134,72,134,247,13,1,1,5,5,0,48,98,49,11,48,9,6,3,85,4,6,19,2,85,83,49,19,48,17,6,3,85,4,10,19,10,65,112,112,108,101,32,73,110,99,46,49,38,48,36,6,3,85,4,11,19,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,22,48,20,6,3,85,4,3,19,13,65,112,112,108,101,32,82,111,111,116,32,67,65,48,30,23,13,48,54,48,52,50,53,50,49,52,48,51,54,90,23,13,51,53,48,50,48,57,50,49,52,48,51,54,90,48,98,49,11,48,9,6,3,85,4,6,19,2,85,83,49,19,48,17,6,3,85,4,10,19,10,65,112,112,108,101,32,73,110,99,46,49,38,48,36,6,3,85,4,11,19,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,22,48,20,6,3,85,4,3,19,13,65,112,112,108,101,32,82,111,111,116,32,67,65,48,130,1,34,48,13,6,9,42,134,72,134,247,13,1,1,1,5,0,3,130,1,15,0,48,130,1,10,2,130,1,1,0,228,145,169,9,31,145,219,30,71,80,235,5,237,94,121,132,45,235,54,162,87,76,85,236,139,25,137,222,249,75,108,245,7,171,34,48,2,232,24,62,248,80,9,211,127,65,168,152,249,209,202,102,156,36,107,17,208,163,187,228,27,42,195,31,149,158,122,12,164,71,139,91,212,22,55,51,203,196,15,77,206,20,105,209,201,25,114,245,93,14,213,127,95,155,242,37,3,186,85,143,77,93,13,241,100,53,35,21,75,21,89,29,179,148,247,246,156,158,207,80,186,193,88,80,103,143,8,180,32,247,203,172,44,32,111,112,182,63,1,48,140,183,67,207,15,157,61,243,43,73,40,26,200,254,206,181,185,14,217,94,28,214,203,61,181,58,173,244,15,14,0,146,11,177,33,22,46,116,213,60,13,219,98,22,171,163,113,146,71,83,85,193,175,47,65,179,248,251,227,112,205,230,163,76,69,126,31,76,107,80,150,65,137,196,116,98,11,16,131,65,135,51,138,129,177,48,88,236,90,4,50,140,104,179,143,29,222,101,115,255,103,94,101,188,73,216,118,159,51,20,101,161,119,148,201,45,2,3,1,0,1,163,130,1,122,48,130,1,118,48,14,6,3,85,29,15,1,1,255,4,4,3,2,1,6,48,15,6,3,85,29,19,1,1,255,4,5,48,3,1,1,255,48,29,6,3,85,29,14,4,22,4,20,43,208,105,71,148,118,9,254,244,107,141,46,64,166,247,71,77,127,8,94,48,31,6,3,85,29,35,4,24,48,22,128,20,43,208,105,71,148,118,9,254,244,107,141,46,64,166,247,71,77,127,8,94,48,130,1,17,6,3,85,29,32,4,130,1,8,48,130,1,4,48,130,1,0,6,9,42,134,72,134,247,99,100,5,1,48,129,242,48,42,6,8,43,6,1,5,5,7,2,1,22,30,104,116,116,112,115,58,47,47,119,119,119,46,97,112,112,108,101,46,99,111,109,47,97,112,112,108,101,99,97,47,48,129,195,6,8,43,6,1,5,5,7,2,2,48,129,182,26,129,179,82,101,108,105,97,110,99,101,32,111,110,32,116,104,105,115,32,99,101,114,116,105,102,105,99,97,116,101,32,98,121,32,97,110,121,32,112,97,114,116,121,32,97,115,115,117,109,101,115,32,97,99,99,101,112,116,97,110,99,101,32,111,102,32,116,104,101,32,116,104,101,110,32,97,112,112,108,105,99,97,98,108,101,32,115,116,97,110,100,97,114,100,32,116,101,114,109,115,32,97,110,100,32,99,111,110,100,105,116,105,111,110,115,32,111,102,32,117,115,101,44,32,99,101,114,116,105,102,105,99,97,116,101,32,112,111,108,105,99,121,32,97,110,100,32,99,101,114,116,105,102,105,99,97,116,105,111,110,32,112,114,97,99,116,105,99,101,32,115,116,97,116,101,109,101,110,116,115,46,48,13,6,9,42,134,72,134,247,13,1,1,5,5,0,3,130,1,1,0,92,54,153,76,45,120,183,237,140,155,220,243,119,155,242,118,210,119,48,79,193,31,133,131,133,27,153,61,71,55,242,169,155,64,142,44,212,177,144,18,216,190,244,115,155,238,210,100,15,203,121,79,52,216,162,62,249,120,255,107,200,7,236,125,57,131,139,83,32,211,56,196,177,191,154,79,10,107,255,43,252,89,167,5,9,124,23,64,86,17,30,116,211,183,139,35,59,71,163,213,111,36,226,235,209,183,112,223,15,69,225,39,202,241,109,120,237,231,181,23,23,168,220,126,34,53,202,37,213,217,15,214,107,212,162,36,35,17,247,161,172,143,115,129,96,198,27,91,9,47,146,178,248,68,72,240,96,56,158,21,245,61,38,103,32,138,51,106,247,13,130,207,222,235,163,47,249,83,106,91,100,192,99,51,119,247,58,7,44,86,235,218,15,33,14,218,186,115,25,79,181,217,54,127,193,135,85,217,167,153,185,50,66,251,216,213,113,158,126,161,82,183,27,189,147,66,36,18,42,199,15,29,182,77,156,94,99,200,75,128,23,80,170,138,213,218,228,252,208,9,7,55,176,117,117,33] +static let AppleWWDRCAG4_cer: [UInt8] = [48,130,4,85,48,130,3,61,160,3,2,1,2,2,20,19,220,119,149,82,113,229,61,198,50,232,204,255,229,33,243,204,197,206,210,48,13,6,9,42,134,72,134,247,13,1,1,11,5,0,48,98,49,11,48,9,6,3,85,4,6,19,2,85,83,49,19,48,17,6,3,85,4,10,19,10,65,112,112,108,101,32,73,110,99,46,49,38,48,36,6,3,85,4,11,19,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,22,48,20,6,3,85,4,3,19,13,65,112,112,108,101,32,82,111,111,116,32,67,65,48,30,23,13,50,48,49,50,49,54,49,57,51,54,48,52,90,23,13,51,48,49,50,49,48,48,48,48,48,48,48,90,48,117,49,68,48,66,6,3,85,4,3,12,59,65,112,112,108,101,32,87,111,114,108,100,119,105,100,101,32,68,101,118,101,108,111,112,101,114,32,82,101,108,97,116,105,111,110,115,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,11,48,9,6,3,85,4,11,12,2,71,52,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,6,19,2,85,83,48,130,1,34,48,13,6,9,42,134,72,134,247,13,1,1,1,5,0,3,130,1,15,0,48,130,1,10,2,130,1,1,0,208,31,120,170,122,39,50,176,70,95,231,23,118,216,160,157,14,41,142,173,61,50,165,196,107,55,201,228,65,145,106,183,121,113,93,12,52,4,96,117,247,174,67,143,71,196,134,30,8,232,191,214,57,82,47,30,103,252,113,241,130,109,60,126,6,82,118,157,44,188,213,67,233,177,180,188,64,58,120,81,93,81,161,37,225,190,108,145,157,107,33,89,24,65,213,15,141,109,65,42,57,74,33,224,144,159,78,19,79,208,140,154,50,184,215,106,146,30,37,106,164,50,206,34,25,133,5,96,220,2,74,242,90,235,119,121,2,125,192,151,84,108,146,142,118,1,230,70,143,229,230,42,251,162,176,173,24,173,109,51,133,56,35,139,234,138,150,237,159,174,102,79,163,12,64,39,109,149,208,98,136,217,67,41,39,253,237,164,191,83,46,144,21,101,60,217,46,98,100,51,29,108,106,221,142,33,170,164,95,21,198,48,237,95,230,140,54,146,148,183,220,57,2,0,251,100,140,212,12,129,242,63,213,52,151,135,117,38,194,111,174,3,99,33,12,123,212,27,177,98,197,2,156,189,253,175,2,3,1,0,1,163,129,239,48,129,236,48,18,6,3,85,29,19,1,1,255,4,8,48,6,1,1,255,2,1,0,48,31,6,3,85,29,35,4,24,48,22,128,20,43,208,105,71,148,118,9,254,244,107,141,46,64,166,247,71,77,127,8,94,48,68,6,8,43,6,1,5,5,7,1,1,4,56,48,54,48,52,6,8,43,6,1,5,5,7,48,1,134,40,104,116,116,112,58,47,47,111,99,115,112,46,97,112,112,108,101,46,99,111,109,47,111,99,115,112,48,51,45,97,112,112,108,101,114,111,111,116,99,97,48,46,6,3,85,29,31,4,39,48,37,48,35,160,33,160,31,134,29,104,116,116,112,58,47,47,99,114,108,46,97,112,112,108,101,46,99,111,109,47,114,111,111,116,46,99,114,108,48,29,6,3,85,29,14,4,22,4,20,91,217,250,29,231,154,26,11,163,153,118,34,80,134,62,145,200,91,119,168,48,14,6,3,85,29,15,1,1,255,4,4,3,2,1,6,48,16,6,10,42,134,72,134,247,99,100,6,2,1,4,2,5,0,48,13,6,9,42,134,72,134,247,13,1,1,11,5,0,3,130,1,1,0,63,86,61,158,229,182,195,121,230,69,32,104,189,191,115,139,44,18,158,2,227,174,128,34,140,4,30,195,82,200,112,128,168,251,206,167,176,213,66,68,130,3,130,79,6,252,59,73,20,251,216,116,82,133,175,167,157,33,231,1,18,3,159,205,64,88,208,1,215,191,50,131,43,83,88,40,60,238,156,159,84,118,61,100,39,198,126,141,29,56,77,45,174,129,230,185,165,184,156,137,148,247,159,199,135,165,81,102,52,27,57,113,57,36,227,135,103,239,165,177,104,123,140,238,61,247,174,182,123,226,210,255,223,97,198,106,117,73,149,34,68,168,4,252,148,184,11,46,57,17,73,18,209,229,129,234,89,0,79,91,60,90,54,218,122,34,115,9,9,105,205,192,124,234,226,36,254,152,68,184,248,239,178,113,63,26,93,212,93,126,51,77,156,29,36,190,0,240,144,3,88,153,65,61,49,90,98,64,175,57,168,81,67,146,171,4,168,156,194,77,177,75,210,171,124,74,95,235,157,59,188,79,136,64,6,19,255,144,23,138,8,71,41,232,98,152,41,165,79,17,5,105,58,207,242,159] +static let AppleWWDRCAG6_cer: [UInt8] = [48,130,3,22,48,130,2,156,160,3,2,1,2,2,20,34,193,161,71,10,116,115,105,239,83,134,18,201,198,159,61,56,243,108,215,48,10,6,8,42,134,72,206,61,4,3,3,48,103,49,27,48,25,6,3,85,4,3,12,18,65,112,112,108,101,32,82,111,111,116,32,67,65,32,45,32,71,51,49,38,48,36,6,3,85,4,11,12,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,6,19,2,85,83,48,30,23,13,50,49,48,51,49,55,50,48,51,55,49,48,90,23,13,51,54,48,51,49,57,48,48,48,48,48,48,90,48,117,49,68,48,66,6,3,85,4,3,12,59,65,112,112,108,101,32,87,111,114,108,100,119,105,100,101,32,68,101,118,101,108,111,112,101,114,32,82,101,108,97,116,105,111,110,115,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,11,48,9,6,3,85,4,11,12,2,71,54,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,6,19,2,85,83,48,118,48,16,6,7,42,134,72,206,61,2,1,6,5,43,129,4,0,34,3,98,0,4,110,196,10,11,222,15,174,85,166,101,121,215,130,220,115,117,82,75,241,61,18,25,137,224,13,17,169,158,9,247,55,163,211,240,147,4,111,177,67,139,134,193,152,248,66,157,157,255,178,174,143,23,247,6,112,165,214,176,190,111,58,215,145,135,34,120,231,29,192,72,254,220,56,31,154,106,143,116,125,202,113,74,255,196,100,86,231,136,6,205,129,145,161,34,26,72,163,129,250,48,129,247,48,18,6,3,85,29,19,1,1,255,4,8,48,6,1,1,255,2,1,0,48,31,6,3,85,29,35,4,24,48,22,128,20,187,176,222,161,88,51,136,154,164,138,153,222,190,189,235,175,218,203,36,171,48,70,6,8,43,6,1,5,5,7,1,1,4,58,48,56,48,54,6,8,43,6,1,5,5,7,48,1,134,42,104,116,116,112,58,47,47,111,99,115,112,46,97,112,112,108,101,46,99,111,109,47,111,99,115,112,48,51,45,97,112,112,108,101,114,111,111,116,99,97,103,51,48,55,6,3,85,29,31,4,48,48,46,48,44,160,42,160,40,134,38,104,116,116,112,58,47,47,99,114,108,46,97,112,112,108,101,46,99,111,109,47,97,112,112,108,101,114,111,111,116,99,97,103,51,46,99,114,108,48,29,6,3,85,29,14,4,22,4,20,63,47,148,35,81,211,80,201,154,40,61,237,176,124,229,207,165,144,98,153,48,14,6,3,85,29,15,1,1,255,4,4,3,2,1,6,48,16,6,10,42,134,72,134,247,99,100,6,2,1,4,2,5,0,48,10,6,8,42,134,72,206,61,4,3,3,3,104,0,48,101,2,48,64,94,20,170,228,140,138,162,3,2,62,220,56,247,64,90,7,174,251,9,161,6,37,123,159,64,31,196,169,145,157,232,36,163,136,43,78,158,227,19,109,117,87,155,28,92,197,249,2,49,0,211,72,252,90,173,149,239,54,185,49,83,116,228,80,194,55,223,44,239,21,85,81,79,140,49,88,144,189,247,131,172,117,198,120,249,17,141,177,128,242,71,152,38,118,189,39,12,104] +static let AppleWWDRCAG8_cer: [UInt8] = [48,130,4,85,48,130,3,61,160,3,2,1,2,2,20,84,181,11,175,121,13,141,127,140,175,104,76,86,47,80,105,10,26,186,95,48,13,6,9,42,134,72,134,247,13,1,1,5,5,0,48,98,49,11,48,9,6,3,85,4,6,19,2,85,83,49,19,48,17,6,3,85,4,10,19,10,65,112,112,108,101,32,73,110,99,46,49,38,48,36,6,3,85,4,11,19,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,22,48,20,6,3,85,4,3,19,13,65,112,112,108,101,32,82,111,111,116,32,67,65,48,30,23,13,50,51,48,54,50,48,50,51,51,55,49,53,90,23,13,50,53,48,49,50,52,48,48,48,48,48,48,90,48,117,49,11,48,9,6,3,85,4,6,19,2,85,83,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,11,12,2,71,56,49,68,48,66,6,3,85,4,3,12,59,65,112,112,108,101,32,87,111,114,108,100,119,105,100,101,32,68,101,118,101,108,111,112,101,114,32,82,101,108,97,116,105,111,110,115,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,48,130,1,34,48,13,6,9,42,134,72,134,247,13,1,1,1,5,0,3,130,1,15,0,48,130,1,10,2,130,1,1,0,208,64,16,212,8,249,82,33,99,79,60,104,247,212,156,130,2,140,131,104,188,183,141,127,136,231,66,77,44,239,34,165,194,137,222,189,12,215,77,97,80,45,57,228,48,115,165,210,213,104,154,51,242,239,217,42,173,46,31,3,186,248,165,241,66,11,87,46,70,198,208,88,211,112,95,247,178,55,239,106,31,62,191,137,57,194,173,254,245,5,251,88,150,49,200,135,44,213,35,186,117,176,44,245,171,160,186,242,32,12,76,235,36,99,105,233,34,64,240,142,88,91,222,100,123,137,27,81,18,4,224,23,178,89,205,223,241,231,206,175,44,50,232,105,193,208,8,126,29,162,71,115,209,75,59,253,185,133,220,71,167,40,130,121,41,72,137,2,40,226,193,236,44,31,91,134,252,216,182,203,113,192,115,1,75,252,5,17,17,108,1,3,52,90,246,129,194,94,31,124,20,14,222,63,107,33,203,79,245,215,97,230,132,3,146,188,231,37,41,158,205,91,12,193,219,227,196,58,217,118,87,172,47,56,20,25,200,183,177,162,85,79,95,78,142,25,89,169,218,253,114,218,176,4,33,5,2,3,1,0,1,163,129,239,48,129,236,48,18,6,3,85,29,19,1,1,255,4,8,48,6,1,1,255,2,1,0,48,31,6,3,85,29,35,4,24,48,22,128,20,43,208,105,71,148,118,9,254,244,107,141,46,64,166,247,71,77,127,8,94,48,68,6,8,43,6,1,5,5,7,1,1,4,56,48,54,48,52,6,8,43,6,1,5,5,7,48,1,134,40,104,116,116,112,58,47,47,111,99,115,112,46,97,112,112,108,101,46,99,111,109,47,111,99,115,112,48,51,45,97,112,112,108,101,114,111,111,116,99,97,48,46,6,3,85,29,31,4,39,48,37,48,35,160,33,160,31,134,29,104,116,116,112,58,47,47,99,114,108,46,97,112,112,108,101,46,99,111,109,47,114,111,111,116,46,99,114,108,48,29,6,3,85,29,14,4,22,4,20,181,189,188,128,196,12,227,56,164,244,183,173,35,179,239,68,206,185,90,133,48,14,6,3,85,29,15,1,1,255,4,4,3,2,1,6,48,16,6,10,42,134,72,134,247,99,100,6,2,1,4,2,5,0,48,13,6,9,42,134,72,134,247,13,1,1,5,5,0,3,130,1,1,0,76,179,235,122,57,148,74,149,102,250,22,49,234,13,138,143,93,66,56,198,7,205,232,201,34,172,252,8,66,126,95,147,95,49,42,168,34,193,118,25,43,238,174,114,233,189,86,207,73,36,146,8,247,205,25,117,195,249,26,39,13,14,220,41,141,63,182,219,165,79,76,114,196,96,145,137,17,126,36,149,67,34,96,103,188,223,255,171,229,60,22,23,157,130,20,78,157,244,117,254,80,185,173,5,178,184,121,246,149,214,16,229,246,83,244,207,35,230,119,15,236,184,75,105,248,226,158,2,49,217,142,217,202,190,99,94,95,225,160,105,206,193,205,110,73,163,120,97,175,57,60,106,75,10,52,37,100,213,48,181,60,171,221,17,129,57,113,232,192,195,88,128,17,16,26,17,210,194,188,89,240,138,217,53,115,192,230,109,228,201,125,98,20,62,130,2,169,16,105,55,121,97,111,240,70,148,62,53,245,171,172,99,208,216,94,89,87,105,209,184,75,110,93,212,95,19,156,112,78,37,17,220,36,44,30,22,158,150,90,4,237,69,238,58,238,103,93,143,195,134,176,245,76,31,104,16] +static let AppleRootCA_G3_cer: [UInt8] = [48,130,2,67,48,130,1,201,160,3,2,1,2,2,8,45,197,252,136,210,197,75,149,48,10,6,8,42,134,72,206,61,4,3,3,48,103,49,27,48,25,6,3,85,4,3,12,18,65,112,112,108,101,32,82,111,111,116,32,67,65,32,45,32,71,51,49,38,48,36,6,3,85,4,11,12,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,6,19,2,85,83,48,30,23,13,49,52,48,52,51,48,49,56,49,57,48,54,90,23,13,51,57,48,52,51,48,49,56,49,57,48,54,90,48,103,49,27,48,25,6,3,85,4,3,12,18,65,112,112,108,101,32,82,111,111,116,32,67,65,32,45,32,71,51,49,38,48,36,6,3,85,4,11,12,29,65,112,112,108,101,32,67,101,114,116,105,102,105,99,97,116,105,111,110,32,65,117,116,104,111,114,105,116,121,49,19,48,17,6,3,85,4,10,12,10,65,112,112,108,101,32,73,110,99,46,49,11,48,9,6,3,85,4,6,19,2,85,83,48,118,48,16,6,7,42,134,72,206,61,2,1,6,5,43,129,4,0,34,3,98,0,4,152,233,47,61,64,114,164,237,147,34,114,129,19,28,221,16,149,241,197,163,78,113,220,20,22,217,14,229,166,5,42,119,100,123,95,78,56,211,187,28,68,181,127,245,31,182,50,98,93,201,233,132,91,79,48,79,17,90,0,253,88,88,12,165,245,15,44,77,7,71,19,117,218,151,151,151,111,49,92,237,43,157,123,32,59,216,185,84,217,94,153,164,58,81,10,49,163,66,48,64,48,29,6,3,85,29,14,4,22,4,20,187,176,222,161,88,51,136,154,164,138,153,222,190,189,235,175,218,203,36,171,48,15,6,3,85,29,19,1,1,255,4,5,48,3,1,1,255,48,14,6,3,85,29,15,1,1,255,4,4,3,2,1,6,48,10,6,8,42,134,72,206,61,4,3,3,3,104,0,48,101,2,49,0,131,233,193,196,22,94,26,93,52,24,217,237,239,244,108,14,0,70,75,184,223,178,70,17,197,15,253,230,122,140,161,166,107,206,194,3,212,156,245,147,198,116,184,106,223,170,35,21,2,48,109,102,138,16,202,212,13,212,79,205,141,67,62,180,138,99,165,51,110,227,109,218,23,183,100,31,200,83,38,249,136,98,116,57,11,23,91,203,81,168,12,232,24,3,231,162,178,40] +} \ No newline at end of file diff --git a/Sources/Workspace/Workspace+Configuration.swift b/Sources/Workspace/Workspace+Configuration.swift index 671ffba09f4..9fecb0a0888 100644 --- a/Sources/Workspace/Workspace+Configuration.swift +++ b/Sources/Workspace/Workspace+Configuration.swift @@ -792,6 +792,12 @@ public struct WorkspaceConfiguration { /// Whether or not to use prebuilt swift-syntax for macros public var usePrebuilts: Bool + /// String URL to allow override of the prebuilts download location + public var prebuiltsDownloadURL: String? + + /// Path to root certificate used when validating the manifest signing during testing + public var prebuiltsRootCertPath: String? + /// Whether to omit unused dependencies. public var pruneDependencies: Bool @@ -812,6 +818,8 @@ public struct WorkspaceConfiguration { defaultRegistry: Registry?, manifestImportRestrictions: (startingToolsVersion: ToolsVersion, allowedImports: [String])?, usePrebuilts: Bool, + prebuiltsDownloadURL: String?, + prebuiltsRootCertPath: String?, pruneDependencies: Bool, traitConfiguration: TraitConfiguration ) { @@ -828,6 +836,8 @@ public struct WorkspaceConfiguration { self.defaultRegistry = defaultRegistry self.manifestImportRestrictions = manifestImportRestrictions self.usePrebuilts = usePrebuilts + self.prebuiltsDownloadURL = prebuiltsDownloadURL + self.prebuiltsRootCertPath = prebuiltsRootCertPath self.pruneDependencies = pruneDependencies self.traitConfiguration = traitConfiguration } @@ -848,6 +858,8 @@ public struct WorkspaceConfiguration { defaultRegistry: .none, manifestImportRestrictions: .none, usePrebuilts: false, + prebuiltsDownloadURL: nil, + prebuiltsRootCertPath: nil, pruneDependencies: false, traitConfiguration: .default ) diff --git a/Sources/Workspace/Workspace+Prebuilts.swift b/Sources/Workspace/Workspace+Prebuilts.swift index 8cca15bde2c..9c16a5edb51 100644 --- a/Sources/Workspace/Workspace+Prebuilts.swift +++ b/Sources/Workspace/Workspace+Prebuilts.swift @@ -87,17 +87,20 @@ extension Workspace { case macos_x86_64 case windows_aarch64 case windows_x86_64 - // noble is currently missing + case ubuntu_noble_aarch64 + case ubuntu_noble_x86_64 case ubuntu_jammy_aarch64 case ubuntu_jammy_x86_64 case ubuntu_focal_aarch64 case ubuntu_focal_x86_64 - // bookworm is currently missing - // fedora39 is currently missing + case fedora_39_aarch64 + case fedora_39_x86_64 case amazonlinux2_aarch64 case amazonlinux2_x86_64 case rhel_ubi9_aarch64 case rhel_ubi9_x86_64 + case debian_12_aarch64 + case debian_12_x86_64 public enum Arch: String { case x86_64 @@ -109,35 +112,16 @@ extension Workspace { case windows case linux } + } + } - public var arch: Arch { - switch self { - case .macos_aarch64, .windows_aarch64, - .ubuntu_jammy_aarch64, .ubuntu_focal_aarch64, - .amazonlinux2_aarch64, - .rhel_ubi9_aarch64: - return .aarch64 - case .macos_x86_64, .windows_x86_64, - .ubuntu_jammy_x86_64, .ubuntu_focal_x86_64, - .amazonlinux2_x86_64, - .rhel_ubi9_x86_64: - return .x86_64 - } - } + public struct SignedPrebuiltsManifest: Codable { + public var manifest: PrebuiltsManifest + public var signature: ManifestSignature - public var os: OS { - switch self { - case .macos_aarch64, .macos_x86_64: - return .macos - case .windows_aarch64, .windows_x86_64: - return .windows - case .ubuntu_jammy_aarch64, .ubuntu_jammy_x86_64, - .ubuntu_focal_aarch64, .ubuntu_focal_x86_64, - .amazonlinux2_aarch64, .amazonlinux2_x86_64, - .rhel_ubi9_aarch64, .rhel_ubi9_x86_64: - return .linux - } - } + public init(manifest: PrebuiltsManifest, signature: ManifestSignature) { + self.manifest = manifest + self.signature = signature } } @@ -146,15 +130,21 @@ extension Workspace { let httpClient: HTTPClient? let archiver: Archiver? let useCache: Bool? + let hostPlatform: PrebuiltsManifest.Platform? + let rootCertPath: AbsolutePath? public init( httpClient: HTTPClient? = .none, archiver: Archiver? = .none, - useCache: Bool? = .none + useCache: Bool? = .none, + hostPlatform: PrebuiltsManifest.Platform? = nil, + rootCertPath: AbsolutePath? = nil ) { self.httpClient = httpClient self.archiver = archiver self.useCache = useCache + self.hostPlatform = hostPlatform + self.rootCertPath = rootCertPath } } @@ -170,50 +160,72 @@ extension Workspace { private let cachePath: AbsolutePath? private let delegate: Delegate? private let hashAlgorithm: HashAlgorithm = SHA256() + private let prebuiltsDownloadURL: URL + private let rootCertPath: AbsolutePath? + let hostPlatform: PrebuiltsManifest.Platform init( fileSystem: FileSystem, + hostPlatform: PrebuiltsManifest.Platform, authorizationProvider: AuthorizationProvider?, scratchPath: AbsolutePath, cachePath: AbsolutePath?, customHTTPClient: HTTPClient?, customArchiver: Archiver?, - delegate: Delegate? + delegate: Delegate?, + prebuiltsDownloadURL: String?, + rootCertPath: AbsolutePath? ) { self.fileSystem = fileSystem + self.hostPlatform = hostPlatform self.authorizationProvider = authorizationProvider self.httpClient = customHTTPClient ?? HTTPClient() + +#if os(Linux) + self.archiver = customArchiver ?? TarArchiver(fileSystem: fileSystem) +#else self.archiver = customArchiver ?? ZipArchiver(fileSystem: fileSystem) +#endif + self.scratchPath = scratchPath self.cachePath = cachePath self.delegate = delegate + if let prebuiltsDownloadURL, let url = URL(string: prebuiltsDownloadURL) { + self.prebuiltsDownloadURL = url + } else { + self.prebuiltsDownloadURL = URL(string: "https://download.swift.org/prebuilts")! + } + self.rootCertPath = rootCertPath + + self.prebuiltPackages = [ + // TODO: we should have this in a manifest somewhere, not hardcoded like this + .init( + identity: .plain("swift-syntax"), + packageRefs: [ + .init( + identity: .plain("swift-syntax"), + kind: .remoteSourceControl("https://github.com/swiftlang/swift-syntax.git") + ), + .init( + identity: .plain("swift-syntax"), + kind: .remoteSourceControl("git@github.com:swiftlang/swift-syntax.git") + ), + ] + ), + ] } struct PrebuiltPackage { - let packageRef: PackageReference - let prebuiltsURL: URL + let identity: PackageIdentity + let packageRefs: [PackageReference] } - private let prebuiltPackages: [PrebuiltPackage] = [ - .init( - packageRef: .init( - identity: .plain("swift-syntax"), - kind: .remoteSourceControl("https://github.com/swiftlang/swift-syntax.git") - ), - prebuiltsURL: URL( - string: - "https://github.com/dschaefer2/swift-syntax/releases/download" - )! - ), - ] + private let prebuiltPackages: [PrebuiltPackage] // Version of the compiler we're building against - private let swiftVersion = - "\(SwiftVersion.current.major).\(SwiftVersion.current.minor)" + private let swiftVersion = "\(SwiftVersion.current.major).\(SwiftVersion.current.minor)" - fileprivate func findPrebuilts(packages: [PackageReference]) - -> [PrebuiltPackage] - { + fileprivate func findPrebuilts(packages: [PackageReference]) -> [PrebuiltPackage] { var prebuilts: [PrebuiltPackage] = [] for packageRef in packages { guard case let .remoteSourceControl(pkgURL) = packageRef.kind else { @@ -222,21 +234,23 @@ extension Workspace { } if let prebuilt = prebuiltPackages.first(where: { - guard case let .remoteSourceControl(prebuiltURL) = $0.packageRef.kind, - $0.packageRef.identity == packageRef.identity else { - return false - } - - if pkgURL == prebuiltURL { - return true - } else if !pkgURL.lastPathComponent.hasSuffix(".git") { - // try with the git extension - // TODO: Does this need to be in the PackageRef Equatable? - let gitURL = SourceControlURL(pkgURL.absoluteString + ".git") - return gitURL == prebuiltURL - } else { - return false - } + $0.packageRefs.contains(where: { + guard case let .remoteSourceControl(prebuiltURL) = $0.kind, + $0.identity == packageRef.identity else { + return false + } + + if pkgURL == prebuiltURL { + return true + } else if !pkgURL.lastPathComponent.hasSuffix(".git") { + // try with the git extension + // TODO: Does this need to be in the PackageRef Equatable? + let gitURL = SourceControlURL(pkgURL.absoluteString + ".git") + return gitURL == prebuiltURL + } else { + return false + } + }) }) { prebuilts.append(prebuilt) } @@ -250,17 +264,41 @@ extension Workspace { observabilityScope: ObservabilityScope ) async throws -> PrebuiltsManifest? { let manifestFile = swiftVersion + "-manifest.json" - let prebuiltsDir = cachePath ?? scratchPath - let destination = prebuiltsDir.appending( - components: package.packageRef.identity.description, - manifestFile - ) - if fileSystem.exists(destination) { + let manifestPath = try RelativePath(validating: "\(package.identity)/\(version)/\(manifestFile)") + let destination = scratchPath.appending(manifestPath) + let cacheDest = cachePath?.appending(manifestPath) + + func loadManifest() async throws -> PrebuiltsManifest? { do { - return try JSONDecoder().decode( - PrebuiltsManifest.self, - from: try Data(contentsOf: destination.asURL) + let signedManifest = try JSONDecoder().decode( + path: destination, + fileSystem: fileSystem, + as: SignedPrebuiltsManifest.self ) + + // Check the signature + // Ignore errors coming from the certificate loading, that will shutdown the build + // instead of letting it continue with build from source. + if let rootCertPath { + try await withTemporaryDirectory(fileSystem: fileSystem) { tmpDir in + try fileSystem.copy(from: rootCertPath, to: tmpDir.appending(rootCertPath.basename)) + let validator = ManifestSigning(trustedRootCertsDir: tmpDir, observabilityScope: ObservabilitySystem.NOOP) + try await validator.validate( + manifest: signedManifest.manifest, + signature: signedManifest.signature, + fileSystem: fileSystem + ) + }.value + } else { + let validator = ManifestSigning(observabilityScope: ObservabilitySystem.NOOP) + try await validator.validate( + manifest: signedManifest.manifest, + signature: signedManifest.signature, + fileSystem: fileSystem + ) + } + + return signedManifest.manifest } catch { // redownload it observabilityScope.emit( @@ -268,56 +306,85 @@ extension Workspace { underlyingError: error ) try fileSystem.removeFileTree(destination) + return nil } } - try fileSystem.createDirectory( - destination.parentDirectory, - recursive: true - ) - let manifestURL = package.prebuiltsURL.appending( - components: version.description, - manifestFile - ) - var headers = HTTPClientHeaders() - headers.add(name: "Accept", value: "application/json") - var request = HTTPClient.Request.download( - url: manifestURL, - headers: headers, - fileSystem: self.fileSystem, - destination: destination - ) - request.options.authorizationProvider = - self.authorizationProvider?.httpAuthorizationHeader(for:) - request.options.retryStrategy = .exponentialBackoff( - maxAttempts: 3, - baseDelay: .milliseconds(50) - ) - request.options.validResponseCodes = [200] + if fileSystem.exists(destination), let manifest = try? await loadManifest() { + return manifest + } else if let cacheDest, fileSystem.exists(cacheDest) { + // Pull it out of the cache + try fileSystem.createDirectory(destination.parentDirectory, recursive: true) + try fileSystem.copy(from: cacheDest, to: destination) - do { - _ = try await self.httpClient.execute(request) { _, _ in - // TODO: send to delegate + if let manifest = try? await loadManifest() { + return manifest } - } catch { - observabilityScope.emit( - info: "Prebuilt \(manifestFile)", - underlyingError: error - ) + } else if fileSystem.exists(destination.parentDirectory) { + // We tried previously and were not able to find the manifest. + // Don't try again to avoid excessive server traffic return nil } - do { - let data = try fileSystem.readFileContents(destination) - return try JSONDecoder().decode( - PrebuiltsManifest.self, - from: Data(data.contents) + try fileSystem.createDirectory(destination.parentDirectory, recursive: true) + + let manifestURL = self.prebuiltsDownloadURL.appending( + components: package.identity.description, version.description, manifestFile + ) + + if manifestURL.scheme == "file" { + let sourcePath = try AbsolutePath(validating: manifestURL.path) + if fileSystem.exists(sourcePath) { + // simply copy it over + try fileSystem.copy(from: sourcePath, to: destination) + } else { + return nil + } + } else { + var headers = HTTPClientHeaders() + headers.add(name: "Accept", value: "application/json") + var request = HTTPClient.Request.download( + url: manifestURL, + headers: headers, + fileSystem: self.fileSystem, + destination: destination ) - } catch { - observabilityScope.emit( - info: "Failed to decode prebuilt manifest", - underlyingError: error + request.options.authorizationProvider = + self.authorizationProvider?.httpAuthorizationHeader(for:) + request.options.retryStrategy = .exponentialBackoff( + maxAttempts: 3, + baseDelay: .milliseconds(50) ) + request.options.validResponseCodes = [200] + + do { + _ = try await self.httpClient.execute(request) { _, _ in + // TODO: send to delegate + } + } catch { + observabilityScope.emit( + info: "Prebuilt \(manifestFile)", + underlyingError: error + ) + // Create an empty manifest so we don't keep trying to download it + let manifest = PrebuiltsManifest(libraries: []) + try? fileSystem.writeFileContents(destination, data: JSONEncoder().encode(manifest)) + return nil + } + } + + if let manifest = try await loadManifest() { + // Cache the manifest + if let cacheDest { + if fileSystem.exists(cacheDest) { + try fileSystem.removeFileTree(cacheDest) + } + try fileSystem.createDirectory(cacheDest.parentDirectory, recursive: true) + try fileSystem.copy(from: destination, to: cacheDest) + } + + return manifest + } else { return nil } } @@ -335,102 +402,125 @@ extension Workspace { artifact: PrebuiltsManifest.Library.Artifact, observabilityScope: ObservabilityScope ) async throws -> AbsolutePath? { - let artifactName = - "\(swiftVersion)-\(library.name)-\(artifact.platform.rawValue)" - let scratchDir = scratchPath.appending( - package.packageRef.identity.description - ) + let artifactName = "\(swiftVersion)-\(library.name)-\(artifact.platform.rawValue)" + let scratchDir = scratchPath.appending(components: package.identity.description, version.description) + let artifactDir = scratchDir.appending(artifactName) guard !fileSystem.exists(artifactDir) else { return artifactDir } - let artifactFile = artifactName + ".zip" - let prebuiltsDir = cachePath ?? scratchPath - let destination = prebuiltsDir.appending( - components: package.packageRef.identity.description, - artifactFile - ) + let artifactFile = artifactName + (hostPlatform.os == .linux ? ".tar.gz" : ".zip") + let destination = scratchDir.appending(artifactFile) + let cacheFile = cachePath?.appending(components: package.identity.description, version.description, artifactFile) let zipExists = fileSystem.exists(destination) if try (!zipExists || !check(path: destination, checksum: artifact.checksum)) { - if zipExists { - observabilityScope.emit(info: "Prebuilt artifact \(artifactFile) checksum mismatch, redownloading.") - try fileSystem.removeFileTree(destination) - } + try fileSystem.createDirectory(destination.parentDirectory, recursive: true) + + if let cacheFile, fileSystem.exists(cacheFile), try check(path: cacheFile, checksum: artifact.checksum) { + // Copy over the cached file + observabilityScope.emit(info: "Using cached \(artifactFile)") + try fileSystem.copy(from: cacheFile, to: destination) + } else { + if zipExists { + // Exists but failed checksum + observabilityScope.emit(info: "Prebuilt artifact \(artifactFile) checksum mismatch, redownloading.") + try fileSystem.removeFileTree(destination) + } - try fileSystem.createDirectory( - destination.parentDirectory, - recursive: true - ) + // Download + let artifactURL = self.prebuiltsDownloadURL.appending( + components: package.identity.description, version.description, artifactFile + ) - // Download - let artifactURL = package.prebuiltsURL.appending( - components: version.description, - artifactFile - ) - let fetchStart = DispatchTime.now() - var headers = HTTPClientHeaders() - headers.add(name: "Accept", value: "application/octet-stream") - var request = HTTPClient.Request.download( - url: artifactURL, - headers: headers, - fileSystem: self.fileSystem, - destination: destination - ) - request.options.authorizationProvider = - self.authorizationProvider?.httpAuthorizationHeader(for:) - request.options.retryStrategy = .exponentialBackoff( - maxAttempts: 3, - baseDelay: .milliseconds(50) - ) - request.options.validResponseCodes = [200] + let fetchStart = DispatchTime.now() + if artifactURL.scheme == "file" { + let artifactPath = try AbsolutePath(validating: artifactURL.path) + if fileSystem.exists(artifactPath) { + try fileSystem.copy(from: artifactPath, to: destination) + self.delegate?.didDownloadPrebuilt( + from: artifactURL.absoluteString, + result: .success((destination, false)), + duration: fetchStart.distance(to: .now()) + ) + } else { + return nil + } + } else { + var headers = HTTPClientHeaders() + headers.add(name: "Accept", value: "application/octet-stream") + var request = HTTPClient.Request.download( + url: artifactURL, + headers: headers, + fileSystem: self.fileSystem, + destination: destination + ) + request.options.authorizationProvider = + self.authorizationProvider?.httpAuthorizationHeader(for:) + request.options.retryStrategy = .exponentialBackoff( + maxAttempts: 3, + baseDelay: .milliseconds(50) + ) + request.options.validResponseCodes = [200] - self.delegate?.willDownloadPrebuilt( - from: artifactURL.absoluteString, - fromCache: false - ) - do { - _ = try await self.httpClient.execute(request) { - bytesDownloaded, - totalBytesToDownload in - self.delegate?.downloadingPrebuilt( + self.delegate?.willDownloadPrebuilt( + from: artifactURL.absoluteString, + fromCache: false + ) + do { + _ = try await self.httpClient.execute(request) { + bytesDownloaded, + totalBytesToDownload in + self.delegate?.downloadingPrebuilt( + from: artifactURL.absoluteString, + bytesDownloaded: bytesDownloaded, + totalBytesToDownload: totalBytesToDownload + ) + } + } catch { + observabilityScope.emit( + info: "Prebuilt artifact \(artifactFile)", + underlyingError: error + ) + self.delegate?.didDownloadPrebuilt( + from: artifactURL.absoluteString, + result: .failure(error), + duration: fetchStart.distance(to: .now()) + ) + return nil + } + + // Check the checksum + if try !check(path: destination, checksum: artifact.checksum) { + let errorString = + "Prebuilt artifact \(artifactFile) checksum mismatch" + observabilityScope.emit(info: errorString) + self.delegate?.didDownloadPrebuilt( + from: artifactURL.absoluteString, + result: .failure(StringError(errorString)), + duration: fetchStart.distance(to: .now()) + ) + return nil + } + + self.delegate?.didDownloadPrebuilt( from: artifactURL.absoluteString, - bytesDownloaded: bytesDownloaded, - totalBytesToDownload: totalBytesToDownload + result: .success((destination, false)), + duration: fetchStart.distance(to: .now()) ) } - } catch { - observabilityScope.emit( - info: "Prebuilt artifact \(artifactFile)", - underlyingError: error - ) - self.delegate?.didDownloadPrebuilt( - from: artifactURL.absoluteString, - result: .failure(error), - duration: fetchStart.distance(to: .now()) - ) - return nil - } - // Check the checksum - if try !check(path: destination, checksum: artifact.checksum) { - let errorString = - "Prebuilt artifact \(artifactFile) checksum mismatch" - observabilityScope.emit(info: errorString) - self.delegate?.didDownloadPrebuilt( - from: artifactURL.absoluteString, - result: .failure(StringError(errorString)), - duration: fetchStart.distance(to: .now()) - ) - return nil + if let cacheFile { + // Cache the zip file + if fileSystem.exists(cacheFile) { + try fileSystem.removeFileTree(cacheFile) + } else { + try fileSystem.createDirectory(cacheFile.parentDirectory, recursive: true) + } + try fileSystem.copy(from: destination, to: cacheFile) + } } - - self.delegate?.didDownloadPrebuilt( - from: artifactURL.absoluteString, - result: .success((destination, false)), - duration: fetchStart.distance(to: .now()) - ) } // Extract @@ -461,18 +551,16 @@ extension Workspace { addedOrUpdatedPackages: [PackageReference], observabilityScope: ObservabilityScope ) async throws { - guard let prebuiltsManager = self.prebuiltsManager else { + guard let prebuiltsManager else { // Disabled return } - for prebuilt in prebuiltsManager.findPrebuilts( - packages: try manifests.requiredPackages - ) { + let addedPrebuilts = ManagedPrebuilts() + + for prebuilt in prebuiltsManager.findPrebuilts(packages: try manifests.requiredPackages) { guard - let manifest = manifests.allDependencyManifests[ - prebuilt.packageRef.identity - ], + let manifest = manifests.allDependencyManifests[prebuilt.identity], let packageVersion = manifest.manifest.version, let prebuiltManifest = try await prebuiltsManager .downloadManifest( @@ -484,14 +572,10 @@ extension Workspace { continue } - let hostPlatform = hostPrebuiltsPlatform + let hostPlatform = prebuiltsManager.hostPlatform for library in prebuiltManifest.libraries { - for artifact in library.artifacts { - guard artifact.platform == hostPlatform else { - continue - } - + for artifact in library.artifacts where artifact.platform == hostPlatform { if let path = try await prebuiltsManager .downloadPrebuilt( package: prebuilt, @@ -500,116 +584,194 @@ extension Workspace { artifact: artifact, observabilityScope: observabilityScope ) - { + { // Add to workspace state let managedPrebuilt = ManagedPrebuilt( - packageRef: prebuilt.packageRef, + identity: prebuilt.identity, + version: packageVersion, libraryName: library.name, path: path, products: library.products, cModules: library.cModules ) + addedPrebuilts.add(managedPrebuilt) await self.state.prebuilts.add(managedPrebuilt) - try await self.state.save() } } } } + + for prebuilt in await self.state.prebuilts.prebuilts { + if !addedPrebuilts.contains(where: { $0.identity == prebuilt.identity && $0.version == prebuilt.version }) { + await self.state.prebuilts.remove(packageIdentity: prebuilt.identity, targetName: prebuilt.libraryName) + } + } + + try await self.state.save() } +} - var hostPrebuiltsPlatform: PrebuiltsManifest.Platform? { - if self.hostToolchain.targetTriple.isDarwin() { - switch self.hostToolchain.targetTriple.arch { - case .aarch64: - return .macos_aarch64 - case .x86_64: - return .macos_x86_64 +extension Workspace.PrebuiltsManifest.Platform { + public var arch: Arch { + switch self { + case .macos_aarch64, .windows_aarch64, + .ubuntu_noble_aarch64, .ubuntu_jammy_aarch64, .ubuntu_focal_aarch64, + .fedora_39_aarch64, + .amazonlinux2_aarch64, + .rhel_ubi9_aarch64, + .debian_12_aarch64: + return .aarch64 + case .macos_x86_64, .windows_x86_64, + .ubuntu_noble_x86_64, .ubuntu_jammy_x86_64, .ubuntu_focal_x86_64, + .fedora_39_x86_64, + .amazonlinux2_x86_64, + .rhel_ubi9_x86_64, + .debian_12_x86_64: + return .x86_64 + } + } + + public var os: OS { + switch self { + case .macos_aarch64, .macos_x86_64: + return .macos + case .windows_aarch64, .windows_x86_64: + return .windows + case .ubuntu_noble_aarch64, .ubuntu_noble_x86_64, + .ubuntu_jammy_aarch64, .ubuntu_jammy_x86_64, + .ubuntu_focal_aarch64, .ubuntu_focal_x86_64, + .fedora_39_aarch64, .fedora_39_x86_64, + .amazonlinux2_aarch64, .amazonlinux2_x86_64, + .rhel_ubi9_aarch64, .rhel_ubi9_x86_64, + .debian_12_aarch64, .debian_12_x86_64: + return .linux + } + } + + /// Determine host platform based on compilation target + public static var hostPlatform: Self? { + let arch: Arch? +#if arch(arm64) + arch = .aarch64 +#elseif arch(x86_64) + arch = .x86_64 +#else + arch = nil +#endif + guard let arch else { + return nil + } + +#if os(macOS) + switch arch { + case .aarch64: + return .macos_aarch64 + case .x86_64: + return .macos_x86_64 + } +#elseif os(Windows) + switch arch { + case .aarch64: + return .windows_aarch64 + case .x86_64: + return .windows_x86_64 + } +#elseif os(Linux) + // Load up the os-release file into a dictionary + guard let osData = try? String(contentsOfFile: "/etc/os-release", encoding: .utf8) + else { + return nil + } + let osLines = osData.split(separator: "\n") + let osDict = osLines.reduce(into: [Substring: String]()) { + (dict, line) in + let parts = line.split(separator: "=", maxSplits: 2) + dict[parts[0]] = parts[1...].joined(separator: "=").trimmingCharacters(in: ["\""]) + } + + switch osDict["ID"] { + case "ubuntu": + switch osDict["VERSION_CODENAME"] { + case "noble": + switch arch { + case .aarch64: + return .ubuntu_noble_aarch64 + case .x86_64: + return .ubuntu_noble_x86_64 + } + case "jammy": + switch arch { + case .aarch64: + return .ubuntu_jammy_aarch64 + case .x86_64: + return .ubuntu_jammy_x86_64 + } + case "focal": + switch arch { + case .aarch64: + return .ubuntu_focal_aarch64 + case .x86_64: + return .ubuntu_focal_x86_64 + } default: return nil } - } else if self.hostToolchain.targetTriple.isWindows() { - switch self.hostToolchain.targetTriple.arch { - case .aarch64: - return .windows_aarch64 - case .x86_64: - return .windows_x86_64 + case "fedora": + switch osDict["VERSION_ID"] { + case "39", "41": + switch arch { + case .aarch64: + return .fedora_39_aarch64 + case .x86_64: + return .fedora_39_x86_64 + } default: return nil } - } else if self.hostToolchain.targetTriple.isLinux() { - // Load up the os-release file into a dictionary - guard let osData = try? String(contentsOfFile: "/etc/os-release", encoding: .utf8) - else { + case "amzn": + switch osDict["VERSION_ID"] { + case "2": + switch arch { + case .aarch64: + return .amazonlinux2_aarch64 + case .x86_64: + return .amazonlinux2_x86_64 + } + default: return nil } - let osLines = osData.split(separator: "\n") - let osDict = osLines.reduce(into: [Substring: String]()) { - (dict, line) in - let parts = line.split(separator: "=", maxSplits: 2) - dict[parts[0]] = parts[1...].joined(separator: "=").trimmingCharacters(in: ["\""]) + case "rhel": + guard let version = osDict["VERSION_ID"] else { + return nil } - - switch osDict["ID"] { - case "ubuntu": - switch osDict["VERSION_CODENAME"] { - case "jammy": - switch self.hostToolchain.targetTriple.arch { - case .aarch64: - return .ubuntu_jammy_aarch64 - case .x86_64: - return .ubuntu_jammy_x86_64 - default: - return nil - } - case "focal": - switch self.hostToolchain.targetTriple.arch { - case .aarch64: - return .ubuntu_focal_aarch64 - case .x86_64: - return .ubuntu_focal_x86_64 - default: - return nil - } - default: - return nil - } - case "amzn": - switch osDict["VERSION_ID"] { - case "2": - switch self.hostToolchain.targetTriple.arch { - case .aarch64: - return .amazonlinux2_aarch64 - case .x86_64: - return .amazonlinux2_x86_64 - default: - return nil - } - default: - return nil - } - case "rhel": - guard let version = osDict["VERSION_ID"] else { - return nil + switch version.split(separator: ".")[0] { + case "9": + switch arch { + case .aarch64: + return .rhel_ubi9_aarch64 + case .x86_64: + return .rhel_ubi9_x86_64 } - switch version.split(separator: ".")[0] { - case "9": - switch self.hostToolchain.targetTriple.arch { - case .aarch64: - return .rhel_ubi9_aarch64 - case .x86_64: - return .rhel_ubi9_x86_64 - default: - return nil - } - default: - return nil + default: + return nil + } + case "debian": + switch osDict["VERSION_ID"] { + case "12": + switch arch { + case .aarch64: + return .debian_12_aarch64 + case .x86_64: + return .debian_12_x86_64 } default: return nil } - } else { + default: return nil } +#else + return nil +#endif } - } diff --git a/Sources/Workspace/Workspace+State.swift b/Sources/Workspace/Workspace+State.swift index 3c395133031..5563f16246e 100644 --- a/Sources/Workspace/Workspace+State.swift +++ b/Sources/Workspace/Workspace+State.swift @@ -209,7 +209,7 @@ extension WorkspaceStateStorage { self.object = .init( dependencies: dependencies.map { .init($0) }.sorted { $0.packageRef.identity < $1.packageRef.identity }, artifacts: artifacts.map { .init($0) }.sorted { $0.packageRef.identity < $1.packageRef.identity }, - prebuilts: prebuilts.map { .init($0) }.sorted { $0.packageRef.identity < $1.packageRef.identity } + prebuilts: prebuilts.map { .init($0) }.sorted { $0.identity < $1.identity } ) } @@ -455,14 +455,16 @@ extension WorkspaceStateStorage { } struct Prebuilt: Codable { - let packageRef: PackageReference + let identity: PackageIdentity + let version: TSCUtility.Version let libraryName: String let path: Basics.AbsolutePath let products: [String] let cModules: [String] init(_ managedPrebuilt: Workspace.ManagedPrebuilt) { - self.packageRef = .init(managedPrebuilt.packageRef) + self.identity = managedPrebuilt.identity + self.version = managedPrebuilt.version self.libraryName = managedPrebuilt.libraryName self.path = managedPrebuilt.path self.products = managedPrebuilt.products @@ -534,8 +536,9 @@ extension Workspace.ManagedArtifact { extension Workspace.ManagedPrebuilt { fileprivate init(_ prebuilt: WorkspaceStateStorage.V7.Prebuilt) throws { - try self.init( - packageRef: .init(prebuilt.packageRef), + self.init( + identity: prebuilt.identity, + version: prebuilt.version, libraryName: prebuilt.libraryName, path: prebuilt.path, products: prebuilt.products, diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index 90e98927e6f..116c8e44bb8 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -568,19 +568,30 @@ public class Workspace { // register the binary artifacts downloader with the cancellation handler cancellator?.register(name: "binary artifacts downloads", handler: binaryArtifactsManager) - let prebuiltsManager: PrebuiltsManager? = configuration.usePrebuilts ? PrebuiltsManager( - fileSystem: fileSystem, - authorizationProvider: authorizationProvider, - scratchPath: location.prebuiltsDirectory, - cachePath: customPrebuiltsManager?.useCache == false || !configuration - .sharedDependenciesCacheEnabled ? .none : location.sharedPrebuiltsCacheDirectory, - customHTTPClient: customPrebuiltsManager?.httpClient, - customArchiver: customPrebuiltsManager?.archiver, - delegate: delegate.map(WorkspacePrebuiltsManagerDelegate.init(workspaceDelegate:)) - ) : .none - // register the prebuilt packages downloader with the cancellation handler - if let prebuiltsManager { + if configuration.usePrebuilts, let hostPlatform = customPrebuiltsManager?.hostPlatform ?? PrebuiltsManifest.Platform.hostPlatform { + let rootCertPath: AbsolutePath? + if let path = configuration.prebuiltsRootCertPath { + rootCertPath = try AbsolutePath(validating: path) + } else { + rootCertPath = nil + } + + let prebuiltsManager = PrebuiltsManager( + fileSystem: fileSystem, + hostPlatform: hostPlatform, + authorizationProvider: authorizationProvider, + scratchPath: location.prebuiltsDirectory, + cachePath: customPrebuiltsManager?.useCache == false || !configuration.sharedDependenciesCacheEnabled ? .none : location.sharedPrebuiltsCacheDirectory, + customHTTPClient: customPrebuiltsManager?.httpClient, + customArchiver: customPrebuiltsManager?.archiver, + delegate: delegate.map(WorkspacePrebuiltsManagerDelegate.init(workspaceDelegate:)), + prebuiltsDownloadURL: configuration.prebuiltsDownloadURL, + rootCertPath: customPrebuiltsManager?.rootCertPath ?? rootCertPath + ) cancellator?.register(name: "package prebuilts downloads", handler: prebuiltsManager) + self.prebuiltsManager = prebuiltsManager + } else { + self.prebuiltsManager = nil } // initialize @@ -599,7 +610,6 @@ public class Workspace { self.registryClient = registryClient self.registryDownloadsManager = registryDownloadsManager self.binaryArtifactsManager = binaryArtifactsManager - self.prebuiltsManager = prebuiltsManager self.identityResolver = identityResolver self.dependencyMapper = dependencyMapper @@ -955,17 +965,12 @@ extension Workspace { } let prebuilts: [PackageIdentity: [String: PrebuiltLibrary]] = await self.state.prebuilts.reduce(into: .init()) { - let prebuilt = PrebuiltLibrary( - packageRef: $1.packageRef, - libraryName: $1.libraryName, - path: $1.path, - products: $1.products, - cModules: $1.cModules - ) + let prebuilt = PrebuiltLibrary(identity: $1.identity, libraryName: $1.libraryName, path: $1.path, products: $1.products, cModules: $1.cModules) for product in $1.products { - $0[$1.packageRef.identity, default: [:]][product] = prebuilt + $0[$1.identity, default: [:]][product] = prebuilt } } + // Load the graph. let packageGraph = try ModulesGraph.load( root: manifests.root, diff --git a/Sources/_InternalTestSupport/ManifestExtensions.swift b/Sources/_InternalTestSupport/ManifestExtensions.swift index 7f65ad5cf2d..225ae53d84c 100644 --- a/Sources/_InternalTestSupport/ManifestExtensions.swift +++ b/Sources/_InternalTestSupport/ManifestExtensions.swift @@ -288,4 +288,28 @@ extension Manifest { pruneDependencies: false ) } + + public func with(dependencies: [PackageDependency]) -> Manifest { + Manifest( + displayName: self.displayName, + packageIdentity: self.packageIdentity, + path: self.path, + packageKind: self.packageKind, + packageLocation: self.packageLocation, + defaultLocalization: self.defaultLocalization, + platforms: self.platforms, + version: self.version, + revision: self.revision, + toolsVersion: self.toolsVersion, + pkgConfig: self.pkgConfig, + providers: self.providers, + cLanguageStandard: self.cLanguageStandard, + cxxLanguageStandard: self.cxxLanguageStandard, + swiftLanguageVersions: self.swiftLanguageVersions, + dependencies: dependencies, + products: self.products, + targets: self.targets, + traits: self.traits + ) + } } diff --git a/Sources/_InternalTestSupport/MockWorkspace.swift b/Sources/_InternalTestSupport/MockWorkspace.swift index b7ce26427a2..441f06ddd8b 100644 --- a/Sources/_InternalTestSupport/MockWorkspace.swift +++ b/Sources/_InternalTestSupport/MockWorkspace.swift @@ -403,6 +403,8 @@ public final class MockWorkspace { defaultRegistry: self.defaultRegistry, manifestImportRestrictions: .none, usePrebuilts: self.customPrebuiltsManager != nil, + prebuiltsDownloadURL: nil, + prebuiltsRootCertPath: nil, pruneDependencies: self.pruneDependencies, traitConfiguration: self.traitConfiguration ), diff --git a/Sources/swift-build-prebuilts/BuildPrebuilts.swift b/Sources/swift-build-prebuilts/BuildPrebuilts.swift index f827ff97357..4db1740dc17 100644 --- a/Sources/swift-build-prebuilts/BuildPrebuilts.swift +++ b/Sources/swift-build-prebuilts/BuildPrebuilts.swift @@ -33,6 +33,7 @@ struct PrebuiltRepos: Identifiable { let tag: String let manifest: Workspace.PrebuiltsManifest let cModulePaths: [String: [String]] + let addProduct: (Workspace.PrebuiltsManifest.Library, AbsolutePath) async throws -> () var id: String { tag } } @@ -48,9 +49,77 @@ var prebuiltRepos: IdentifiableSet = [ .init( name: "MacroSupport", products: [ + "SwiftBasicFormat", + "SwiftCompilerPlugin", + "SwiftDiagnostics", + "SwiftIDEUtils", + "SwiftOperators", + "SwiftParser", + "SwiftParserDiagnostics", + "SwiftRefactor", + "SwiftSyntax", + "SwiftSyntaxBuilder", + "SwiftSyntaxMacros", + "SwiftSyntaxMacroExpansion", "SwiftSyntaxMacrosTestSupport", + "SwiftSyntaxMacrosGenericTestSupport", + "_SwiftCompilerPluginMessageHandling", + "_SwiftLibraryPluginProvider", + ], + cModules: [ + "_SwiftSyntaxCShims", + ] + ), + ]), + cModulePaths: [ + "_SwiftSyntaxCShims": ["Sources", "_SwiftSyntaxCShims"] + ], + addProduct: { library, repoDir in + let targets = [ + "SwiftBasicFormat", + "SwiftCompilerPlugin", + "SwiftDiagnostics", + "SwiftIDEUtils", + "SwiftOperators", + "SwiftParser", + "SwiftParserDiagnostics", + "SwiftRefactor", + "SwiftSyntax", + "SwiftSyntaxBuilder", + "SwiftSyntaxMacros", + "SwiftSyntaxMacroExpansion", + "SwiftSyntaxMacrosTestSupport", + "SwiftSyntaxMacrosGenericTestSupport", + "SwiftCompilerPluginMessageHandling", + "SwiftLibraryPluginProvider", + ] + try await shell("swift package add-product \(library.name) --type static-library --targets \(targets.joined(separator: " "))", cwd: repoDir) + } + ), + .init( + tag:"601.0.1", + manifest: .init(libraries: [ + .init( + name: "MacroSupport", + products: [ + "SwiftBasicFormat", "SwiftCompilerPlugin", - "SwiftSyntaxMacros" + "SwiftDiagnostics", + "SwiftIDEUtils", + "SwiftIfConfig", + "SwiftLexicalLookup", + "SwiftOperators", + "SwiftParser", + "SwiftParserDiagnostics", + "SwiftRefactor", + "SwiftSyntax", + "SwiftSyntaxBuilder", + "SwiftSyntaxMacros", + "SwiftSyntaxMacroExpansion", + "SwiftSyntaxMacrosTestSupport", + "SwiftSyntaxMacrosGenericTestSupport", + "_SwiftCompilerPluginMessageHandling", + "_SwiftLibraryPluginProvider", ], cModules: [ "_SwiftSyntaxCShims", @@ -60,15 +129,47 @@ var prebuiltRepos: IdentifiableSet = [ ]), cModulePaths: [ "_SwiftSyntaxCShims": ["Sources", "_SwiftSyntaxCShims"] - ] + ], + addProduct: { library, repoDir in + let targets = [ + "SwiftBasicFormat", + "SwiftCompilerPlugin", + "SwiftDiagnostics", + "SwiftIDEUtils", + "SwiftIfConfig", + "SwiftLexicalLookup", + "SwiftOperators", + "SwiftParser", + "SwiftParserDiagnostics", + "SwiftRefactor", + "SwiftSyntax", + "SwiftSyntaxBuilder", + "SwiftSyntaxMacros", + "SwiftSyntaxMacroExpansion", + "SwiftSyntaxMacrosTestSupport", + "SwiftSyntaxMacrosGenericTestSupport", + "SwiftCompilerPluginMessageHandling", + "SwiftLibraryPluginProvider", + ] + // swift package add-product doesn't work here since it's now computed + let packageFile = repoDir.appending(component: "Package.swift") + var package = try String(contentsOf: packageFile.asURL) + package.replace("products: products,", with: """ + products: products + [ + .library(name: "\(library.name)", type: .static, targets: [ + \(targets.map({ "\"\($0)\"" }).joined(separator: ",")) + ]) + ], + """) + try package.write(to: packageFile.asURL, atomically: true, encoding: .utf8) + } ), ] ), ] -let manifestHost = URL(string: "https://github.com/dschaefer2/swift-syntax/releases/download")! let swiftVersion = "\(SwiftVersion.current.major).\(SwiftVersion.current.minor)" -let dockerImageRoot = "swiftlang/swift:nightly-" +let dockerImageRoot = "swiftlang/swift:nightly-6.1-" @main struct BuildPrebuilts: AsyncParsableCommand { @@ -84,49 +185,110 @@ struct BuildPrebuilts: AsyncParsableCommand { @Option(help: "The command to use for docker.") var dockerCommand: String = "docker" + @Flag(help: "Whether to build the prebuilt artifacts") + var build = false + + @Flag(help: "Whether to sign the manifest") + var sign = false + + @Option(name: .customLong("private-key-path"), help: "The path to certificate's private key (PEM encoded)") + var privateKeyPathStr: String? + + @Option(name: .customLong("cert-chain-path"), help: "Path to a certificate (DER encoded) in the chain. The certificate used for signing must be first and the root certificate last.") + var certChainPathStrs: [String] = [] + + @Flag(help: .hidden) + var testSigning: Bool = false + + func validate() throws { + if sign && !testSigning { + guard privateKeyPathStr != nil else { + throw ValidationError("No private key path provided") + } + + guard !certChainPathStrs.isEmpty else { + throw ValidationError("No certificates provided") + } + } + + if !build && !sign && !testSigning { + throw ValidationError("Requires one of --build or --sign or both") + } + } + mutating func run() async throws { - let fm = FileManager.default + if build { + try await build() + } + + if sign || testSigning { + try await sign() + } + } + + mutating func build() async throws { + let fileSystem = localFileSystem + let encoder = JSONEncoder() + encoder.outputFormatting = .prettyPrinted print("Stage directory: \(stageDir)") - try fm.removeItem(atPath: stageDir.pathString) - try fm.createDirectory(atPath: stageDir.pathString, withIntermediateDirectories: true) - _ = fm.changeCurrentDirectoryPath(stageDir.pathString) + + let srcDir = stageDir.appending("src") + let libDir = stageDir.appending("lib") + let modulesDir = stageDir.appending("Modules") + let includesDir = stageDir.appending("include") + + if fileSystem.exists(srcDir) { + try fileSystem.removeFileTree(srcDir) + } + try fileSystem.createDirectory(srcDir, recursive: true) + + if fileSystem.exists(libDir) { + try fileSystem.removeFileTree(libDir) + } + + if fileSystem.exists(modulesDir) { + try fileSystem.removeFileTree(modulesDir) + } + + if fileSystem.exists(includesDir) { + try fileSystem.removeFileTree(includesDir) + } for repo in prebuiltRepos.values { - let repoDir = stageDir.appending(repo.url.lastPathComponent) - let libDir = stageDir.appending("lib") - let modulesDir = stageDir.appending("modules") - let includesDir = stageDir.appending("include") + let repoDir = srcDir.appending(repo.url.lastPathComponent) let scratchDir = repoDir.appending(".build") let buildDir = scratchDir.appending("release") let srcModulesDir = buildDir.appending("Modules") + let prebuiltDir = stageDir.appending(repo.url.lastPathComponent) - try await shell("git clone \(repo.url)") + try await shell("git clone \(repo.url)", cwd: srcDir) for version in repo.versions { - _ = fm.changeCurrentDirectoryPath(repoDir.pathString) - try await shell("git checkout \(version.tag)") + let versionDir = prebuiltDir.appending(version.tag) + if !fileSystem.exists(versionDir) { + try fileSystem.createDirectory(versionDir, recursive: true) + } + + try await shell("git checkout \(version.tag)", cwd: repoDir) var newLibraries: IdentifiableSet = [] for library in version.manifest.libraries { - // TODO: this is assuming products map to target names which is not always true - try await shell("swift package add-product \(library.name) --type static-library --targets \(library.products.joined(separator: " "))") - - var newArtifacts: [Workspace.PrebuiltsManifest.Library.Artifact] = [] + try await version.addProduct(library, repoDir) for platform in Workspace.PrebuiltsManifest.Platform.allCases { guard canBuild(platform) else { continue } - try fm.createDirectory(atPath: libDir.pathString, withIntermediateDirectories: true) - try fm.createDirectory(atPath: modulesDir.pathString, withIntermediateDirectories: true) - try fm.createDirectory(atPath: includesDir.pathString, withIntermediateDirectories: true) + try fileSystem.createDirectory(libDir, recursive: true) + try fileSystem.createDirectory(modulesDir, recursive: true) + try fileSystem.createDirectory(includesDir, recursive: true) // Clean out the scratch dir - if fm.fileExists(atPath: scratchDir.pathString) { - try fm.removeItem(atPath: scratchDir.pathString) + if fileSystem.exists(scratchDir) { + try fileSystem.removeFileTree(scratchDir) } // Build @@ -134,87 +296,155 @@ struct BuildPrebuilts: AsyncParsableCommand { if docker, let dockerTag = platform.dockerTag, let dockerPlatform = platform.arch.dockerPlatform { cmd += "\(dockerCommand) run --rm --platform \(dockerPlatform) -v \(repoDir):\(repoDir) -w \(repoDir) \(dockerImageRoot)\(dockerTag) " } - cmd += "swift build -c release --arch \(platform.arch) --product \(library.name)" - try await shell(cmd) + cmd += "swift build -c release -debug-info-format none --arch \(platform.arch) --product \(library.name)" + try await shell(cmd, cwd: repoDir) // Copy the library to staging let lib = "lib\(library.name).a" - try fm.copyItem(atPath: buildDir.appending(lib).pathString, toPath: libDir.appending(lib).pathString) + try fileSystem.copy(from: buildDir.appending(lib), to: libDir.appending(lib)) // Copy the swiftmodules - for file in try fm.contentsOfDirectory(atPath: srcModulesDir.pathString) { - try fm.copyItem(atPath: srcModulesDir.appending(file).pathString, toPath: modulesDir.appending(file).pathString) + for file in try fileSystem.getDirectoryContents(srcModulesDir) { + try fileSystem.copy(from: srcModulesDir.appending(file), to: modulesDir.appending(file)) } - // Copy the C module headers + // Do a deep copy of the C module headers for cModule in library.cModules { let cModuleDir = version.cModulePaths[cModule] ?? ["Sources", cModule] let srcIncludeDir = repoDir.appending(components: cModuleDir).appending("include") let destIncludeDir = includesDir.appending(cModule) - try fm.createDirectory(atPath: destIncludeDir.pathString, withIntermediateDirectories: true) - for file in try fm.contentsOfDirectory(atPath: srcIncludeDir.pathString) { - try fm.copyItem(atPath: srcIncludeDir.appending(file).pathString, toPath: destIncludeDir.appending(file).pathString) + + try fileSystem.createDirectory(destIncludeDir, recursive: true) + try fileSystem.enumerate(directory: srcIncludeDir) { srcPath in + let destPath = destIncludeDir.appending(srcPath.relative(to: srcIncludeDir)) + try fileSystem.createDirectory(destPath.parentDirectory) + try fileSystem.copy(from: srcPath, to: destPath) } } // Zip it up - _ = fm.changeCurrentDirectoryPath(stageDir.pathString) - let zipFile = stageDir.appending("\(swiftVersion)-\(library.name)-\(platform).zip") let contentDirs = ["lib", "Modules"] + (library.cModules.isEmpty ? [] : ["include"]) #if os(Windows) - try await shell("tar -acf \(zipFile.pathString) \(contentDirs.joined(separator: " "))") + let zipFile = versionDir.appending("\(swiftVersion)-\(library.name)-\(platform).zip") + try await shell("tar -acf \(zipFile.pathString) \(contentDirs.joined(separator: " "))", cwd: stageDir) + let contents = try ByteString(Data(contentsOf: zipFile.asURL)) +#elseif os(Linux) + let tarFile = versionDir.appending("\(swiftVersion)-\(library.name)-\(platform).tar.gz") + try await shell("tar -zcf \(tarFile.pathString) \(contentDirs.joined(separator: " "))", cwd: stageDir) + let contents = try ByteString(Data(contentsOf: tarFile.asURL)) #else - try await shell("zip -r \(zipFile.pathString) \(contentDirs.joined(separator: " "))") + let zipFile = versionDir.appending("\(swiftVersion)-\(library.name)-\(platform).zip") + try await shell("zip -r \(zipFile.pathString) \(contentDirs.joined(separator: " "))", cwd: stageDir) + let contents = try ByteString(Data(contentsOf: zipFile.asURL)) #endif - _ = fm.changeCurrentDirectoryPath(repoDir.pathString) - let contents = try ByteString(Data(contentsOf: zipFile.asURL)) let checksum = SHA256().hash(contents).hexadecimalRepresentation + let artifact: Workspace.PrebuiltsManifest.Library.Artifact = + .init(platform: platform, checksum: checksum) - newArtifacts.append(.init(platform: platform, checksum: checksum)) + let artifactJsonFile = versionDir.appending("\(swiftVersion)-\(library.name)-\(platform).zip.json") + try fileSystem.writeFileContents(artifactJsonFile, data: encoder.encode(artifact)) - try fm.removeItem(atPath: libDir.pathString) - try fm.removeItem(atPath: modulesDir.pathString) - try fm.removeItem(atPath: includesDir.pathString) + try fileSystem.removeFileTree(libDir) + try fileSystem.removeFileTree(modulesDir) + try fileSystem.removeFileTree(includesDir) } + let decoder = JSONDecoder() let newLibrary = Workspace.PrebuiltsManifest.Library( name: library.name, products: library.products, cModules: library.cModules, - artifacts: newArtifacts + artifacts: try fileSystem.getDirectoryContents(versionDir) + .filter({ $0.hasSuffix(".zip.json")}) + .compactMap({ + let data: Data = try fileSystem.readFileContents(versionDir.appending($0)) + return try? decoder.decode(Workspace.PrebuiltsManifest.Library.Artifact.self, from: data) + }) ) newLibraries.insert(newLibrary) - try await shell("git reset --hard") + try await shell("git restore .", cwd: repoDir) } + } + } - if let oldManifest = try await downloadManifest(version: version) { - // Add in elements from the old manifest we haven't generated - for library in oldManifest.libraries { - if var newLibrary = newLibraries[library.name] { - var newArtifacts = IdentifiableSet(newLibrary.artifacts) - for oldArtifact in library.artifacts { - if !newArtifacts.contains(id: oldArtifact.id) { - newArtifacts.insert(oldArtifact) - } - } - newLibrary.artifacts = .init(newArtifacts.values) - newLibraries.insert(newLibrary) - } else { - newLibraries.insert(library) - } - } + try fileSystem.changeCurrentWorkingDirectory(to: stageDir) + try fileSystem.removeFileTree(srcDir) + } + + mutating func sign() async throws { + let fileSystem = localFileSystem + let encoder = JSONEncoder() + encoder.outputFormatting = .prettyPrinted + let decoder = JSONDecoder() + + for repo in prebuiltRepos.values { + let prebuiltDir = stageDir.appending(repo.url.lastPathComponent) + for version in repo.versions { + let versionDir = prebuiltDir.appending(version.tag) + let manifestFile = versionDir.appending("\(swiftVersion)-manifest.json") + + var manifest = version.manifest + manifest.libraries = try manifest.libraries.map({ + .init(name: $0.name, + products: $0.products, + cModules: $0.cModules, + artifacts: try fileSystem.getDirectoryContents(versionDir) + .filter({ $0.hasSuffix(".zip.json")}) + .compactMap({ + let data: Data = try fileSystem.readFileContents(versionDir.appending($0)) + return try? decoder.decode(Workspace.PrebuiltsManifest.Library.Artifact.self, from: data) + }) + ) + }) + + if testSigning { + // Use SwiftPM's test certificate chain and private key for testing + let certsPath = try AbsolutePath(validating: #file) + .parentDirectory.parentDirectory.parentDirectory + .appending(components: "Fixtures", "Signing", "Certificates") + privateKeyPathStr = certsPath.appending("Test_rsa_key.pem").pathString + certChainPathStrs = [ + certsPath.appending("Test_rsa.cer").pathString, + certsPath.appending("TestIntermediateCA.cer").pathString, + certsPath.appending("TestRootCA.cer").pathString + ] + } + + guard let privateKeyPathStr else { + fatalError("No private key path provided") } - let newManifest = Workspace.PrebuiltsManifest(libraries: .init(newLibraries.values)) - let encoder = JSONEncoder() - encoder.outputFormatting = .prettyPrinted - let manifestData = try encoder.encode(newManifest) - let manifestFile = stageDir.appending("\(swiftVersion)-manifest.json") - try manifestData.write(to: manifestFile.asURL) + let certChainPaths = try certChainPathStrs.map { try make(path: $0) } + + guard let rootCertPath = certChainPaths.last else { + fatalError("No certificates provided") + } + + let privateKeyPath = try make(path: privateKeyPathStr) + + try await withTemporaryDirectory { tmpDir in + try fileSystem.copy(from: rootCertPath, to: tmpDir.appending(rootCertPath.basename)) + + let signer = ManifestSigning( + trustedRootCertsDir: tmpDir, + observabilityScope: ObservabilitySystem { _, diagnostic in print(diagnostic) }.topScope + ) + + let signature = try await signer.sign( + manifest: manifest, + certChainPaths: certChainPaths, + certPrivateKeyPath: privateKeyPath, + fileSystem: fileSystem + ) + + let signedManifest = Workspace.SignedPrebuiltsManifest(manifest: manifest, signature: signature) + try encoder.encode(signedManifest).write(to: manifestFile.asURL) + } } } + } func canBuild(_ platform: Workspace.PrebuiltsManifest.Platform) -> Bool { @@ -229,70 +459,53 @@ struct BuildPrebuilts: AsyncParsableCommand { if platform.os == .windows { return true } +#elseif os(Linux) + if platform == Workspace.PrebuiltsManifest.Platform.hostPlatform { + return true + } #endif return docker && platform.os == .linux } - func shell(_ command: String) async throws { + func make(path: String) throws -> AbsolutePath { + if let path = try? AbsolutePath(validating: path) { + // It's already absolute + return path + } + + return try AbsolutePath(validating: FileManager.default.currentDirectoryPath) + .appending(RelativePath(validating: path)) + } + +} + +func shell(_ command: String, cwd: AbsolutePath) async throws { + _ = FileManager.default.changeCurrentDirectoryPath(cwd.pathString) + #if os(Windows) - let arguments = ["C:\\Windows\\System32\\cmd.exe", "/c", command] + let arguments = ["C:\\Windows\\System32\\cmd.exe", "/c", command] #else - let arguments = ["/bin/bash", "-c", command] + let arguments = ["/bin/bash", "-c", command] #endif - let process = AsyncProcess( - arguments: arguments, - outputRedirection: .none - ) - print("Running:", command) - try process.launch() - let result = try await process.waitUntilExit() - switch result.exitStatus { - case .terminated(code: let code): - if code != 0 { - throw StringError("Command exited with code \(code): \(command)") - } + let process = AsyncProcess( + arguments: arguments, + outputRedirection: .none + ) + print("Running:", command) + try process.launch() + let result = try await process.waitUntilExit() + switch result.exitStatus { + case .terminated(code: let code): + if code != 0 { + throw StringError("Command exited with code \(code): \(command)") + } #if os(Windows) - case .abnormal(exception: let exception): - throw StringError("Command threw exception \(exception): \(command)") + case .abnormal(exception: let exception): + throw StringError("Command threw exception \(exception): \(command)") #else - case .signalled(signal: let signal): - throw StringError("Command exited on signal \(signal): \(command)") + case .signalled(signal: let signal): + throw StringError("Command exited on signal \(signal): \(command)") #endif - } - } - - func downloadManifest(version: PrebuiltRepos.Version) async throws -> Workspace.PrebuiltsManifest? { - let fm = FileManager.default - let manifestFile = swiftVersion + "-manifest.json" - let destination = stageDir.appending(manifestFile) - if fm.fileExists(atPath: destination.pathString) { - do { - return try JSONDecoder().decode( - Workspace.PrebuiltsManifest.self, - from: Data(contentsOf: destination.asURL) - ) - } catch { - // redownload it - try fm.removeItem(atPath: destination.pathString) - } - } - let manifestURL = manifestHost.appending(components: version.tag, manifestFile) - print("Downloading:", manifestURL.absoluteString) - let httpClient = HTTPClient() - var headers = HTTPClientHeaders() - headers.add(name: "Accept", value: "application/json") - var request = HTTPClient.Request(kind: .generic(.get), url: manifestURL) - request.options.validResponseCodes = [200] - - let response = try? await httpClient.execute(request) { _, _ in } - if let body = response?.body { - return try JSONDecoder().decode( - Workspace.PrebuiltsManifest.self, - from: body - ) - } - - return nil } } diff --git a/Tests/BasicsTests/HTTPClientTests.swift b/Tests/BasicsTests/HTTPClientTests.swift index 1ced370499f..6ea4442b3db 100644 --- a/Tests/BasicsTests/HTTPClientTests.swift +++ b/Tests/BasicsTests/HTTPClientTests.swift @@ -21,15 +21,15 @@ class HTTPClientXCTest: XCTestCase { func testEponentialBackoff() async throws { try XCTSkipOnWindows(because: "https://github.com/swiftlang/swift-package-manager/issues/8501") let counter = SendableBox(0) - let lastCall = SendableBox() + let lastCall = SendableBox(Date()) let maxAttempts = 5 let errorCode = Int.random(in: 500 ..< 600) let delay = SendableTimeInterval.milliseconds(100) let httpClient = HTTPClient { _, _ in - let count = await counter.value! + let count = await counter.value let expectedDelta = pow(2.0, Double(count - 1)) * delay.timeInterval()! - let delta = await lastCall.value.flatMap { Date().timeIntervalSince($0) } ?? 0 + let delta = await Date().timeIntervalSince(lastCall.value) XCTAssertEqual(delta, expectedDelta, accuracy: 0.1) await counter.increment() @@ -425,7 +425,7 @@ struct HTTPClientTests { let httpClient = HTTPClient(configuration: configuration) { request, _ in await concurrentRequests.increment() - if await concurrentRequests.value! > maxConcurrentRequests { + if await concurrentRequests.value > maxConcurrentRequests { Issue.record("too many concurrent requests \(concurrentRequests), expected \(maxConcurrentRequests)") } diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index a222ee9e8ad..be957b476b5 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -4667,6 +4667,104 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { } } + func testPrebuiltsFlags() async throws { + // Make sure the include path for the prebuilts get passed to the + // generated test entry point and discover targets on Linux/Windows + let observability = ObservabilitySystem.makeForTesting() + + let prebuiltLibrary = PrebuiltLibrary( + identity: .plain("swift-syntax"), + libraryName: "MacroSupport", + path: "/MyPackage/.build/prebuilts/swift-syntax/600.0.1/6.1-MacroSupport-macos_aarch64", + products: [ + "SwiftBasicFormat", + "SwiftCompilerPlugin", + "SwiftDiagnostics", + "SwiftIDEUtils", + "SwiftOperators", + "SwiftParser", + "SwiftParserDiagnostics", + "SwiftRefactor", + "SwiftSyntax", + "SwiftSyntaxBuilder", + "SwiftSyntaxMacros", + "SwiftSyntaxMacroExpansion", + "SwiftSyntaxMacrosTestSupport", + "SwiftSyntaxMacrosGenericTestSupport", + "_SwiftCompilerPluginMessageHandling", + "_SwiftLibraryPluginProvider" + ], + cModules: ["_SwiftSyntaxCShims"] + ) + + let fs = InMemoryFileSystem( + emptyFiles: [ + "/MyPackage/Sources/MyMacroMacros/MyMacroMacros.swift", + "/MyPackage/Sources/MyMacros/MyMacros.swift", + "/MyPackage/Sources/MyMacroTests/MyMacroTests.swift" + ] + ) + + let graph = try loadModulesGraph( + fileSystem: fs, + manifests: [ + Manifest.createRootManifest( + displayName: "MyPackage", + path: "/MyPackage", + targets: [ + TargetDescription(name: "MyMacroMacros", type: .macro), + TargetDescription( + name: "MyMacros", + dependencies: [ + "MyMacroMacros", + .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), + .product(name: "SwiftCompilerPlugin", package: "swift-syntax"), + ] + ), + TargetDescription( + name: "MyMacroTests", + dependencies: [ + "MyMacroMacros", + .product(name: "SwiftSyntaxMacrosTestSupport", package: "swift-syntax"), + ], + type: .test + ) + ] + ) + ], + prebuilts: [prebuiltLibrary.identity: prebuiltLibrary.products.reduce(into: [:]) { + $0[$1] = prebuiltLibrary + }], + observabilityScope: observability.topScope + ) + + func checkTriple(triple: Basics.Triple) async throws { + let result = try await BuildPlanResult( + plan: mockBuildPlan( + triple: triple, + graph: graph, + fileSystem: fs, + observabilityScope: observability.topScope + ) + ) + +#if os(Windows) + let modulesDir = "-I\(prebuiltLibrary.path.pathString)\\Modules" +#else + let modulesDir = "-I\(prebuiltLibrary.path.pathString)/Modules" +#endif + let mytest = try XCTUnwrap(result.allTargets(named: "MyMacroTests").first) + XCTAssert(try mytest.swift().compileArguments().contains(modulesDir)) + let entryPoint = try XCTUnwrap(result.allTargets(named: "MyPackagePackageTests").first) + XCTAssert(try entryPoint.swift().compileArguments().contains(modulesDir)) + let discovery = try XCTUnwrap(result.allTargets(named: "MyPackagePackageDiscoveredTests").first) + XCTAssert(try discovery.swift().compileArguments().contains(modulesDir)) + } + + try await checkTriple(triple: .x86_64Linux) + try await checkTriple(triple: .x86_64Windows) + } + func testExtraBuildFlags() async throws { let fs = InMemoryFileSystem( emptyFiles: @@ -7002,7 +7100,7 @@ class BuildPlanTestCase: BuildSystemProviderTestCase { let myLib = try XCTUnwrap(plan.targets.first(where: { $0.module.name == "MyLib" })).swift() XCTAssertFalse(myLib.additionalFlags.contains(where: { $0.contains("-tool")}), "flags shouldn't contain tools items") - + // Make sure the tests do have the include path and the module map from the lib let myMacroTests = try XCTUnwrap(plan.targets.first(where: { $0.module.name == "MyMacroTests" })).swift() let flags = myMacroTests.additionalFlags.joined(separator: " ") diff --git a/Tests/WorkspaceTests/PrebuiltsTests.swift b/Tests/WorkspaceTests/PrebuiltsTests.swift index 8af88b8464e..c36f73e0b37 100644 --- a/Tests/WorkspaceTests/PrebuiltsTests.swift +++ b/Tests/WorkspaceTests/PrebuiltsTests.swift @@ -18,6 +18,7 @@ import Basics import struct TSCBasic.SHA256 import struct TSCBasic.ByteString import struct TSCUtility.Version +import PackageGraph import PackageModel import Workspace import XCTest @@ -26,89 +27,137 @@ import _InternalTestSupport final class PrebuiltsTests: XCTestCase { let swiftVersion = "\(SwiftVersion.current.major).\(SwiftVersion.current.minor)" - func initData(artifact: Data, swiftSyntaxVersion: String) throws -> (Workspace.PrebuiltsManifest, MockPackage, MockPackage) { - let manifest = Workspace.PrebuiltsManifest(libraries: [ - .init( - name: "MacroSupport", - products: [ - "SwiftSyntaxMacrosTestSupport", - "SwiftCompilerPlugin", - "SwiftSyntaxMacros" - ], - cModules: [ - "_SwiftSyntaxCShims" - ], - artifacts: [ - .init( - platform: .macos_aarch64, - checksum: SHA256().hash(ByteString(artifact)).hexadecimalRepresentation - ) - ] - ) - ]) - - let rootPackage = try MockPackage( - name: "Foo", - targets: [ - MockTarget( - name: "FooMacros", - dependencies: [ - .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), - .product(name: "SwiftCompilerPlugin", package: "swift-syntax"), + func with( + fileSystem: FileSystem, + artifact: Data, + swiftSyntaxVersion: String, + swiftSyntaxURL: String? = nil, + run: (Workspace.SignedPrebuiltsManifest, AbsolutePath, MockPackage, MockPackage) async throws -> () + ) async throws { + try await fixture(name: "Signing") { fixturePath in + let swiftSyntaxURL = swiftSyntaxURL ?? "https://github.com/swiftlang/swift-syntax" + + let manifest = Workspace.PrebuiltsManifest(libraries: [ + .init( + name: "MacroSupport", + products: [ + "SwiftSyntaxMacrosTestSupport", + "SwiftCompilerPlugin", + "SwiftSyntaxMacros" + ], + cModules: [ + "_SwiftSyntaxCShims" ], - type: .macro - ), - MockTarget( - name: "Foo", - dependencies: ["FooMacros"] - ), - MockTarget( - name: "FooClient", - dependencies: ["Foo"], - type: .executable - ), - MockTarget( - name: "FooTests", - dependencies: [ - "FooMacros", - .product(name: "SwiftSyntaxMacrosTestSupport", package: "swift-syntax"), + artifacts: [ + .init( + platform: .macos_aarch64, + checksum: SHA256().hash(ByteString(artifact)).hexadecimalRepresentation + ) ] ) - ], - dependencies: [ - .sourceControl( - url: "https://github.com/swiftlang/swift-syntax", - requirement: .exact(try XCTUnwrap(Version(swiftSyntaxVersion))) - ) + ]) + + let certsPath = fixturePath.appending("Certificates") + + let certPaths = [ + certsPath.appending("Test_rsa.cer"), + certsPath.appending("TestIntermediateCA.cer"), + certsPath.appending("TestRootCA.cer"), ] - ) - - let swiftSyntax = try MockPackage( - name: "swift-syntax", - url: "https://github.com/swiftlang/swift-syntax", - targets: [ - MockTarget(name: "SwiftSyntaxMacrosTestSupport"), - MockTarget(name: "SwiftCompilerPlugin"), - MockTarget(name: "SwiftSyntaxMacros"), - ], - products: [ - MockProduct(name: "SwiftSyntaxMacrosTestSupport", modules: ["SwiftSyntaxMacrosTestSupport"]), - MockProduct(name: "SwiftCompilerPlugin", modules: ["SwiftCompilerPlugin"]), - MockProduct(name: "SwiftSyntaxMacros", modules: ["SwiftSyntaxMacros"]), - ], - versions: ["600.0.1", "600.0.2"] - ) - - return (manifest, rootPackage, swiftSyntax) + let privateKeyPath = certsPath.appending("Test_rsa_key.pem") + + // Copy into in memory file system + for path in certPaths + [privateKeyPath] { + try fileSystem.writeFileContents(path, data: Data(contentsOf: path.asURL)) + } + + let rootCertPath = certPaths.last! + let trustDir = certsPath.appending("Trust") + try fileSystem.createDirectory(trustDir, recursive: true) + try fileSystem.copy(from: rootCertPath, to: trustDir.appending(rootCertPath.basename)) + + let signer = ManifestSigning( + trustedRootCertsDir: trustDir, + observabilityScope: ObservabilitySystem { _, diagnostic in print(diagnostic) }.topScope + ) + + let signature = try await signer.sign( + manifest: manifest, + certChainPaths: certPaths, + certPrivateKeyPath: privateKeyPath, + fileSystem: fileSystem + ) + + // Make sure the signing is valid + try await signer.validate(manifest: manifest, signature: signature, fileSystem: fileSystem) + + let signedManifest = Workspace.SignedPrebuiltsManifest(manifest: manifest, signature: signature) + + let rootPackage = try MockPackage( + name: "Foo", + targets: [ + MockTarget( + name: "FooMacros", + dependencies: [ + .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), + .product(name: "SwiftCompilerPlugin", package: "swift-syntax"), + ], + type: .macro + ), + MockTarget( + name: "Foo", + dependencies: ["FooMacros"] + ), + MockTarget( + name: "FooClient", + dependencies: ["Foo"], + type: .executable + ), + MockTarget( + name: "FooTests", + dependencies: [ + "FooMacros", + .product(name: "SwiftSyntaxMacrosTestSupport", package: "swift-syntax"), + ], + type: .test + ), + ], + dependencies: [ + .sourceControl( + url: swiftSyntaxURL, + requirement: .exact(try XCTUnwrap(Version(swiftSyntaxVersion))) + ) + ] + ) + + let swiftSyntax = try MockPackage( + name: "swift-syntax", + url: swiftSyntaxURL, + targets: [ + MockTarget(name: "SwiftSyntaxMacrosTestSupport"), + MockTarget(name: "SwiftCompilerPlugin"), + MockTarget(name: "SwiftSyntaxMacros"), + ], + products: [ + MockProduct(name: "SwiftSyntaxMacrosTestSupport", modules: ["SwiftSyntaxMacrosTestSupport"]), + MockProduct(name: "SwiftCompilerPlugin", modules: ["SwiftCompilerPlugin"]), + MockProduct(name: "SwiftSyntaxMacros", modules: ["SwiftSyntaxMacros"]), + ], + versions: ["600.0.1", "600.0.2", "601.0.0"] + ) + + try await run(signedManifest, rootCertPath, rootPackage, swiftSyntax) + } } - func checkSettings(_ target: Module, usePrebuilt: Bool) throws { + func checkSettings(_ rootPackage: ResolvedPackage, _ targetName: String, usePrebuilt: Bool) throws { + let target = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == targetName })) if usePrebuilt { let swiftFlags = try XCTUnwrap(target.buildSettings.assignments[.OTHER_SWIFT_FLAGS]).flatMap({ $0.values }) - XCTAssertTrue(swiftFlags.contains("-I\(AbsolutePath("/tmp/ws/.build/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64/Modules").pathString)")) - XCTAssertTrue(swiftFlags.contains("-I\(AbsolutePath("/tmp/ws/.build/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64/include/_SwiftSyntaxCShims").pathString)")) + XCTAssertTrue(swiftFlags.contains("-I/tmp/ws/.build/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64/Modules".fixwin)) + XCTAssertTrue(swiftFlags.contains("-I/tmp/ws/.build/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64/include/_SwiftSyntaxCShims".fixwin)) let ldFlags = try XCTUnwrap(target.buildSettings.assignments[.OTHER_LDFLAGS]).flatMap({ $0.values }) - XCTAssertTrue(ldFlags.contains(AbsolutePath("/tmp/ws/.build/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64/lib/libMacroSupport.a").pathString)) + XCTAssertTrue(ldFlags.contains("/tmp/ws/.build/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64/lib/libMacroSupport.a".fixwin)) } else { XCTAssertNil(target.buildSettings.assignments[.OTHER_SWIFT_FLAGS]) XCTAssertNil(target.buildSettings.assignments[.OTHER_LDFLAGS]) @@ -118,466 +167,736 @@ final class PrebuiltsTests: XCTestCase { func testSuccessPath() async throws { let sandbox = AbsolutePath("/tmp/ws") let fs = InMemoryFileSystem() + let artifact = Data() + + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.1") { manifest, rootCertPath, rootPackage, swiftSyntax in + let manifestData = try JSONEncoder().encode(manifest) + + let httpClient = HTTPClient { request, progressHandler in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" { + try fileSystem.writeFileContents(destination, data: manifestData) + return .okay() + } else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" { + try fileSystem.writeFileContents(destination, data: artifact) + return .okay() + } else { + XCTFail("Unexpected URL \(request.url)") + return .notFound() + } + } + + let archiver = MockArchiver(handler: { _, archivePath, destination, completion in + XCTAssertEqual(archivePath, sandbox.appending(components: ".build", "prebuilts", "swift-syntax", "600.0.1", "\(self.swiftVersion)-MacroSupport-macos_aarch64.zip")) + XCTAssertEqual(destination, sandbox.appending(components: ".build", "prebuilts", "swift-syntax", "600.0.1", "\(self.swiftVersion)-MacroSupport-macos_aarch64")) + completion(.success(())) + }) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ], + prebuiltsManager: .init( + httpClient: httpClient, + archiver: archiver, + hostPlatform: .macos_aarch64, + rootCertPath: rootCertPath + ) + ) + + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error || $0.severity == .warning }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: true) + try checkSettings(rootPackage, "FooTests", usePrebuilt: true) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } + } + } + func testVersionChange() async throws { + let sandbox = AbsolutePath("/tmp/ws") + let fs = InMemoryFileSystem() let artifact = Data() - let (manifest, rootPackage, swiftSyntax) = try initData(artifact: artifact, swiftSyntaxVersion: "600.0.1") - let manifestData = try JSONEncoder().encode(manifest) - let httpClient = HTTPClient { request, progressHandler in - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.1") { manifest, rootCertPath, rootPackage, swiftSyntax in + let manifestData = try JSONEncoder().encode(manifest) + + let httpClient = HTTPClient { request, progressHandler in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" { + try fileSystem.writeFileContents(destination, data: manifestData) + return .okay() + } else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" { + try fileSystem.writeFileContents(destination, data: artifact) + return .okay() + } else { + // make sure it's the updated one + XCTAssertEqual( + request.url, + "https://download.swift.org/prebuilts/swift-syntax/601.0.0/\(self.swiftVersion)-manifest.json" + ) + return .notFound() + } + } + + let archiver = MockArchiver(handler: { _, archivePath, destination, completion in + XCTAssertEqual(archivePath, sandbox.appending(components: ".build", "prebuilts", "swift-syntax", "600.0.1", "\(self.swiftVersion)-MacroSupport-macos_aarch64.zip")) + XCTAssertEqual(destination, sandbox.appending(components: ".build", "prebuilts", "swift-syntax", "600.0.1", "\(self.swiftVersion)-MacroSupport-macos_aarch64")) + completion(.success(())) + }) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ], + prebuiltsManager: .init( + httpClient: httpClient, + archiver: archiver, + hostPlatform: .macos_aarch64, + rootCertPath: rootCertPath + ) + ) + + try await workspace.checkPackageGraph(roots: [rootPackage.name]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error || $0.severity == .warning }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: true) + try checkSettings(rootPackage, "FooTests", usePrebuilt: true) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } + + // Change the version of swift syntax to one that doesn't have prebuilts + try await workspace.closeWorkspace(resetState: false, resetResolvedFile: false) + let key = MockManifestLoader.Key(url: sandbox.appending(components: "roots", rootPackage.name).pathString) + let oldManifest = try XCTUnwrap(workspace.manifestLoader.manifests[key]) + let oldSCM: PackageDependency.SourceControl + if case let .sourceControl(scm) = oldManifest.dependencies[0] { + oldSCM = scm + } else { + XCTFail("not source control") + return + } + let newDep = PackageDependency.sourceControl( + identity: oldSCM.identity, + nameForTargetDependencyResolutionOnly: oldSCM.nameForTargetDependencyResolutionOnly, + location: oldSCM.location, + requirement: .exact(try XCTUnwrap(Version("601.0.0"))), + productFilter: oldSCM.productFilter + ) + let newManifest = oldManifest.with(dependencies: [newDep]) + workspace.manifestLoader.manifests[key] = newManifest + + try await workspace.checkPackageGraph(roots: [rootPackage.name]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error || $0.severity == .warning }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: false) + try checkSettings(rootPackage, "FooTests", usePrebuilt: false) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) } - if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" { - try fileSystem.writeFileContents(destination, data: manifestData) - return .okay() - } else if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" { - try fileSystem.writeFileContents(destination, data: artifact) - return .okay() - } else { - XCTFail("Unexpected URL \(request.url)") - return .notFound() + // Change it back + try await workspace.closeWorkspace(resetState: false, resetResolvedFile: false) + workspace.manifestLoader.manifests[key] = oldManifest + + try await workspace.checkPackageGraph(roots: [rootPackage.name]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error || $0.severity == .warning }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: true) + try checkSettings(rootPackage, "FooTests", usePrebuilt: true) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) } } + } - let archiver = MockArchiver(handler: { _, archivePath, destination, completion in - XCTAssertEqual(archivePath.pathString, AbsolutePath("/home/user/caches/org.swift.swiftpm/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip").pathString) - XCTAssertEqual(destination.pathString, AbsolutePath("/tmp/ws/.build/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64").pathString) - completion(.success(())) - }) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - rootPackage - ], - packages: [ - swiftSyntax - ], - prebuiltsManager: .init( - httpClient: httpClient, - archiver: archiver - ), - customHostTriple: Triple("arm64-apple-macosx15.0") - ) - - try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in - XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) - let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) - let macroTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooMacros" })) - try checkSettings(macroTarget, usePrebuilt: true) - let testTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooTests" })) - try checkSettings(testTarget, usePrebuilt: true) + func testSSHURL() async throws { + let sandbox = AbsolutePath("/tmp/ws") + let fs = InMemoryFileSystem() + let artifact = Data() + + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.1", swiftSyntaxURL: "git@github.com:swiftlang/swift-syntax.git") { + manifest, rootCertPath, rootPackage, swiftSyntax in + + let manifestData = try JSONEncoder().encode(manifest) + + let httpClient = HTTPClient { request, progressHandler in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" { + try fileSystem.writeFileContents(destination, data: manifestData) + return .okay() + } else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" { + try fileSystem.writeFileContents(destination, data: artifact) + return .okay() + } else { + XCTFail("Unexpected URL \(request.url)") + return .notFound() + } + } + + let archiver = MockArchiver(handler: { _, archivePath, destination, completion in + XCTAssertEqual(archivePath, sandbox.appending(components: ".build", "prebuilts", "swift-syntax", "600.0.1", "\(self.swiftVersion)-MacroSupport-macos_aarch64.zip")) + XCTAssertEqual(destination, sandbox.appending(components: ".build", "prebuilts", "swift-syntax", "600.0.1", "\(self.swiftVersion)-MacroSupport-macos_aarch64")) + completion(.success(())) + }) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ], + prebuiltsManager: .init( + httpClient: httpClient, + archiver: archiver, + hostPlatform: .macos_aarch64, + rootCertPath: rootCertPath + ) + ) + + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: true) + try checkSettings(rootPackage, "FooTests", usePrebuilt: true) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } } } func testCachedArtifact() async throws { let sandbox = AbsolutePath("/tmp/ws") let fs = InMemoryFileSystem() - let artifact = Data() - let cacheFile = try AbsolutePath(validating: "/home/user/caches/org.swift.swiftpm/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip") + let cacheFile = try AbsolutePath(validating: "/home/user/caches/org.swift.swiftpm/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip") try fs.writeFileContents(cacheFile, data: artifact) - let (manifest, rootPackage, swiftSyntax) = try initData(artifact: artifact, swiftSyntaxVersion: "600.0.1") - let manifestData = try JSONEncoder().encode(manifest) - - let httpClient = HTTPClient { request, progressHandler in - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.1") { manifest, rootCertPath, rootPackage, swiftSyntax in + let manifestData = try JSONEncoder().encode(manifest) + + let httpClient = HTTPClient { request, progressHandler in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" { + try fileSystem.writeFileContents(destination, data: manifestData) + return .okay() + } else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" { + XCTFail("Unexpect download of archive") + try fileSystem.writeFileContents(destination, data: artifact) + return .okay() + } else { + XCTFail("Unexpected URL \(request.url)") + return .notFound() + } } - if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" { - try fileSystem.writeFileContents(destination, data: manifestData) - return .okay() - } else if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" { - XCTFail("Unexpect download of archive") - try fileSystem.writeFileContents(destination, data: artifact) - return .okay() - } else { - XCTFail("Unexpected URL \(request.url)") - return .notFound() - } - } + let archiver = MockArchiver(handler: { _, archivePath, destination, completion in + XCTAssertEqual(archivePath, sandbox.appending(components: ".build", "prebuilts", "swift-syntax", "600.0.1", "\(self.swiftVersion)-MacroSupport-macos_aarch64.zip")) + XCTAssertEqual(destination, sandbox.appending(components: ".build", "prebuilts", "swift-syntax", "600.0.1", "\(self.swiftVersion)-MacroSupport-macos_aarch64")) + completion(.success(())) + }) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ], + prebuiltsManager: .init( + httpClient: httpClient, + archiver: archiver, + hostPlatform: .macos_aarch64, + rootCertPath: rootCertPath + ) + ) - let archiver = MockArchiver(handler: { _, archivePath, destination, completion in - XCTAssertEqual(archivePath.pathString, AbsolutePath("/home/user/caches/org.swift.swiftpm/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip").pathString) - XCTAssertEqual(destination.pathString, AbsolutePath("/tmp/ws/.build/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64").pathString) - completion(.success(())) - }) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - rootPackage - ], - packages: [ - swiftSyntax - ], - prebuiltsManager: .init( - httpClient: httpClient, - archiver: archiver - ), - customHostTriple: Triple("arm64-apple-macosx15.0") - ) - - try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in - XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) - let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) - let macroTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooMacros" })) - try checkSettings(macroTarget, usePrebuilt: true) - let testTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooTests" })) - try checkSettings(testTarget, usePrebuilt: true) + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: true) + try checkSettings(rootPackage, "FooTests", usePrebuilt: true) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } } } func testUnsupportedSwiftSyntaxVersion() async throws { let sandbox = AbsolutePath("/tmp/ws") let fs = InMemoryFileSystem() - let artifact = Data() - let (_, rootPackage, swiftSyntax) = try initData(artifact: artifact, swiftSyntaxVersion: "600.0.2") - - let httpClient = HTTPClient { request, progressHandler in - if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.2/\(self.swiftVersion)-manifest.json" { - return .notFound() - } else { - XCTFail("Unexpected URL \(request.url)") - return .notFound() - } - } - let archiver = MockArchiver(handler: { _, archivePath, destination, completion in - XCTFail("Unexpected call to archiver") - completion(.success(())) - }) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - rootPackage - ], - packages: [ - swiftSyntax - ], - prebuiltsManager: .init( - httpClient: httpClient, - archiver: archiver + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.2") { _, rootCertPath, rootPackage, swiftSyntax in + let secondFetch = SendableBox(false) + + let httpClient = HTTPClient { request, progressHandler in + if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.2/\(self.swiftVersion)-manifest.json" { + let secondFetch = await secondFetch.value + XCTAssertFalse(secondFetch, "unexpected second fetch") + return .notFound() + } else { + XCTFail("Unexpected URL \(request.url)") + return .notFound() + } + } + + let archiver = MockArchiver(handler: { _, archivePath, destination, completion in + XCTFail("Unexpected call to archiver") + completion(.success(())) + }) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ], + prebuiltsManager: .init( + httpClient: httpClient, + archiver: archiver, + rootCertPath: rootCertPath + ) ) - ) - - try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in - XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) - let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) - let macroTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooMacros" })) - try checkSettings(macroTarget, usePrebuilt: false) - let testTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooTests" })) - try checkSettings(testTarget, usePrebuilt: false) + + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: false) + try checkSettings(rootPackage, "FooTests", usePrebuilt: false) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } + + await secondFetch.set(true) + + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: false) + try checkSettings(rootPackage, "FooTests", usePrebuilt: false) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } } } func testUnsupportedArch() async throws { let sandbox = AbsolutePath("/tmp/ws") let fs = InMemoryFileSystem() - let artifact = Data() - let (manifest, rootPackage, swiftSyntax) = try initData(artifact: artifact, swiftSyntaxVersion: "600.0.1") - let manifestData = try JSONEncoder().encode(manifest) - let httpClient = HTTPClient { request, progressHandler in - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.1") { manifest, rootCertPath, rootPackage, swiftSyntax in + let manifestData = try JSONEncoder().encode(manifest) + + let httpClient = HTTPClient { request, progressHandler in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" { + try fileSystem.writeFileContents(destination, data: manifestData) + return .okay() + } else { + XCTFail("Unexpected URL \(request.url)") + return .notFound() + } } - if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" { - try fileSystem.writeFileContents(destination, data: manifestData) - return .okay() - } else { - XCTFail("Unexpected URL \(request.url)") - return .notFound() - } - } + let archiver = MockArchiver(handler: { _, archivePath, destination, completion in + XCTFail("Unexpected call to archiver") + completion(.success(())) + }) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ], + prebuiltsManager: .init( + httpClient: httpClient, + archiver: archiver, + hostPlatform: .ubuntu_noble_x86_64, + rootCertPath: rootCertPath + ) + ) - let archiver = MockArchiver(handler: { _, archivePath, destination, completion in - XCTFail("Unexpected call to archiver") - completion(.success(())) - }) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - rootPackage - ], - packages: [ - swiftSyntax - ], - prebuiltsManager: .init( - httpClient: httpClient, - archiver: archiver - ), - customHostTriple: Triple("86_64-unknown-linux-gnu") - ) - - try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in - XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) - let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) - let macroTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooMacros" })) - try checkSettings(macroTarget, usePrebuilt: false) - let testTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooTests" })) - try checkSettings(testTarget, usePrebuilt: false) + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: false) + try checkSettings(rootPackage, "FooTests", usePrebuilt: false) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } } } func testUnsupportedSwiftVersion() async throws { let sandbox = AbsolutePath("/tmp/ws") let fs = InMemoryFileSystem() - let artifact = Data() - let (_, rootPackage, swiftSyntax) = try initData(artifact: artifact, swiftSyntaxVersion: "600.0.1") - - let httpClient = HTTPClient { request, progressHandler in - if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" { - // Pretend it's a different swift version - return .notFound() - } else { - XCTFail("Unexpected URL \(request.url)") - return .notFound() + + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.1") { _, rootCertPath, rootPackage, swiftSyntax in + let httpClient = HTTPClient { request, progressHandler in + if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" { + // Pretend it's a different swift version + return .notFound() + } else { + XCTFail("Unexpected URL \(request.url)") + return .notFound() + } } - } - let archiver = MockArchiver(handler: { _, archivePath, destination, completion in - XCTFail("Unexpected call to archiver") - completion(.success(())) - }) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - rootPackage - ], - packages: [ - swiftSyntax - ], - prebuiltsManager: .init( - httpClient: httpClient, - archiver: archiver + let archiver = MockArchiver(handler: { _, archivePath, destination, completion in + XCTFail("Unexpected call to archiver") + completion(.success(())) + }) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ], + prebuiltsManager: .init( + httpClient: httpClient, + archiver: archiver, + rootCertPath: rootCertPath + ) ) - ) - - try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in - XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) - let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) - let macroTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooMacros" })) - try checkSettings(macroTarget, usePrebuilt: false) - let testTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooTests" })) - try checkSettings(testTarget, usePrebuilt: false) - } + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: false) + try checkSettings(rootPackage, "FooTests", usePrebuilt: false) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } + } } - func testBadChecksumHttp() async throws { + func testBadSignature() async throws { let sandbox = AbsolutePath("/tmp/ws") let fs = InMemoryFileSystem() - let artifact = Data() - let (manifest, rootPackage, swiftSyntax) = try initData(artifact: artifact, swiftSyntaxVersion: "600.0.1") - let manifestData = try JSONEncoder().encode(manifest) - - let fakeArtifact = Data([56]) - let httpClient = HTTPClient { request, progressHandler in - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.1") { goodManifest, rootCertPath, rootPackage, swiftSyntax in + // Make a change in the manifest + var manifest = goodManifest.manifest + manifest.libraries[0].artifacts[0] = .init(platform: manifest.libraries[0].artifacts[0].platform, checksum: "BAD") + let badManifest = Workspace.SignedPrebuiltsManifest( + manifest: manifest, + signature: goodManifest.signature + ) + let manifestData = try JSONEncoder().encode(badManifest) + + let fakeArtifact = Data([56]) + + let httpClient = HTTPClient { request, progressHandler in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" { + try fileSystem.writeFileContents(destination, data: manifestData) + return .okay() + } else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" { + try fileSystem.writeFileContents(destination, data: fakeArtifact) + return .okay() + } else { + XCTFail("Unexpected URL \(request.url)") + return .notFound() + } } - if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" { - try fileSystem.writeFileContents(destination, data: manifestData) - return .okay() - } else if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" { - try fileSystem.writeFileContents(destination, data: fakeArtifact) - return .okay() - } else { - XCTFail("Unexpected URL \(request.url)") - return .notFound() + let archiver = MockArchiver(handler: { _, archivePath, destination, completion in + XCTFail("Unexpected call to archiver") + completion(.success(())) + }) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ], + prebuiltsManager: .init( + httpClient: httpClient, + archiver: archiver, + hostPlatform: .macos_aarch64, + rootCertPath: rootCertPath + ) + ) + + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) + XCTAssertTrue(diagnostics.contains(where: { $0.message == "Failed to decode prebuilt manifest: invalidSignature" })) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: false) + try checkSettings(rootPackage, "FooTests", usePrebuilt: false) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) } } + } + + func testBadChecksumHttp() async throws { + let sandbox = AbsolutePath("/tmp/ws") + let fs = InMemoryFileSystem() + let artifact = Data() + + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.1") { manifest, rootCertPath, rootPackage, swiftSyntax in + let manifestData = try JSONEncoder().encode(manifest) + + let fakeArtifact = Data([56]) + + let httpClient = HTTPClient { request, progressHandler in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" { + try fileSystem.writeFileContents(destination, data: manifestData) + return .okay() + } else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" { + try fileSystem.writeFileContents(destination, data: fakeArtifact) + return .okay() + } else { + XCTFail("Unexpected URL \(request.url)") + return .notFound() + } + } + + let archiver = MockArchiver(handler: { _, archivePath, destination, completion in + XCTFail("Unexpected call to archiver") + completion(.success(())) + }) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ], + prebuiltsManager: .init( + httpClient: httpClient, + archiver: archiver, + hostPlatform: .macos_aarch64, + rootCertPath: rootCertPath + ) + ) - let archiver = MockArchiver(handler: { _, archivePath, destination, completion in - XCTFail("Unexpected call to archiver") - completion(.success(())) - }) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - rootPackage - ], - packages: [ - swiftSyntax - ], - prebuiltsManager: .init( - httpClient: httpClient, - archiver: archiver - ), - customHostTriple: Triple("arm64-apple-macosx15.0") - ) - - try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in - XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) - let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) - let macroTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooMacros" })) - try checkSettings(macroTarget, usePrebuilt: false) - let testTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooTests" })) - try checkSettings(testTarget, usePrebuilt: false) + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: false) + try checkSettings(rootPackage, "FooTests", usePrebuilt: false) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } } } func testBadChecksumCache() async throws { let sandbox = AbsolutePath("/tmp/ws") let fs = InMemoryFileSystem() - let artifact = Data() - let (manifest, rootPackage, swiftSyntax) = try initData(artifact: artifact, swiftSyntaxVersion: "600.0.1") - let manifestData = try JSONEncoder().encode(manifest) - - let fakeArtifact = Data([56]) - let cacheFile = try AbsolutePath(validating: "/home/user/caches/org.swift.swiftpm/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip") - try fs.writeFileContents(cacheFile, data: fakeArtifact) - let httpClient = HTTPClient { request, progressHandler in - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.1") { manifest, rootCertPath, rootPackage, swiftSyntax in + let manifestData = try JSONEncoder().encode(manifest) + + let fakeArtifact = Data([56]) + let cacheFile = try AbsolutePath(validating: "/home/user/caches/org.swift.swiftpm/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip") + try fs.writeFileContents(cacheFile, data: fakeArtifact) + + let httpClient = HTTPClient { request, progressHandler in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" { + try fileSystem.writeFileContents(destination, data: manifestData) + return .okay() + } else if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" { + try fileSystem.writeFileContents(destination, data: artifact) + return .okay() + } else { + XCTFail("Unexpected URL \(request.url)") + return .notFound() + } } - if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" { - try fileSystem.writeFileContents(destination, data: manifestData) - return .okay() - } else if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip" { - try fileSystem.writeFileContents(destination, data: artifact) - return .okay() - } else { - XCTFail("Unexpected URL \(request.url)") - return .notFound() - } - } + let archiver = MockArchiver(handler: { _, archivePath, destination, completion in + XCTAssertEqual(archivePath, sandbox.appending(components: ".build", "prebuilts", "swift-syntax", "600.0.1", "\(self.swiftVersion)-MacroSupport-macos_aarch64.zip")) + XCTAssertEqual(destination, sandbox.appending(components: ".build", "prebuilts", "swift-syntax", "600.0.1", "\(self.swiftVersion)-MacroSupport-macos_aarch64")) + completion(.success(())) + }) + + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ], + prebuiltsManager: .init( + httpClient: httpClient, + archiver: archiver, + hostPlatform: .macos_aarch64, + rootCertPath: rootCertPath + ) + ) - let archiver = MockArchiver(handler: { _, archivePath, destination, completion in - XCTAssertEqual(archivePath.pathString, AbsolutePath("/home/user/caches/org.swift.swiftpm/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64.zip").pathString) - XCTAssertEqual(destination.pathString, AbsolutePath("/tmp/ws/.build/prebuilts/swift-syntax/\(self.swiftVersion)-MacroSupport-macos_aarch64").pathString) - completion(.success(())) - }) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - rootPackage - ], - packages: [ - swiftSyntax - ], - prebuiltsManager: .init( - httpClient: httpClient, - archiver: archiver - ), - customHostTriple: Triple("arm64-apple-macosx15.0") - ) - - try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in - XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) - let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) - let macroTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooMacros" })) - try checkSettings(macroTarget, usePrebuilt: true) - let testTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooTests" })) - try checkSettings(testTarget, usePrebuilt: true) + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: true) + try checkSettings(rootPackage, "FooTests", usePrebuilt: true) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } } } func testBadManifest() async throws { let sandbox = AbsolutePath("/tmp/ws") let fs = InMemoryFileSystem() - let artifact = Data() - let (manifest, rootPackage, swiftSyntax) = try initData(artifact: artifact, swiftSyntaxVersion: "600.0.1") - let manifestData = try JSONEncoder().encode(manifest) - let httpClient = HTTPClient { request, progressHandler in - guard case .download(let fileSystem, let destination) = request.kind else { - throw StringError("invalid request \(request.kind)") + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.1") { manifest, _, rootPackage, swiftSyntax in + let manifestData = try JSONEncoder().encode(manifest) + + let httpClient = HTTPClient { request, progressHandler in + guard case .download(let fileSystem, let destination) = request.kind else { + throw StringError("invalid request \(request.kind)") + } + + if request.url == "https://download.swift.org/prebuilts/swift-syntax/600.0.1/\(self.swiftVersion)-manifest.json" { + let badManifestData = manifestData + Data("bad".utf8) + try fileSystem.writeFileContents(destination, data: badManifestData) + return .okay() + } else { + XCTFail("Unexpected URL \(request.url)") + return .notFound() + } } - if request.url == "https://github.com/dschaefer2/swift-syntax/releases/download/600.0.1/\(self.swiftVersion)-manifest.json" { - let badManifestData = manifestData + Data("bad".utf8) - try fileSystem.writeFileContents(destination, data: badManifestData) - return .okay() - } else { - XCTFail("Unexpected URL \(request.url)") - return .notFound() - } - } + let archiver = MockArchiver(handler: { _, archivePath, destination, completion in + XCTFail("Unexpected call to archiver") + completion(.success(())) + }) - let archiver = MockArchiver(handler: { _, archivePath, destination, completion in - XCTFail("Unexpected call to archiver") - completion(.success(())) - }) - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - rootPackage - ], - packages: [ - swiftSyntax - ], - prebuiltsManager: .init( - httpClient: httpClient, - archiver: archiver + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ], + prebuiltsManager: .init( + httpClient: httpClient, + archiver: archiver + ) ) - ) - - try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in - XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) - let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) - let macroTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooMacros" })) - try checkSettings(macroTarget, usePrebuilt: false) - let testTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooTests" })) - try checkSettings(testTarget, usePrebuilt: false) + + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: false) + try checkSettings(rootPackage, "FooTests", usePrebuilt: false) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } } } func testDisabled() async throws { let sandbox = AbsolutePath("/tmp/ws") let fs = InMemoryFileSystem() - let artifact = Data() - let (_, rootPackage, swiftSyntax) = try initData(artifact: artifact, swiftSyntaxVersion: "600.0.1") - - let workspace = try await MockWorkspace( - sandbox: sandbox, - fileSystem: fs, - roots: [ - rootPackage - ], - packages: [ - swiftSyntax - ], - customHostTriple: Triple("arm64-apple-macosx15.0") - ) - - try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in - XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) - let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) - let macroTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooMacros" })) - try checkSettings(macroTarget, usePrebuilt: false) - let testTarget = try XCTUnwrap(rootPackage.underlying.modules.first(where: { $0.name == "FooTests" })) - try checkSettings(testTarget, usePrebuilt: false) + + try await with(fileSystem: fs, artifact: artifact, swiftSyntaxVersion: "600.0.1") { _, _, rootPackage, swiftSyntax in + let workspace = try await MockWorkspace( + sandbox: sandbox, + fileSystem: fs, + roots: [ + rootPackage + ], + packages: [ + swiftSyntax + ] + ) + + try await workspace.checkPackageGraph(roots: ["Foo"]) { modulesGraph, diagnostics in + XCTAssertTrue(diagnostics.filter({ $0.severity == .error }).isEmpty) + let rootPackage = try XCTUnwrap(modulesGraph.rootPackages.first) + try checkSettings(rootPackage, "FooMacros", usePrebuilt: false) + try checkSettings(rootPackage, "FooTests", usePrebuilt: false) + try checkSettings(rootPackage, "Foo", usePrebuilt: false) + try checkSettings(rootPackage, "FooClient", usePrebuilt: false) + } } } } + +extension String { + var fixwin: String { + #if os(Windows) + return self.replacingOccurrences(of: "/", with: "\\") + #else + return self + #endif + } +}