diff --git a/Plugins/AWSLambdaPackager/Plugin.swift b/Plugins/AWSLambdaPackager/Plugin.swift index a0ea999b..2e091c74 100644 --- a/Plugins/AWSLambdaPackager/Plugin.swift +++ b/Plugins/AWSLambdaPackager/Plugin.swift @@ -12,13 +12,11 @@ // //===----------------------------------------------------------------------===// -import Dispatch -import Foundation import PackagePlugin -import Synchronization +import Foundation -@available(macOS 15.0, *) @main +@available(macOS 15.0, *) struct AWSLambdaPackager: CommandPlugin { func performCommand(context: PackagePlugin.PluginContext, arguments: [String]) async throws { let configuration = try Configuration(context: context, arguments: arguments) @@ -33,7 +31,7 @@ struct AWSLambdaPackager: CommandPlugin { ) } - let builtProducts: [LambdaProduct: Path] + let builtProducts: [LambdaProduct: URL] if self.isAmazonLinux2() { // build directly on the machine builtProducts = try self.build( @@ -46,9 +44,9 @@ struct AWSLambdaPackager: CommandPlugin { // build with docker builtProducts = try self.buildInDocker( packageIdentity: context.package.id, - packageDirectory: context.package.directory, + packageDirectory: context.package.directoryURL, products: configuration.products, - toolsProvider: { name in try context.tool(named: name).path }, + toolsProvider: { name in try context.tool(named: name).url }, outputDirectory: configuration.outputDirectory, baseImage: configuration.baseDockerImage, disableDockerImageUpdate: configuration.disableDockerImageUpdate, @@ -61,7 +59,7 @@ struct AWSLambdaPackager: CommandPlugin { let archives = try self.package( packageName: context.package.displayName, products: builtProducts, - toolsProvider: { name in try context.tool(named: name).path }, + toolsProvider: { name in try context.tool(named: name).url }, outputDirectory: configuration.outputDirectory, verboseLogging: configuration.verboseLogging ) @@ -70,21 +68,21 @@ struct AWSLambdaPackager: CommandPlugin { "\(archives.count > 0 ? archives.count.description : "no") archive\(archives.count != 1 ? "s" : "") created" ) for (product, archivePath) in archives { - print(" * \(product.name) at \(archivePath.string)") + print(" * \(product.name) at \(archivePath)") } } private func buildInDocker( packageIdentity: Package.ID, - packageDirectory: Path, + packageDirectory: URL, products: [Product], - toolsProvider: (String) throws -> Path, - outputDirectory: Path, + toolsProvider: (String) throws -> URL, + outputDirectory: URL, baseImage: String, disableDockerImageUpdate: Bool, buildConfiguration: PackageManager.BuildConfiguration, verboseLogging: Bool - ) throws -> [LambdaProduct: Path] { + ) throws -> [LambdaProduct: URL] { let dockerToolPath = try toolsProvider("docker") print("-------------------------------------------------------------------------") @@ -94,7 +92,7 @@ struct AWSLambdaPackager: CommandPlugin { if !disableDockerImageUpdate { // update the underlying docker image, if necessary print("updating \"\(baseImage)\" docker image") - try self.execute( + try Utils.execute( executable: dockerToolPath, arguments: ["pull", baseImage], logLevel: .output @@ -103,10 +101,10 @@ struct AWSLambdaPackager: CommandPlugin { // get the build output path let buildOutputPathCommand = "swift build -c \(buildConfiguration.rawValue) --show-bin-path" - let dockerBuildOutputPath = try self.execute( + let dockerBuildOutputPath = try Utils.execute( executable: dockerToolPath, arguments: [ - "run", "--rm", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", baseImage, "bash", + "run", "--rm", "-v", "\(packageDirectory.path()):/workspace", "-w", "/workspace", baseImage, "bash", "-cl", buildOutputPathCommand, ], logLevel: verboseLogging ? .debug : .silent @@ -114,12 +112,10 @@ struct AWSLambdaPackager: CommandPlugin { guard let buildPathOutput = dockerBuildOutputPath.split(separator: "\n").last else { throw Errors.failedParsingDockerOutput(dockerBuildOutputPath) } - let buildOutputPath = Path( - buildPathOutput.replacingOccurrences(of: "/workspace", with: packageDirectory.string) - ) + let buildOutputPath = URL(string: buildPathOutput.replacingOccurrences(of: "/workspace/", with: packageDirectory.description))! // build the products - var builtProducts = [LambdaProduct: Path]() + var builtProducts = [LambdaProduct: URL]() for product in products { print("building \"\(product.name)\"") let buildCommand = @@ -128,30 +124,32 @@ struct AWSLambdaPackager: CommandPlugin { // when developing locally, we must have the full swift-aws-lambda-runtime project in the container // because Examples' Package.swift have a dependency on ../.. // just like Package.swift's examples assume ../.., we assume we are two levels below the root project - let lastComponent = packageDirectory.lastComponent - let beforeLastComponent = packageDirectory.removingLastComponent().lastComponent - try self.execute( + let slice = packageDirectory.pathComponents.suffix(2) + let beforeLastComponent = packageDirectory.pathComponents[slice.startIndex] + let lastComponent = packageDirectory.pathComponents[slice.endIndex-1] + try Utils.execute( executable: dockerToolPath, arguments: [ "run", "--rm", "--env", "LAMBDA_USE_LOCAL_DEPS=true", "-v", - "\(packageDirectory.string)/../..:/workspace", "-w", + "\(packageDirectory.path())../..:/workspace", "-w", "/workspace/\(beforeLastComponent)/\(lastComponent)", baseImage, "bash", "-cl", buildCommand, ], logLevel: verboseLogging ? .debug : .output ) } else { - try self.execute( + try Utils.execute( executable: dockerToolPath, arguments: [ - "run", "--rm", "-v", "\(packageDirectory.string):/workspace", "-w", "/workspace", baseImage, + "run", "--rm", "-v", "\(packageDirectory.path()):/workspace", "-w", "/workspace", baseImage, "bash", "-cl", buildCommand, ], logLevel: verboseLogging ? .debug : .output ) } - let productPath = buildOutputPath.appending(product.name) - guard FileManager.default.fileExists(atPath: productPath.string) else { - Diagnostics.error("expected '\(product.name)' binary at \"\(productPath.string)\"") + let productPath = buildOutputPath.appending(path: product.name) + + guard FileManager.default.fileExists(atPath: productPath.path()) else { + Diagnostics.error("expected '\(product.name)' binary at \"\(productPath.path())\"") throw Errors.productExecutableNotFound(product.name) } builtProducts[.init(product)] = productPath @@ -164,12 +162,12 @@ struct AWSLambdaPackager: CommandPlugin { products: [Product], buildConfiguration: PackageManager.BuildConfiguration, verboseLogging: Bool - ) throws -> [LambdaProduct: Path] { + ) throws -> [LambdaProduct: URL] { print("-------------------------------------------------------------------------") print("building \"\(packageIdentity)\"") print("-------------------------------------------------------------------------") - var results = [LambdaProduct: Path]() + var results = [LambdaProduct: URL]() for product in products { print("building \"\(product.name)\"") var parameters = PackageManager.BuildParameters() @@ -184,7 +182,7 @@ struct AWSLambdaPackager: CommandPlugin { guard let artifact = result.executableArtifact(for: product) else { throw Errors.productExecutableNotFound(product.name) } - results[.init(product)] = artifact.path + results[.init(product)] = artifact.url } return results } @@ -192,34 +190,34 @@ struct AWSLambdaPackager: CommandPlugin { // TODO: explore using ziplib or similar instead of shelling out private func package( packageName: String, - products: [LambdaProduct: Path], - toolsProvider: (String) throws -> Path, - outputDirectory: Path, + products: [LambdaProduct: URL], + toolsProvider: (String) throws -> URL, + outputDirectory: URL, verboseLogging: Bool - ) throws -> [LambdaProduct: Path] { + ) throws -> [LambdaProduct: URL] { let zipToolPath = try toolsProvider("zip") - var archives = [LambdaProduct: Path]() + var archives = [LambdaProduct: URL]() for (product, artifactPath) in products { print("-------------------------------------------------------------------------") print("archiving \"\(product.name)\"") print("-------------------------------------------------------------------------") // prep zipfile location - let workingDirectory = outputDirectory.appending(product.name) - let zipfilePath = workingDirectory.appending("\(product.name).zip") - if FileManager.default.fileExists(atPath: workingDirectory.string) { - try FileManager.default.removeItem(atPath: workingDirectory.string) + let workingDirectory = outputDirectory.appending(path: product.name) + let zipfilePath = workingDirectory.appending(path: "\(product.name).zip") + if FileManager.default.fileExists(atPath: workingDirectory.path()) { + try FileManager.default.removeItem(atPath: workingDirectory.path()) } - try FileManager.default.createDirectory(atPath: workingDirectory.string, withIntermediateDirectories: true) + try FileManager.default.createDirectory(atPath: workingDirectory.path(), withIntermediateDirectories: true) // rename artifact to "bootstrap" - let relocatedArtifactPath = workingDirectory.appending(artifactPath.lastComponent) - let symbolicLinkPath = workingDirectory.appending("bootstrap") - try FileManager.default.copyItem(atPath: artifactPath.string, toPath: relocatedArtifactPath.string) + let relocatedArtifactPath = workingDirectory.appending(path: artifactPath.lastPathComponent) + let symbolicLinkPath = workingDirectory.appending(path: "bootstrap") + try FileManager.default.copyItem(atPath: artifactPath.path(), toPath: relocatedArtifactPath.path()) try FileManager.default.createSymbolicLink( - atPath: symbolicLinkPath.string, - withDestinationPath: relocatedArtifactPath.lastComponent + atPath: symbolicLinkPath.path(), + withDestinationPath: relocatedArtifactPath.lastPathComponent ) var arguments: [String] = [] @@ -227,29 +225,31 @@ struct AWSLambdaPackager: CommandPlugin { arguments = [ "--recurse-paths", "--symlinks", - zipfilePath.lastComponent, - relocatedArtifactPath.lastComponent, - symbolicLinkPath.lastComponent, + zipfilePath.lastPathComponent, + relocatedArtifactPath.lastPathComponent, + symbolicLinkPath.lastPathComponent, ] #else throw Errors.unsupportedPlatform("can't or don't know how to create a zip file on this platform") #endif // add resources - let artifactDirectory = artifactPath.removingLastComponent() + var artifactPathComponents = artifactPath.pathComponents + _ = artifactPathComponents.removeLast() + let artifactDirectory = artifactPathComponents.joined(separator: "/") let resourcesDirectoryName = "\(packageName)_\(product.name).resources" let resourcesDirectory = artifactDirectory.appending(resourcesDirectoryName) - let relocatedResourcesDirectory = workingDirectory.appending(resourcesDirectoryName) - if FileManager.default.fileExists(atPath: resourcesDirectory.string) { + let relocatedResourcesDirectory = workingDirectory.appending(path: resourcesDirectoryName) + if FileManager.default.fileExists(atPath: resourcesDirectory) { try FileManager.default.copyItem( - atPath: resourcesDirectory.string, - toPath: relocatedResourcesDirectory.string + atPath: resourcesDirectory, + toPath: relocatedResourcesDirectory.path() ) arguments.append(resourcesDirectoryName) } // run the zip tool - try self.execute( + try Utils.execute( executable: zipToolPath, arguments: arguments, customWorkingDirectory: workingDirectory, @@ -261,100 +261,6 @@ struct AWSLambdaPackager: CommandPlugin { return archives } - @discardableResult - private func execute( - executable: Path, - arguments: [String], - customWorkingDirectory: Path? = .none, - logLevel: ProcessLogLevel - ) throws -> String { - if logLevel >= .debug { - print("\(executable.string) \(arguments.joined(separator: " "))") - } - - let fd = dup(1) - let stdout = fdopen(fd, "rw")! - defer { fclose(stdout) } - - // We need to use an unsafe transfer here to get the fd into our Sendable closure. - // This transfer is fine, because we guarantee that the code in the outputHandler - // is run before we continue the functions execution, where the fd is used again. - // See `process.waitUntilExit()` and the following `outputSync.wait()` - struct UnsafeTransfer: @unchecked Sendable { - let value: Value - } - - let outputMutex = Mutex("") - let outputSync = DispatchGroup() - let outputQueue = DispatchQueue(label: "AWSLambdaPackager.output") - let unsafeTransfer = UnsafeTransfer(value: stdout) - let outputHandler = { @Sendable (data: Data?) in - dispatchPrecondition(condition: .onQueue(outputQueue)) - - outputSync.enter() - defer { outputSync.leave() } - - guard - let _output = data.flatMap({ - String(data: $0, encoding: .utf8)?.trimmingCharacters(in: CharacterSet(["\n"])) - }), !_output.isEmpty - else { - return - } - - outputMutex.withLock { output in - output += _output + "\n" - } - - switch logLevel { - case .silent: - break - case .debug(let outputIndent), .output(let outputIndent): - print(String(repeating: " ", count: outputIndent), terminator: "") - print(_output) - fflush(unsafeTransfer.value) - } - } - - let pipe = Pipe() - pipe.fileHandleForReading.readabilityHandler = { fileHandle in - outputQueue.async { outputHandler(fileHandle.availableData) } - } - - let process = Process() - process.standardOutput = pipe - process.standardError = pipe - process.executableURL = URL(fileURLWithPath: executable.string) - process.arguments = arguments - if let workingDirectory = customWorkingDirectory { - process.currentDirectoryURL = URL(fileURLWithPath: workingDirectory.string) - } - process.terminationHandler = { _ in - outputQueue.async { - outputHandler(try? pipe.fileHandleForReading.readToEnd()) - } - } - - try process.run() - process.waitUntilExit() - - // wait for output to be full processed - outputSync.wait() - - let output = outputMutex.withLock { $0 } - - if process.terminationStatus != 0 { - // print output on failure and if not already printed - if logLevel < .output { - print(output) - fflush(stdout) - } - throw Errors.processFailed([executable.string] + arguments, process.terminationStatus) - } - - return output - } - private func isAmazonLinux2() -> Bool { if let data = FileManager.default.contents(atPath: "/etc/system-release"), let release = String(data: data, encoding: .utf8) @@ -368,7 +274,7 @@ struct AWSLambdaPackager: CommandPlugin { @available(macOS 15.0, *) private struct Configuration: CustomStringConvertible { - public let outputDirectory: Path + public let outputDirectory: URL public let products: [Product] public let explicitProducts: Bool public let buildConfiguration: PackageManager.BuildConfiguration @@ -392,14 +298,18 @@ private struct Configuration: CustomStringConvertible { self.verboseLogging = verboseArgument if let outputPath = outputPathArgument.first { + #if os(Linux) + var isDirectory: Bool = false + #else var isDirectory: ObjCBool = false - guard FileManager.default.fileExists(atPath: outputPath, isDirectory: &isDirectory), isDirectory.boolValue + #endif + guard FileManager.default.fileExists(atPath: outputPath, isDirectory: &isDirectory) else { throw Errors.invalidArgument("invalid output directory '\(outputPath)'") } - self.outputDirectory = Path(outputPath) + self.outputDirectory = URL(string: outputPath)! } else { - self.outputDirectory = context.pluginWorkDirectory.appending(subpath: "\(AWSLambdaPackager.self)") + self.outputDirectory = context.pluginWorkDirectoryURL.appending(path: "\(AWSLambdaPackager.self)") } self.explicitProducts = !productsArgument.isEmpty @@ -537,7 +447,7 @@ private struct LambdaProduct: Hashable { extension PackageManager.BuildResult { // find the executable produced by the build func executableArtifact(for product: Product) -> PackageManager.BuildResult.BuiltArtifact? { - let executables = self.builtArtifacts.filter { $0.kind == .executable && $0.path.lastComponent == product.name } + let executables = self.builtArtifacts.filter { $0.kind == .executable && $0.url.lastPathComponent == product.name } guard !executables.isEmpty else { return nil } diff --git a/Plugins/AWSLambdaPackager/PluginUtils.swift b/Plugins/AWSLambdaPackager/PluginUtils.swift new file mode 100644 index 00000000..6f60f7c4 --- /dev/null +++ b/Plugins/AWSLambdaPackager/PluginUtils.swift @@ -0,0 +1,156 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftAWSLambdaRuntime open source project +// +// Copyright (c) 2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftAWSLambdaRuntime project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Dispatch +import PackagePlugin +import Synchronization +import Foundation + +@available(macOS 15.0, *) +struct Utils { + @discardableResult + static func execute( + executable: URL, + arguments: [String], + customWorkingDirectory: URL? = .none, + logLevel: ProcessLogLevel + ) throws -> String { + if logLevel >= .debug { + print("\(executable.absoluteString) \(arguments.joined(separator: " "))") + } + + let fd = dup(1) + let stdout = fdopen(fd, "rw") + defer { fclose(stdout) } + + // We need to use an unsafe transfer here to get the fd into our Sendable closure. + // This transfer is fine, because we write to the variable from a single SerialDispatchQueue here. + // We wait until the process is run below process.waitUntilExit(). + // This means no further writes to output will happen. + // This makes it save for us to read the output + struct UnsafeTransfer: @unchecked Sendable { + let value: Value + } + + let outputMutex = Mutex("") + let outputSync = DispatchGroup() + let outputQueue = DispatchQueue(label: "AWSLambdaPackager.output") + let unsafeTransfer = UnsafeTransfer(value: stdout) + let outputHandler = { @Sendable (data: Data?) in + dispatchPrecondition(condition: .onQueue(outputQueue)) + + outputSync.enter() + defer { outputSync.leave() } + + guard + let _output = data.flatMap({ + String(data: $0, encoding: .utf8)?.trimmingCharacters(in: CharacterSet(["\n"])) + }), !_output.isEmpty + else { + return + } + + outputMutex.withLock { output in + output += _output + "\n" + } + + switch logLevel { + case .silent: + break + case .debug(let outputIndent), .output(let outputIndent): + print(String(repeating: " ", count: outputIndent), terminator: "") + print(_output) + fflush(unsafeTransfer.value) + } + } + + let pipe = Pipe() + pipe.fileHandleForReading.readabilityHandler = { fileHandle in + outputQueue.async { outputHandler(fileHandle.availableData) } + } + + let process = Process() + process.standardOutput = pipe + process.standardError = pipe + process.executableURL = URL(fileURLWithPath: executable.description) + process.arguments = arguments + if let workingDirectory = customWorkingDirectory { + process.currentDirectoryURL = URL(fileURLWithPath: workingDirectory.path()) + } + process.terminationHandler = { _ in + outputQueue.async { + outputHandler(try? pipe.fileHandleForReading.readToEnd()) + } + } + + try process.run() + process.waitUntilExit() + + // wait for output to be full processed + outputSync.wait() + + let output = outputMutex.withLock { $0 } + + if process.terminationStatus != 0 { + // print output on failure and if not already printed + if logLevel < .output { + print(output) + fflush(stdout) + } + throw ProcessError.processFailed([executable.path()] + arguments, process.terminationStatus) + } + + return output + } + + enum ProcessError: Error, CustomStringConvertible { + case processFailed([String], Int32) + + var description: String { + switch self { + case .processFailed(let arguments, let code): + return "\(arguments.joined(separator: " ")) failed with code \(code)" + } + } + } + + enum ProcessLogLevel: Comparable { + case silent + case output(outputIndent: Int) + case debug(outputIndent: Int) + + var naturalOrder: Int { + switch self { + case .silent: + return 0 + case .output: + return 1 + case .debug: + return 2 + } + } + + static var output: Self { + .output(outputIndent: 2) + } + + static var debug: Self { + .debug(outputIndent: 2) + } + + static func < (lhs: ProcessLogLevel, rhs: ProcessLogLevel) -> Bool { + lhs.naturalOrder < rhs.naturalOrder + } + } +}