Merge remote-tracking branch 'origin/darwin' into develop

bindings/swift/OuisyncLib/.gitignore

@@ -1,3 +1,4 @@
+output
 .DS_Store
 /.build
 /Packages

bindings/swift/OuisyncLib/Package.swift

@@ -1,32 +1,39 @@
-// swift-tools-version: 5.8
-// The swift-tools-version declares the minimum version of Swift required to build this package.
-
+// swift-tools-version: 5.9
 import PackageDescription
 
 let package = Package(
     name: "OuisyncLib",
     platforms: [.macOS(.v13), .iOS(.v13), .macCatalyst(.v13)],
     products: [
-        // Products define the executables and libraries a package produces, making them visible to other packages.
-        .library(
-            name: "OuisyncLib",
-            targets: ["OuisyncLib"]),
+        .library(name: "OuisyncLib",
+                 type: .static,
+                 targets: ["OuisyncLib"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/a2/MessagePack.swift.git", from: "4.0.0")
+        .package(url: "https://github.com/a2/MessagePack.swift.git", from: "4.0.0"),
     ],
     targets: [
-        // Targets are the basic building blocks of a package, defining a module or a test suite.
-        // Targets can depend on other targets in this package and products from dependencies.
-        .target(
-            name: "OuisyncLib",
-            dependencies: [.product(name:"MessagePack", package: "MessagePack.swift"), "OuisyncLibFFI"]
-        ),
-        .target(name: "OuisyncLibFFI", dependencies: ["OuisyncDyLibBuilder"]),
-        .plugin(name: "OuisyncDyLibBuilder", capability: .buildTool()),
-
-        .testTarget(
-            name: "OuisyncLibTests",
-            dependencies: ["OuisyncLib"]),
+        .target(name: "OuisyncLib",
+                dependencies: [.product(name: "MessagePack",
+                                        package: "MessagePack.swift"),
+                               "FFIBuilder", "OuisyncLibFFI"],
+                path: "Sources"),
+        .testTarget(name: "OuisyncLibTests",
+                    dependencies: ["OuisyncLib"],
+                    path: "Tests"),
+        // FIXME: move this to a separate package / framework
+        .binaryTarget(name: "OuisyncLibFFI",
+                      path: "output/OuisyncLibFFI.xcframework"),
+        .plugin(name: "FFIBuilder",
+                capability: .buildTool(),
+                path: "Plugins/Builder"),
+        .plugin(name: "Update rust dependencies",
+                capability: .command(intent: .custom(verb: "cargo-fetch",
+                                                     description: "Updates rust dependencies"),
+                                     permissions: [
+                .allowNetworkConnections(scope: .all(),
+                                         reason: "Downloads dependencies defined by Cargo.toml"),
+                .writeToPackageDirectory(reason: "These are not the droids you are looking for")]),
+                path: "Plugins/Updater"),
     ]
 )

bindings/swift/OuisyncLib/Plugins/Builder/builder.swift

@@ -0,0 +1,102 @@
+/* Swift package manager build plugin: currently invokes `Builder` before every build.
+
+
+ Ideally, a `.buildTool()`[1] plugin[2][3][4] is expected to provide makefile-like rules mapping
+ supplied files to their requirements, which are then used by the build system to only compile the
+ necessary targets when they are needed upstream and their inputs have changed.
+
+ While this would be sufficient when coupled with re-running the builder plugin (e.g. re-reading
+ the makefile) on every build, certain acknowledged[5] missing features[6] currently prevent us
+ from using `.buildTool()` as intended. Instead, we currently opt for `.prebuildTool()` and mostly
+ rely on `cargo` to compile incrementally while avoiding unnecessary work.
+
+ [1] https://developer.apple.com/documentation/packagedescription/target/plugincapability-swift.enum/buildtool())
+ [2] https://github.com/swiftlang/swift-package-manager/blob/main/Documentation/Plugins.md
+ [3] https://github.com/swiftlang/swift-evolution/blob/main/proposals/0303-swiftpm-extensible-build-tools.md
+ [4] https://github.com/swiftlang/swift-evolution/blob/main/proposals/0325-swiftpm-additional-plugin-apis.md
+ [5] https://forums.swift.org/t/swiftpm-trouble-accessing-non-swift-buildtool-artifacts-in-derived-data-path/64444
+ [6] https://forums.swift.org/t/build-dependent-packages-automatically/69648
+ */
+import Foundation
+import PackagePlugin
+
+@main struct TreeReconciler: BuildToolPlugin {
+    func panic(_ msg: String) -> Never {
+        Diagnostics.error(msg)
+        fatalError("Unable to build LibOuisyncFFI.xcframework")
+    }
+
+    func createBuildCommands(context: PackagePlugin.PluginContext,
+                             target: PackagePlugin.Target) async throws -> [PackagePlugin.Command] {
+        let output = context.pluginWorkDirectory
+
+        let dependencies = output
+            .removingLastComponent() // FFIBuilder
+            .removingLastComponent() // OuisyncLibFFI
+            .removingLastComponent() // ouisync.output
+            .appending("Update rust dependencies.output")
+
+        guard FileManager.default.fileExists(atPath: dependencies.string)
+        else { panic("Please run `Update rust dependencies` on the OuisyncLib package") }
+
+        let manifest = context.package.directory
+            .removingLastComponent() // OuisyncLib
+            .removingLastComponent() // swift
+            .removingLastComponent() // bindings
+            .appending("Cargo.toml")
+
+        return [.prebuildCommand(displayName: "Build OuisyncLibFFI.xcframework",
+                                 executable: context.package.directory.appending("Plugins").appending("build.sh"),
+                                 arguments: [which("cargo"),
+                                             which("rustc"),
+                                             manifest.string,
+                                             output.string,
+                                             dependencies],
+                                 environment: ProcessInfo.processInfo.environment,
+                                 outputFilesDirectory: output.appending("dummy"))]
+    }
+
+    // runs `which binary` in the default shell and returns the path after confirming that it exists
+    private func which(_ binary: String) -> String {
+        let path = shell("which \(binary)").trimmingCharacters(in: .whitespacesAndNewlines)
+
+        guard FileManager.default.fileExists(atPath: path)
+        else { panic("Unable to find `\(binary)` in environment.") }
+
+        Diagnostics.remark("Found `\(binary)` at \(path)")
+        return path
+    }
+
+    // runs `command` in the default shell and returns stdout on clean exit; throws otherwise
+    private func shell(_ command: String) -> String {
+        exec(command: ProcessInfo.processInfo.environment["SHELL"] ?? "/bin/zsh",
+             with: ["-c", command])
+    }
+
+    // runs `exe` using `args` in `env` and returns `stdout`; panics on non-zero exit
+    @discardableResult private func exec(command exe: String,
+                                         with args: [String] = [],
+                                         in cwd: String? = nil,
+                                         using env: [String: String]? = nil) -> String {
+        let pipe = Pipe()
+        let task = Process()
+        task.standardInput = nil
+        task.standardOutput = pipe
+        task.executableURL = URL(fileURLWithPath: exe)
+        task.arguments = args
+        if let env { task.environment = env }
+        if let cwd { task.currentDirectoryURL = URL(fileURLWithPath: cwd) }
+
+        do { try task.run() } catch { panic("Unable to start \(exe): \(error)") }
+        var stdout: Data?
+        do { stdout = try pipe.fileHandleForReading.readToEnd() }
+        catch { Diagnostics.warning(String(describing: error)) }
+        task.waitUntilExit()
+
+        guard task.terminationReason ~= .exit else { panic("\(exe) killed by \(task.terminationStatus)") }
+        guard task.terminationStatus == 0 else { panic("\(exe) returned \(task.terminationStatus)") }
+        guard let res = String(data: stdout ?? Data(), encoding: .utf8) else { panic("\(exe) produced binary data") }
+
+        return res
+    }
+}

bindings/swift/OuisyncLib/Plugins/Updater/updater.swift

@@ -0,0 +1,120 @@
+/* Swift package manager command plugin: Used to download and compile any rust dependencies
+
+ Due to apple's policies regarding plugin access, this must be run manually manually and granted
+ permission to bypass the sandbox restrictions. Can be run by right clicking OuisyncLib from Xcode
+ or directly from the command line via: `swift package cargo-fetch`.
+
+ For automated tasks, the permissions can be automatically granted on invocation via the
+ `--allow-network-connections` and `--allow-writing-to-package-directory` flags respectively or
+ the sandbox can be disabled altogether via `--disable-sandbox` though the latter is untested. */
+import Foundation
+import PackagePlugin
+
+@main struct UpdateDeps: CommandPlugin {
+    func panic(_ msg: String) -> Never {
+        Diagnostics.error(msg)
+        fatalError("Unable to update rust dependencies")
+    }
+
+    func performCommand(context: PackagePlugin.PluginContext,
+                        arguments: [String] = []) async throws {
+        let package = context.package.directory
+        let manifest = package
+            .removingLastComponent() // OuisyncLib
+            .removingLastComponent() // swift
+            .removingLastComponent() // bindings
+            .appending("Cargo.toml")
+
+        // install package deps
+        let cargo = which("cargo")
+        let rust = which("rustc")
+        let output = context.pluginWorkDirectory
+        exec(command: cargo,
+             with: ["fetch"],
+             using: ["CARGO_HOME": output.string,  // package plugin sandbox forces us to write here
+                     "CARGO_HTTP_CHECK_REVOKE": "false", // this fails in the sandbox, for "reasons"
+                     "MANIFEST_PATH": manifest.string, // path to Cargo.toml, avoids having to chdir
+                     "RUSTC": rust])  // without this cargo assumes $CARGO_HOME/bin/rustc
+
+        // we also have to install cbindgen because running it automatically doesn't always work
+        exec(command:cargo,
+             with: ["install", "--force", "cbindgen"],
+             using: ["CARGO_HOME": output.string,  // package plugin sandbox forces us to write here)
+                     "CARGO_HTTP_CHECK_REVOKE": "false", // this fails in the sandbox, for "reasons"
+                     "RUSTC": rust, // without this cargo assumes $CARGO_HOME/bin/rustc
+                     "PATH": Path(which("cc")).removingLastComponent().string]) // (╯°□°）╯︵ ┻━┻
+        Diagnostics.remark("Dependencies up to date in \(output)!")
+
+        // link project workspace to output folder
+        let dest = context.pluginWorkDirectory
+            .removingLastComponent()
+            .appending("\(context.package.id).output")
+            .appending("OuisyncLib")
+            .appending("FFIBuilder")
+        let link = package.appending("output")
+
+        // create stub framework in output folder
+        exec(command: package.appending("reset-output.sh").string, in: dest.string)
+
+        // replace link
+        let fm = FileManager.default
+        try? fm.removeItem(atPath: link.string)
+        try fm.createSymbolicLink(atPath: link.string, withDestinationPath: dest.appending("output").string)
+
+        // run a build if possible
+        do {
+            let res = try packageManager.build(PackageManager.BuildSubset.target("OuisyncLib"),
+                                               parameters: .init())
+            guard res.succeeded else {
+                Diagnostics.warning(res.logText)
+                throw NSError(domain: "Build failed", code: 1)
+            }
+        } catch {
+            Diagnostics.warning("Unable to auto rebuild: \(error)")
+        }
+    }
+
+    // runs `which binary` in the default shell and returns the path after confirming that it exists
+    private func which(_ binary: String) -> String {
+        let path = shell("which \(binary)").trimmingCharacters(in: .whitespacesAndNewlines)
+
+        guard FileManager.default.fileExists(atPath: path)
+        else { panic("Unable to find `\(binary)` in environment.") }
+
+        Diagnostics.remark("Found `\(binary)` at \(path)")
+        return path
+    }
+
+    // runs `command` in the default shell and returns stdout on clean exit; throws otherwise
+    private func shell(_ command: String) -> String {
+        exec(command: ProcessInfo.processInfo.environment["SHELL"] ?? "/bin/zsh",
+             with: ["-c", command])
+    }
+
+    // runs `exe` using `args` in `env` and returns `stdout`; panics on non-zero exit
+    @discardableResult private func exec(command exe: String,
+                                         with args: [String] = [],
+                                         in cwd: String? = nil,
+                                         using env: [String: String]? = nil) -> String {
+        let pipe = Pipe()
+        let task = Process()
+        task.standardInput = nil
+        task.standardOutput = pipe
+        task.executableURL = URL(fileURLWithPath: exe)
+        task.arguments = args
+        if let env { task.environment = env }
+        if let cwd { task.currentDirectoryURL = URL(fileURLWithPath: cwd) }
+
+        do { try task.run() } catch { panic("Unable to start \(exe): \(error)") }
+        var stdout: Data?
+        do { stdout = try pipe.fileHandleForReading.readToEnd() }
+        catch { Diagnostics.warning(String(describing: error)) }
+        task.waitUntilExit()
+
+        guard task.terminationReason ~= .exit else { panic("\(exe) killed by \(task.terminationStatus)") }
+        guard task.terminationStatus == 0 else { panic("\(exe) returned \(task.terminationStatus)") }
+        guard let res = String(data: stdout ?? Data(), encoding: .utf8) else { panic("\(exe) produced binary data") }
+
+        return res
+    }
+}