Fix filtered tests (migueldeicaza#624)

* suppress formatting for now

* Reimplemented test harness using EmbeddedXCTest

* Tidied up

Sources/SwiftGodotTestability/EmbeddedXCTest/EmbeddedTestCase.swift

@@ -0,0 +1,34 @@
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+//  Created by Sam Deane on 04/12/24.
+//  All code (c) 2024 - present day, Elegant Chaos Limited.
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+// This code is copied with permission from https://github.com/elegantchaos/EmbeddedXCTest.
+// Check back there for updates.
+
+import XCTest
+
+/// Superclass for test cases that are embedded.
+///
+/// When running normally, the tests will be silent and will do nothing.
+/// When re-run the embedded context, the test case will perform its normal
+/// actions.
+open class EmbeddedTestCase<T: TestHost>: XCTestCase {
+  override open var name: String {
+    return EmbeddingController.isRunningEmbedded ? "Embedded(\(super.name))" : super.name
+  }
+  open override var testRunClass: AnyClass? {
+    EmbeddingController.isRunningEmbedded ? SilentTestRun.self : super.testRunClass
+  }
+
+  open override class func setUp() {
+    EmbeddingController.setUp(hostClass: T.self)
+    super.setUp()
+  }
+
+  open override func run() {
+    if EmbeddingController.isRunningEmbedded {
+      super.run()
+    }
+  }
+}

Sources/SwiftGodotTestability/EmbeddedXCTest/EmbeddedTestCaseSuite.swift

@@ -0,0 +1,33 @@
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+//  Created by Sam Deane on 05/12/24.
+//  All code (c) 2024 - present day, Elegant Chaos Limited.
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+// This code is copied with permission from https://github.com/elegantchaos/EmbeddedXCTest.
+// Check back there for updates.
+
+import XCTest
+
+/// Test suite which contains all the test cases for a given embedded test case class.
+///
+/// The suite runs the class setup and teardown methods before and after running the tests.
+internal class EmbeddedTestCaseSuite: XCTestSuite {
+  private let testCaseClass: XCTestCase.Type
+
+  init(for testClass: XCTestCase.Type, tests: [XCTest]) {
+    let testCaseClass = testClass
+    self.testCaseClass = testCaseClass
+    super.init(name: "\(testCaseClass) (Embedded)")
+    for test in tests {
+      addTest(test)
+    }
+  }
+
+  override func setUp() {
+    testCaseClass.setUp()
+  }
+
+  override func tearDown() {
+    testCaseClass.tearDown()
+  }
+}

Sources/SwiftGodotTestability/EmbeddedXCTest/EmbeddingController.swift

@@ -0,0 +1,120 @@
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+//  Created by Sam Deane on 04/12/2024.
+//  All code (c) 2024 - present day, Sam Deane.
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+// This code is copied with permission from https://github.com/elegantchaos/EmbeddedXCTest.
+// Check back there for updates.
+
+import Foundation
+import XCTest
+
+/// Object that observes XCTest events and collects information about
+/// the tests that are being run.
+/// During the initial run, the tests are collected into a suite, but
+/// not actually run properly.
+/// Once the normal test run is complete, we call back to the embedding
+/// host, supplying it a closure that will re-run the tests.
+/// The host can then run the tests in a different context.
+public class EmbeddingController: NSObject, XCTestObservation {
+  /// The shared instance of the observer.
+  nonisolated(unsafe) static var _instance: EmbeddingController?
+
+  /// The host that will re-run the tests.
+  let host: TestHost
+
+  /// Initialise the observer with a host.
+  init(host: TestHost) {
+    self.host = host
+  }
+
+  /// Test suite we build up by observing the normal test run.
+  /// We'll re-run this suite later, with the isRunning flag set to true,
+  let embeddedSuite = XCTestSuite(name: "Embedded Tests")
+
+  /// Failures that occurred during the embedded test run.
+  var embeddedFailures: [(XCTestCase, String, String?, Int)] = []
+
+  /// Are we currently running the embedded tests?
+  var isRunningEmbedded = false
+
+  /// Are the embedded tests currently running?
+  public static var isRunningEmbedded: Bool { _instance?.isRunningEmbedded ?? false }
+
+  /// Install hooks into the testing system.
+  ///
+  /// We add ourselves as an observer of XCTest events, so that we can
+  /// collect information about the tests that are being run.
+  ///
+  /// Once the normal test run is complete, we re-run the tests that
+  /// we've collected, with our isRunning flag set to true, so that
+  /// the test bodies are actually executed.
+  public static func setUp(hostClass: TestHost.Type) {
+    if _instance == nil {
+      // turn off buffering on stdout so that we see the output immediately
+      setbuf(__stdoutp, nil)
+
+      let observer = EmbeddingController(host: hostClass.init())
+      _instance = observer
+      XCTestObservationCenter.shared.addTestObserver(observer)
+    }
+  }
+
+  /// Record a test suite that has finished running.
+  func registerSuite(_ suite: XCTestSuite) {
+    if !isRunningEmbedded {
+      if let test = suite.tests.first as? XCTestCase {
+        let testClass = type(of: test)
+        let injected = EmbeddedTestCaseSuite(for: testClass, tests: suite.tests)
+        embeddedSuite.addTest(injected)
+      }
+    }
+  }
+
+  public func runEmbeddedTests() -> Int {
+    print(
+      """
+
+      ----------------------------------------------------------
+      Running embedded tests
+      ----------------------------------------------------------
+
+      """)
+
+    isRunningEmbedded = true
+    embeddedSuite.run()
+    isRunningEmbedded = false
+
+    print(
+      """
+
+      ----------------------------------------------------------
+
+      Finished running embedded tests with \(embeddedFailures.count) failures.
+
+      """)
+
+    return embeddedFailures.count
+  }
+
+  public func testCase(
+    _ test: XCTestCase, didFailWithDescription description: String, inFile path: String?,
+    atLine line: Int
+  ) {
+    embeddedFailures.append((test, description, path, line))
+  }
+
+  public func testSuiteDidFinish(_ testSuite: XCTestSuite) {
+    if testSuite.className == "XCTestCaseSuite" {
+      registerSuite(testSuite)
+      // } else {
+      // print("SKIPPED \(testSuite) \(type(of: testSuite))")
+    }
+  }
+
+  public func testBundleDidFinish(_ testBundle: Bundle) {
+    host.embedTests {
+      self.runEmbeddedTests()
+    }
+  }
+}

Sources/SwiftGodotTestability/EmbeddedXCTest/SilentTestRun.swift

@@ -0,0 +1,30 @@
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+//  Created by Sam Deane on 04/12/24.
+//  All code (c) 2024 - present day, Elegant Chaos Limited.
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+// This code is copied with permission from https://github.com/elegantchaos/EmbeddedXCTest.
+// Check back there for updates.
+
+import XCTest
+
+/// Test run which does nothing.
+/// We return one of these when an embedded test is run
+/// without the embedded engine running. This avoids having
+/// duplicate runs of the test appear in the output.
+open class SilentTestRun: XCTestCaseRun {
+  override init(test: XCTest) { super.init(test: XCTestCase()) }
+  open override func start() {}
+  open override func stop() {}
+  open override func record(_ issue: XCTIssue) {}
+  open override var hasBeenSkipped: Bool { true }
+  open override var hasSucceeded: Bool { false }
+  open override var skipCount: Int { 0 }
+  open override var failureCount: Int { 0 }
+  open override var executionCount: Int { 0 }
+  open override var testCaseCount: Int { 0 }
+  open override var unexpectedExceptionCount: Int { 0 }
+  open override var totalFailureCount: Int { 0 }
+  open override var totalDuration: TimeInterval { 0 }
+  open override var testDuration: TimeInterval { 0 }
+}

Sources/SwiftGodotTestability/EmbeddedXCTest/SimpleTestHost.swift

@@ -0,0 +1,21 @@
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+//  Created by Sam Deane on 04/12/24.
+//  All code (c) 2024 - present day, Elegant Chaos Limited.
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+// This code is copied with permission from https://github.com/elegantchaos/EmbeddedXCTest.
+// Check back there for updates.
+
+import Foundation
+
+/// Test embedding host that just runs the tests,
+/// and exits with a status code if there was a failure.
+public struct SimpleTestHost: TestHost {
+  public init() {}
+  public func embedTests(_ runEmbeddedTests: () -> Int) {
+    let failures = runEmbeddedTests()
+    if failures > 0 {
+      exit(Int32(failures))
+    }
+  }
+}

Sources/SwiftGodotTestability/EmbeddedXCTest/TestHost.swift

@@ -0,0 +1,20 @@
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+//  Created by Sam Deane on 04/12/24.
+//  All code (c) 2024 - present day, Elegant Chaos Limited.
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+// This code is copied with permission from https://github.com/elegantchaos/EmbeddedXCTest.
+// Check back there for updates.
+
+import Foundation
+
+/// Implement this protocol to provide a custom test host.
+public protocol TestHost {
+  /// Create the host.
+  init()
+
+  /// Perform any necessary setup, then call the supplied
+  /// closure to run the tests.
+  /// The closure returns the number of failures.
+  func embedTests(_ runner: @escaping () -> Int)
+}

Sources/SwiftGodotTestability/GodotTestCase.swift

@@ -9,23 +9,11 @@ import XCTest
 
 @testable import SwiftGodot
 
-/// Base class for all test cases that run in the Godot runtime.
-open class GodotTestCase: XCTestCase {
-    open override var testRunClass: AnyClass? {
-        // use a dummy run if the engine isn't running to avoid generating output
-        !GodotRuntime.isRunning ? DummyTestRun.self : super.testRunClass
-    }
 
-    override open func run() {
-        // We will be run twice - once in the normal XCTest runtime,
-        // and once in the Godot runtime. We only want to actually
-        // run the tests in the Godot runtime.
-        if GodotRuntime.isRunning {
-            super.run()
-        }
-    }
-
-    override open class func setUp() {
+/// Base class for all test cases that run in the Godot runtime.
+open class GodotTestCase: EmbeddedTestCase<GodotTestHost> {
+    open override class func setUp() {
+        super.setUp()
         if GodotRuntime.isRunning {
             // register any types that are needed for the tests
             for subclass in godotSubclasses {
@@ -41,6 +29,7 @@ open class GodotTestCase: XCTestCase {
                 unregister(type: subclass)
             }
         }
+        super.tearDown()
     }
 
     override open func tearDown() async throws {
@@ -76,33 +65,6 @@ open class GodotTestCase: XCTestCase {
 
 }
 
-/// Test run which does nothing.
-/// We return one of these when a Godot test is run
-/// without the test engine running. This avoids having
-/// duplicate runs of the test appear in the output.
-open class DummyTestRun: XCTestCaseRun {
-    override init(test: XCTest) {
-        super.init(test: XCTestCase())
-    }
-    open override func start() {
-    }
-    open override func stop() {
-    }
-    open override func record(_ issue: XCTIssue) {
-    }
-    open override var hasBeenSkipped: Bool { true }
-    open override var hasSucceeded: Bool { false }
-    open override var skipCount: Int { 0 }
-    open override var failureCount: Int { 0 }
-    open override var executionCount: Int { 0 }
-    open override var testCaseCount: Int { 0 }
-    open override var unexpectedExceptionCount: Int { 0 }
-    open override var totalFailureCount: Int { 0 }
-
-    open override var totalDuration: TimeInterval { 0 }
-    open override var testDuration: TimeInterval { 0 }
-}
-/// Godot testing support.
 
 public extension GodotTestCase {
 

Sources/SwiftGodotTestability/GodotTestHost.swift

@@ -0,0 +1,20 @@
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+//  Created by Sam Deane on 05/12/24.
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+import Foundation
+
+public struct GodotTestHost: TestHost {
+    public init() {}
+    public func embedTests(_ runEmbeddedTests: @escaping () -> Int) {
+        var failures = 0
+        GodotRuntime.run {
+            failures = runEmbeddedTests()
+            GodotRuntime.stop()
+        }
+
+        if failures > 0 {
+            exit(Int32(failures))
+        }
+    }
+}