Fail the GH build/test steps if the swift build/test actually failed. (…

…migueldeicaza#598)

Currently, if the `swift build` or `swift test` commands fail, the GH action step will still be marked as succeeded.

This seems to be an artefact of the way we were re-running the tests from within the Godot engine; it resulted in the overall failure count being 0 even if there were actual failures.

This PR changes the way we run the tests, and should result in a non-zero exit status if a test does fail.

It also separates the XCTest run and the Swift testing run into two steps.

The default behaviour of `swift test` is to first run the XCTest engine, then run Swift testing engine. 

This results in the following confusing output at the end of the test run:

```
􀟈  Test run started.
􀄵  Testing Library Version: 102 (arm64e-apple-macos13.0)
􁁛  Test run with 0 tests passed after 0.001 seconds.
```

This is the output from the Swift testing engine. Because all our tests are using XCTest, and we have no Swift testing tests, it is saying that 0 tests passed. As it's the last thing in the report, it's a bit confusing and for a while I thought that this was why the exit status was 0.

To avoid this confusion, we can explicitly choose a test ending by passing `--disable-swift-testing` or `--disable-xctest`.

I have done this in two separate steps, so that the output from each testing engine is clearer. 

The Swift testing engine step is a null-op right now, so we could remove it, but at some point we might want to migrate some tests to the new testing framework, so I figured it was worth having it there to avoid later confusion.

.github/workflows/swift.yml

@@ -24,11 +24,13 @@ jobs:
     - name: Set up Xcode
       uses: maxim-lobanov/setup-xcode@v1
       with:
-        xcode-version: "15.3"
+        xcode-version: "16.0"
     - name: Build
-      run: swift build -v
-    - name: Run tests
-      run: swift test -v
+      run: swift build --build-tests --quiet
+    - name: Run tests (XCTest)
+      run: swift test --skip-build --no-parallel --disable-swift-testing
+    - name: Run tests (Swift testing)
+      run: swift test --skip-build --no-parallel --disable-xctest
 
   build-windows:
     runs-on: windows-latest

Sources/SwiftGodotTestability/GodotTestCase.swift

@@ -6,94 +6,73 @@
 //
 
 import XCTest
+
 @testable import SwiftGodot
 
+/// Base class for all test cases that run in the Godot runtime.
 open class GodotTestCase: XCTestCase {
-
-    private static var testSuites: [XCTestSuite] = []
-
-    override open class var defaultTestSuite: XCTestSuite {
-        let testSuite = super.defaultTestSuite
-        testSuites.append (testSuite)
-        return testSuite
-    }
-
-    override open func run () {
+    override open func run() {
+        // We will be run twice - once in the normal XCTest runtime,
+        // and once in the Godot runtime. We only want to actually
+        // run the tests in the Godot runtime.
         if GodotRuntime.isRunning {
-            super.run ()
-        } else {
-            guard !GodotRuntime.isInitialized else { return }
-            GodotRuntime.run {
-                if !Self.testSuites.isEmpty {
-                    // Executing all test suites from the context
-                    for testSuite in Self.testSuites {
-                        testSuite.perform (XCTestSuiteRun (test: testSuite))
-                    }
-                } else {
-                    Self.godotSetUp ()
-                    // Executing single test method
-                    super.run ()
-                    Self.godotTearDown ()
-                }
-
-                GodotRuntime.stop ()
-            }
-        }
-    }
-
-    open class var godotSubclasses: [Wrapped.Type] {
-        return []
-    }
-
-    open class func godotSetUp () {
-        for subclass in godotSubclasses {
-            register (type: subclass)
+            super.run()
         }
     }
-
-    open class func godotTearDown () {
-        for subclass in godotSubclasses {
-            unregister (type: subclass)
-        }
-    }
-
-    override open class func setUp () {
+
+    override open class func setUp() {
         if GodotRuntime.isRunning {
-            godotSetUp ()
+            // register any types that are needed for the tests
+            for subclass in godotSubclasses {
+                register(type: subclass)
+            }
         }
     }
-    
-    override open class func tearDown () {
+
+    override open class func tearDown() {
         if GodotRuntime.isRunning {
-            godotTearDown ()
+            // unregister any types that were registered for the tests
+            for subclass in godotSubclasses {
+                unregister(type: subclass)
+            }
         }
     }
-
-    override open func tearDown () async throws {
-        // Cleaning up test objects
-        let liveObjects: [Wrapped] = Array (liveFrameworkObjects.values) + Array (liveSubtypedObjects.values)
-        for liveObject in liveObjects {
-            switch liveObject {
-            case let node as Node:
-                node.queueFree ()
-            case let refCounted as RefCounted:
-                refCounted._exp_unref ()
-            case let object as Object:
-                _ = object.call (method: "free")
-            default:
-                print ("Unable to free \(liveObject)")
+
+    override open func tearDown() async throws {
+        if GodotRuntime.isRunning {
+            // clean up test objects
+            let liveObjects: [Wrapped] = Array(liveFrameworkObjects.values) + Array(liveSubtypedObjects.values)
+            for liveObject in liveObjects {
+                switch liveObject {
+                case let node as Node:
+                    node.queueFree()
+                case let refCounted as RefCounted:
+                    refCounted._exp_unref()
+                case let object as Object:
+                    _ = object.call(method: "free")
+                default:
+                    print("Unable to free \(liveObject)")
+                }
             }
+            liveFrameworkObjects.removeAll()
+            liveSubtypedObjects.removeAll()
+
+            // waiting for queueFree to take effect
+            let scene = try GodotRuntime.getScene()
+            await scene.processFrame.emitted
         }
-        liveFrameworkObjects.removeAll ()
-        liveSubtypedObjects.removeAll ()
-
-        // Waiting for queueFree to take effect
-        let scene = try GodotRuntime.getScene ()
-        await scene.processFrame.emitted
     }
-
+
+    /// List of types that need to be registered in the Godot runtime.
+    /// Subclasses should override this to return the types they need.
+    open class var godotSubclasses: [Wrapped.Type] {
+        return []
+    }
+
 }
 
+/// Godot testing support.
+
 public extension GodotTestCase {
 
     /// Asserts approximate equality of two floating point values based on `Math::is_equal_approx` implementation in Godot

Sources/SwiftGodotTestability/GodotTestRunner.swift

@@ -0,0 +1,47 @@
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+//  Created by Sam Deane on 31/10/24.
+// -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
+import XCTest
+
+@testable import SwiftGodot
+
+/// Test case which runs all the other tests from within the Godot runtime.
+/// It doesn't actually matter when this suite is run, but we name it with
+/// __ to try to make it run first, since `swift test` seems to run test
+/// suites in alphabetical order.
+class __GodotTestRunner: XCTestCase {
+    /// Failure count from the tests run in Godot
+    static var failureCount = 0
+
+    /// By the time this test runs, all the other tests have already run
+    /// in the Godot runtime. We can check the failure count here to see
+    /// if any tests failed.
+    func testRunEverythingInGodot() {
+        XCTAssert(Self.failureCount == 0, "Some tests failed when running in Godot")
+    }
+
+    /// Set up the Godot runtime and run all the tests in it.
+    override func run() {
+        // this call will be re-entered inside the Godot runtime, so we
+        // need to check if the runtime is already initialized.
+        if !GodotRuntime.isInitialized {
+            GodotRuntime.run {
+                /// make a copy of all the tests and run them in Godot
+                let allTests = XCTestSuite.default
+                let suite = XCTestSuite(name: "All Tests In Godot")
+                for test in allTests.tests {
+                    suite.addTest(test)
+                }
+                suite.run()
+
+                // record the failure count
+                Self.failureCount = suite.testRun!.totalFailureCount
+
+                // shut down the Godot runtime
+                GodotRuntime.stop()
+            }
+            super.run()
+        }
+    }
+}