Merge pull request #1536 from hylo-lang/refactor-test-generation

Add support for parsing arbitrary test runner arguments

Sources/GenerateHyloFileTests/GenerateHyloFileTests.swift

@@ -2,8 +2,8 @@ import ArgumentParser
 import Foundation
 import Utils
 
-/// A command-line tool that generates XCTest cases for a list of annotated ".hylo"
-/// files as part of our build process.
+/// A command-line tool that generates XCTest cases for a list of annotated ".hylo" files as part
+/// of our build process.
 @main
 struct GenerateHyloFileTests: ParsableCommand {
 
@@ -23,20 +23,21 @@ struct GenerateHyloFileTests: ParsableCommand {
     transform: URL.init(fileURLWithPath:))
   var hyloSourceFiles: [URL]
 
-  /// Returns the Swift source of the test function for the Hylo file at `source`.
-  func swiftFunctionTesting(valAt source: URL) throws -> String {
+  /// Returns the Swift source of the test function for the Hylo program at `source`, which is the
+  /// URL of a single source file or the root directory of a module.
+  func swiftFunctionTesting(hyloProgramAt source: URL) throws -> String {
     let firstLine = try String(contentsOf: source).prefix { !$0.isNewline }
     let parsed = try firstLine.parsedAsFirstLineOfAnnotatedHyloFileTest()
     let testID = source.deletingPathExtension().lastPathComponent.asSwiftIdentifier
 
-    return parsed.reduce(into: "") { (o, p) in
-      o += """
+    return parsed.reduce(into: "") { (swiftCode, test) in
+      let trailing = test.arguments.reduce(into: "", { (s, a) in s.write(", \(a)") })
+      swiftCode += """
 
-        func test_\(p.methodName)_\(testID)() throws {
-          try \(p.methodName)(
+        func test_\(test.methodName)_\(testID)() throws {
+          try \(test.methodName)(
             \(String(reflecting: source.fileSystemPath)),
-            extending: programToExtend!,
-            expectingSuccess: \(p.expectSuccess))
+            extending: programToExtend!\(trailing))
         }
 
       """
@@ -55,7 +56,7 @@ struct GenerateHyloFileTests: ParsableCommand {
 
     for f in hyloSourceFiles {
       do {
-        output += try swiftFunctionTesting(valAt: f)
+        output += try swiftFunctionTesting(hyloProgramAt: f)
       } catch let e as FirstLineError {
         try! FileHandle.standardError.write(
           contentsOf: Data("\(f.fileSystemPath):1: error: \(e.details)\n".utf8))
@@ -88,50 +89,116 @@ extension StringProtocol where Self.SubSequence == Substring {
   fileprivate func parsedAsFirstLineOfAnnotatedHyloFileTest() throws -> [TestDescription] {
     var text = self[...]
     if !text.removeLeading("//- ") {
-      throw FirstLineError("first line of annotated test file must begin with “//-”.")
+      throw FirstLineError("first line of annotated test file must begin with '//-'")
     }
 
-    let methodName = text.removeFirstUntil(it: \.isWhitespace)
-    if methodName.isEmpty {
-      throw FirstLineError("missing test method name.")
+    let m = text.removeFirstUntil(it: \.isWhitespace)
+    guard let methodName = TestMethod(rawValue: String(m)) else {
+      let s = m.isEmpty ? "missing test method name" : "unknown test method '\(m)'"
+      throw FirstLineError(s)
     }
-    text.removeFirstWhile(it: \.isWhitespace)
 
-    if !text.removeLeading("expecting:") {
-      throw FirstLineError("missing “expecting:” after test method name.")
+    var arguments: [TestArgument] = []
+    while !text.isEmpty {
+      // Parse a label.
+      text.removeFirstWhile(it: \.isWhitespace)
+      let l = text.removeFirstUntil(it: { $0 == ":" })
+      if l.isEmpty {
+        break
+      } else if !text.removeLeading(":") {
+        throw FirstLineError("missing colon after argument label")
+      }
+
+      // Parse a value.
+      text.removeFirstWhile(it: \.isWhitespace)
+      let v = text.removeFirstUntil(it: \.isWhitespace)
+      if v.isEmpty { throw FirstLineError("missing value after '\(l):'") }
+
+      if let a = TestArgument(l, v) {
+        arguments.append(a)
+      } else {
+        throw FirstLineError("invalid argument '\(l): \(v)'")
+      }
     }
-    text.removeFirstWhile(it: \.isWhitespace)
 
-    let expectation = text.removeFirstUntil(it: \.isWhitespace)
-    if expectation != "success" && expectation != "failure" {
-      throw FirstLineError(
-        "illegal expectation “\(expectation)” must be “success” or “failure”."
-      )
+    var tests = [TestDescription(methodName: methodName, arguments: arguments)]
+    if methodName == .compileAndRun {
+      tests.append(TestDescription(methodName: .compileAndRunOptimized, arguments: arguments))
     }
-    let expectSuccess = expectation == "success"
+    return tests
+  }
+
+}
+
+/// The name of a method implementing the logic of a test runner.
+fileprivate enum TestMethod: String {
+
+  /// Compiles and runs the program.
+  case compileAndRun
+
+  /// Compiles and runs the program with optimizations.
+  case compileAndRunOptimized
+
+  /// Compiles the program down to LLVM IR.
+  case compileToLLVM
+
+  /// Compiles the program down to Hylo IR.
+  case lowerToFinishedIR
+
+  /// Parses the program.
+  case parse
 
-    if !text.drop(while: \.isWhitespace).isEmpty {
-      throw FirstLineError("illegal trailing text “\(text)”.")
+  /// Type checks the program.
+  case typeCheck
+
+}
+
+/// An argument of a test runner.
+fileprivate struct TestArgument: CustomStringConvertible {
+
+  /// The label of an argument.
+  enum Label: String {
+
+    /// The label of an argument specifying the expected outcome of the test.
+    case expecting
+
+  }
+
+  /// The label of the argument.
+  let label: Label
+
+  /// The value of the argument.
+  let value: String
+
+  /// Creates an instance with the given properties or returns `nil` if the argument is invalid.
+  init?(_ label: Substring, _ value: Substring) {
+    // Validate the label.
+    guard let l = Label(rawValue: String(label)) else {
+      return nil
     }
 
-    var tests = [TestDescription(methodName: methodName, expectSuccess: expectSuccess)]
-    if methodName == "compileAndRun" {
-      tests.append(
-        .init(methodName: "compileAndRunWithOptimizations", expectSuccess: expectSuccess))
+    // Validate the value.
+    switch l {
+    case .expecting:
+      if (value != ".success") && (value != ".failure") { return nil }
     }
-    return tests
+
+    self.label = l
+    self.value = String(value)
   }
 
+  var description: String { "\(label): \(value)" }
+
 }
 
 /// Information necessary to generate a test case.
 fileprivate struct TestDescription {
 
   /// The name of the method implementing the logic of the test runner.
-  let methodName: Substring
+  let methodName: TestMethod
 
-  /// `true` iff the invoked compilation is expected to succeed.
-  let expectSuccess: Bool
+  /// The arguments of the method.
+  let arguments: [TestArgument]
 
 }
 

Sources/TestUtils/AnnotatedHyloFileTest.swift

@@ -30,6 +30,13 @@ extension Result {
 
 }
 
+/// The expected outcome of a test.
+public enum ExpectedTestOutcome {
+
+  case success, failure
+
+}
+
 extension XCTestCase {
 
   /// The effects of running the `processAndCheck` parameter to `checkAnnotatedHyloFiles`.
@@ -102,7 +109,7 @@ extension XCTestCase {
   @nonobjc
   public func checkAnnotatedHyloFileDiagnostics(
     inFileAt hyloFilePath: String,
-    expectingSuccess expectSuccess: Bool,
+    expecting expectation: ExpectedTestOutcome,
     _ process: (_ file: SourceFile, _ diagnostics: inout DiagnosticSet) throws -> Void
   ) throws {
     let f = try SourceFile(at: hyloFilePath)
@@ -115,7 +122,7 @@ extension XCTestCase {
       return []
     }
 
-    if (thrownError == nil) != expectSuccess {
+    if (thrownError == nil) != (expectation == .success) {
       record(XCTIssue(unexpectedOutcomeDiagnostic(thrownError: thrownError, at: f.wholeRange)))
     }
   }
@@ -167,19 +174,19 @@ extension XCTestCase {
   /// Calls `compileAndRun` with optimizations disabled.
   @nonobjc
   public func compileAndRun(
-    _ hyloFilePath: String, extending p: TypedProgram, expectingSuccess expectSuccess: Bool
+    _ hyloFilePath: String, extending p: TypedProgram, expecting expectation: ExpectedTestOutcome
   ) throws {
     try compileAndRun(
-      hyloFilePath, withOptimizations: false, extending: p, expectingSuccess: expectSuccess)
+      hyloFilePath, withOptimizations: false, extending: p, expecting: expectation)
   }
 
   /// Calls `compileAndRun` with optimizations enabled.
   @nonobjc
-  public func compileAndRunWithOptimizations(
-    _ hyloFilePath: String, extending p: TypedProgram, expectingSuccess expectSuccess: Bool
+  public func compileAndRunOptimized(
+    _ hyloFilePath: String, extending p: TypedProgram, expecting expectation: ExpectedTestOutcome
   ) throws {
     try compileAndRun(
-      hyloFilePath, withOptimizations: true, extending: p, expectingSuccess: expectSuccess)
+      hyloFilePath, withOptimizations: true, extending: p, expecting: expectation)
   }
 
   /// Compiles and runs the hylo file at `hyloFilePath`, applying program optimizations iff
@@ -188,11 +195,11 @@ extension XCTestCase {
   @nonobjc
   public func compileAndRun(
     _ hyloFilePath: String, withOptimizations: Bool, extending p: TypedProgram,
-    expectingSuccess expectSuccess: Bool
+    expecting expectation: ExpectedTestOutcome
   ) throws {
     if swiftyLLVMMandatoryPassesCrash { return }
     try checkAnnotatedHyloFileDiagnostics(
-      inFileAt: hyloFilePath, expectingSuccess: expectSuccess
+      inFileAt: hyloFilePath, expecting: expectation
     ) { (hyloSource, log) in
       try compileAndRun(
         hyloSource, withOptimizations: withOptimizations, extending: p,
@@ -221,11 +228,11 @@ extension XCTestCase {
   /// diagnostics and exit codes match annotated expectations.
   @nonobjc
   public func compileToLLVM(
-    _ hyloFilePath: String, extending p: TypedProgram, expectingSuccess expectSuccess: Bool
+    _ hyloFilePath: String, extending p: TypedProgram, expecting expectation: ExpectedTestOutcome
   ) throws {
     if swiftyLLVMMandatoryPassesCrash { return }
     try checkAnnotatedHyloFileDiagnostics(
-      inFileAt: hyloFilePath, expectingSuccess: expectSuccess
+      inFileAt: hyloFilePath, expecting: expectation
     ) { (hyloSource, log) in
       try XCTestCase.capturingThrownInstances(into: &log) {
         _ = try compile(hyloSource.url, with: ["--emit", "llvm"])

Sources/TestUtils/LoweringTests.swift

@@ -10,11 +10,11 @@ extension XCTestCase {
   /// that diagnostics and thrown errors match annotated expectations.
   @nonobjc
   public func lowerToFinishedIR(
-    _ hyloFilePath: String, extending p: TypedProgram, expectingSuccess expectSuccess: Bool
+    _ hyloFilePath: String, extending p: TypedProgram, expecting expectation: ExpectedTestOutcome
   ) throws {
 
     try checkAnnotatedHyloFileDiagnostics(
-      inFileAt: hyloFilePath, expectingSuccess: expectSuccess
+      inFileAt: hyloFilePath, expecting: expectation
     ) { (hyloSource, log) in
 
       let (p, m) = try p.loadModule(reportingDiagnosticsTo: &log) { (ast, log, space) in

Sources/TestUtils/ParseTest.swift

@@ -7,11 +7,11 @@ extension XCTestCase {
   /// errors match annotated expectations.
   @nonobjc
   public func parse(
-    _ hyloFilePath: String, extending p: TypedProgram, expectingSuccess expectSuccess: Bool
+    _ hyloFilePath: String, extending p: TypedProgram, expecting expectation: ExpectedTestOutcome
   ) throws {
 
     try checkAnnotatedHyloFileDiagnostics(
-      inFileAt: hyloFilePath, expectingSuccess: expectSuccess
+      inFileAt: hyloFilePath, expecting: expectation
     ) { (hyloSource, log) in
       var ast = AST()
       _ = try ast.loadModule(

Sources/TestUtils/TypeCheckerTests.swift

@@ -9,11 +9,11 @@ extension XCTestCase {
   /// errors match annotated expectations.
   @nonobjc
   public func typeCheck(
-    _ hyloFilePath: String, extending p: TypedProgram, expectingSuccess expectSuccess: Bool
+    _ hyloFilePath: String, extending p: TypedProgram, expecting expectation: ExpectedTestOutcome
   ) throws {
 
     try checkAnnotatedHyloFileDiagnostics(
-      inFileAt: hyloFilePath, expectingSuccess: expectSuccess
+      inFileAt: hyloFilePath, expecting: expectation
     ) { (hyloSource, log) in
       _ = try p.loadModule(reportingDiagnosticsTo: &log) { (ast, log, space) in
         try ast.loadModule(

Tests/EndToEndTests/TestCases/AddressOf.hylo

@@ -1,4 +1,4 @@
-//- compileAndRun expecting: success
+//- compileAndRun expecting: .success
 
 public fun main() {
   let x = 42

Tests/EndToEndTests/TestCases/Autoclosure.hylo

@@ -1,4 +1,4 @@
-//- compileAndRun expecting: success
+//- compileAndRun expecting: .success
 
 let counter: Int = 0
 

Tests/EndToEndTests/TestCases/BoundParameter.hylo

@@ -1,4 +1,4 @@
-//- compileAndRun expecting: success
+//- compileAndRun expecting: .success
 
 trait P { fun foo() -> Int }
 

Tests/EndToEndTests/TestCases/Break.hylo

@@ -1,4 +1,4 @@
-//- compileAndRun expecting: success
+//- compileAndRun expecting: .success
 
 public fun main() {
   // Break from a do-while loop.

Tests/EndToEndTests/TestCases/BufferInitialization.hylo

@@ -1,4 +1,4 @@
-//- compileAndRun expecting: success
+//- compileAndRun expecting: .success
 
 fun use(_ x: Bool[2]) {}
 

Tests/EndToEndTests/TestCases/Concurrency/concurrent_greeting.hylo

@@ -1,4 +1,4 @@
-//- compileToLLVM expecting: success
+//- compileToLLVM expecting: .success
 
 fun do_greet() -> Int {
   print("Hello, concurrent world!")

Tests/EndToEndTests/TestCases/Concurrency/concurrent_sort.hylo

@@ -1,4 +1,4 @@
-//- compileAndRun expecting: success
+//- compileAndRun expecting: .success
 
 let size_threshold = 16
 

Tests/EndToEndTests/TestCases/Concurrency/escaping_spawn.hylo

@@ -1,4 +1,4 @@
-//- compileToLLVM expecting: success
+//- compileToLLVM expecting: .success
 
 // TODO: type-erasure for the future type
 fun do_spawn() -> EscapingFuture<{}> {

Tests/EndToEndTests/TestCases/Concurrency/mutating_spawn.hylo

@@ -1,4 +1,4 @@
-//- compileToLLVM expecting: success
+//- compileToLLVM expecting: .success
 
 
 fun test_mutating_spawn() -> Int {

Tests/EndToEndTests/TestCases/ConditionalCompilation.hylo

@@ -1,4 +1,4 @@
-//- compileAndRun expecting: success
+//- compileAndRun expecting: .success
 
 fun use<T>(_ x: T) {}
 

Tests/EndToEndTests/TestCases/CustomMove.hylo

@@ -1,4 +1,4 @@
-//- compileAndRun expecting: success
+//- compileAndRun expecting: .success
 
 type A: Deinitializable {
   public var witness: Int

Tests/EndToEndTests/TestCases/DefaultImplementation.hylo

@@ -1,4 +1,4 @@
-//- compileAndRun expecting: success
+//- compileAndRun expecting: .success
 
 public trait Eq {
   fun infix== (_ other: Self) -> Bool

Tests/EndToEndTests/TestCases/Destroy.hylo

@@ -1,4 +1,4 @@
-//- compileAndRun expecting: success
+//- compileAndRun expecting: .success
 
 public fun main() {
   _ = 42