Rewrite lexer, parser, and evaluator

This rewrites the lexer, parser, and evaluator to have *much* simpler
code, as well as better error reporting and other things generally considered
nice in modern programming language implementations.

Sources/LeafKit/LeafAST.swift

@@ -10,91 +10,15 @@ public struct LeafAST: Hashable {
     // MARK: - Internal/Private Only
     let name: String
 
-    init(name: String, ast: [Syntax]) {
+    init(name: String, ast: [Statement]) {
         self.name = name
         self.ast = ast
         self.rawAST = nil
-        self.flat = false
-
-        updateRefs([:])
     }
-
-    init(from: LeafAST, referencing externals: [String: LeafAST]) {
-        self.name = from.name
-        self.ast = from.ast
-        self.rawAST = from.rawAST
-        self.externalRefs = from.externalRefs
-        self.unresolvedRefs = from.unresolvedRefs
-        self.flat = from.flat
 
-        updateRefs(externals)
-    }
+    internal private(set) var ast: [Statement]
 
-    internal private(set) var ast: [Syntax]
-    internal private(set) var externalRefs = Set<String>()
-    internal private(set) var unresolvedRefs = Set<String>()
-    internal private(set) var flat: Bool
-
     // MARK: - Private Only
 
-    private var rawAST: [Syntax]?
-
-    mutating private func updateRefs(_ externals: [String: LeafAST]) {
-        var firstRun = false
-        if rawAST == nil, flat == false { rawAST = ast; firstRun = true }
-        unresolvedRefs.removeAll()
-        var pos = ast.startIndex
-
-        // inline provided externals
-        while pos < ast.endIndex {
-            // get desired externals for this Syntax - if none, continue
-            let wantedExts = ast[pos].externals()
-            if wantedExts.isEmpty {
-                pos = ast.index(after: pos)
-                continue
-            }
-            // see if we can provide any of them - if not, continue
-            let providedExts = externals.filter { wantedExts.contains($0.key) }
-            if providedExts.isEmpty {
-                unresolvedRefs.formUnion(wantedExts)
-                pos = ast.index(after: pos)
-                continue
-            }
-
-            // replace the original Syntax with the results of inlining, potentially 1...n
-            let replacementSyntax: [Syntax]
-            if case .extend(let extend) = ast[pos], let context = extend.context {
-                let inner = ast[pos].inlineRefs(providedExts, [:])
-                replacementSyntax = [.with(.init(context: context, body: inner))]
-            } else {
-                replacementSyntax = ast[pos].inlineRefs(providedExts, [:])
-            }
-            ast.replaceSubrange(pos...pos, with: replacementSyntax)
-            // any returned new inlined syntaxes can't be further resolved at this point
-            // but we need to add their unresolvable references to the global set
-            var offset = replacementSyntax.startIndex
-            while offset < replacementSyntax.endIndex {
-                unresolvedRefs.formUnion(ast[pos].externals())
-                offset = replacementSyntax.index(after: offset)
-                pos = ast.index(after: pos)
-            }
-        }
-
-        // compress raws
-        pos = ast.startIndex
-        while pos < ast.index(before: ast.endIndex) {
-            if case .raw(var syntax) = ast[pos] {
-                if case .raw(var add) = ast[ast.index(after: pos)] {
-                    var buffer = ByteBufferAllocator().buffer(capacity: syntax.readableBytes + add.readableBytes)
-                    buffer.writeBuffer(&syntax)
-                    buffer.writeBuffer(&add)
-                    ast[pos] = .raw(buffer)
-                    ast.remove(at: ast.index(after: pos) )
-                } else { pos = ast.index(after: pos) }
-            } else { pos = ast.index(after: pos) }
-        }
-
-        flat = unresolvedRefs.isEmpty ? true : false
-        if firstRun && flat { rawAST = nil }
-    }
+    private var rawAST: [Statement]?
 }

Sources/LeafKit/LeafData/LeafData.swift

@@ -259,6 +259,9 @@ public struct LeafData: CustomStringConvertible,
     /// Try to convert one concrete object to a second type.
     internal func convert(to output: NaturalType, _ level: DataConvertible = .castable) -> LeafData {
         guard celf != output else  { return self }
+        if self.isNil && output == .bool {
+            return .bool(false)
+        }
         if case .lazy(let f,_,_) = self.storage { return f().convert(to: output, level) }
         guard let input = storage.unwrap,
               let conversion = _ConverterMap.symbols.get(input.concreteType!, output),
@@ -373,7 +376,7 @@ fileprivate enum _ConverterMap {
         }),
                                         //  String == "true" || "false"
         Converter(.string  , .bool         , is: .castable, via: {
-            ($0 as? String).map { Bool($0) }?.map { .bool($0) } ?? .trueNil
+            ($0 as? String).map { Bool($0) }?.map { .bool($0) } ?? (($0 as? String).map { .bool(!$0.isEmpty) } ?? .trueNil)
         }),
                                         // True = 1; False = 0
         Converter(.bool    , .double       , is: .castable, via: {
@@ -415,9 +418,9 @@ fileprivate enum _ConverterMap {
 
         // MARK: - .coercible (One-direction defined conversion)
 
-                                          // Array.isEmpty == truthiness
+                                          // !Array.isEmpty == truthiness
         Converter(.array      , .bool       , is: .coercible, via: {
-            ($0 as? [LeafData]).map { $0.isEmpty }.map { .bool($0) } ?? .trueNil
+            ($0 as? [LeafData]).map { .bool(!$0.isEmpty) } ?? .trueNil
         }),
                                           // Data.isEmpty == truthiness
         Converter(.data       , .bool       , is: .coercible, via: {

Sources/LeafKit/LeafData/LeafDataRepresentable.swift

@@ -11,6 +11,9 @@ public protocol LeafDataRepresentable {
 extension String: LeafDataRepresentable {
     public var leafData: LeafData { .string(self) }
 }
+extension Substring: LeafDataRepresentable {
+    public var leafData: LeafData { .string(String(self)) }
+}
 
 extension FixedWidthInteger {
     public var leafData: LeafData {

Sources/LeafKit/LeafData/LeafDataStorage.swift

@@ -146,7 +146,7 @@ internal indirect enum LeafDataStorage: Equatable, CustomStringConvertible {
                  .dictionary(_) : data = try serialize()!.data(using: encoding)
             case .data(let d)   : data = d
         }
-        guard let validData = data else { throw "Serialization Error" }
+        guard let validData = data else { throw LeafError(.serializationError) }
         buffer.writeBytes(validData)
     }
 

Sources/LeafKit/LeafError.swift

@@ -5,7 +5,7 @@
 ///
 public struct LeafError: Error {
     /// Possible cases of a LeafError.Reason, with applicable stored values where useful for the type
-    public enum Reason {
+    public enum Reason: Equatable {
         // MARK: Errors related to loading raw templates
         /// Attempted to access a template blocked for security reasons
         case illegalAccess(String)
@@ -34,13 +34,43 @@ public struct LeafError: Error {
 
         // MARK: Wrapped Errors related to Lexing or Parsing
         /// Errors due to malformed template syntax or grammar
-        case lexerError(LexerError)
+        case lexerError(LeafScannerError)
 
         // MARK: Errors lacking specificity
         /// Errors from protocol adherents that do not support newer features
         case unsupportedFeature(String)
         /// Errors only when no existing error reason is adequately clear
         case unknownError(String)
+
+        /// Errors when something goes wrong internally
+        case internalError(what: String)
+
+        /// Errors when an import is not found
+        case importNotFound(name: String)
+
+        /// Errors when a tag is not found
+        case tagNotFound(name: String)
+
+        /// Errors when one type was expected, but another was obtained
+        case typeError(shouldHaveBeen: LeafData.NaturalType, got: LeafData.NaturalType)
+
+        /// A typeError specialised for Double | Int
+        case expectedNumeric(got: LeafData.NaturalType)
+
+        /// A typeError specialised for binary operators of (T, T) -> T
+        case badOperation(on: LeafData.NaturalType, what: String)
+
+        /// Errors when a tag receives a bad parameter count
+        case badParameterCount(tag: String, expected: Int, got: Int)
+
+        /// Errors when a tag receives a body, but doesn't want one
+        case extraneousBody(tag: String)
+
+        /// Errors when a tag doesn't receive a body, but wants one
+        case missingBody(tag: String)
+
+        /// Serialization error
+        case serializationError
     }
 
     /// Source file name causing error
@@ -84,7 +114,27 @@ public struct LeafError: Error {
                 return "\(src) - \(key) cyclically referenced in [\(chain.joined(separator: " -> "))]"
             case .lexerError(let e):
                 return "Lexing error - \(e.localizedDescription)"
-        }
+            case .importNotFound(let name):
+                return "Import \(name) was not found"
+            case .internalError(let what):
+                return "Something in Leaf broke: \(what)\nPlease report this to https://github.com/vapor/leaf-kit"
+            case .tagNotFound(let name):
+                return "Tag \(name) was not found"
+            case .typeError(let shouldHaveBeen, let got):
+                return "Type error: I was expecting \(shouldHaveBeen), but I got \(got) instead"
+            case .badOperation(let on, let what):
+                return "Type error: \(on) cannot do \(what)"
+            case .expectedNumeric(let got):
+                return "Type error: I was expecting a numeric type, but I got \(got) instead"
+            case .badParameterCount(let tag, let expected, let got):
+                return "Type error: \(tag) was expecting \(expected) parameters, but got \(got) parameters instead"
+            case .extraneousBody(let tag):
+                return "Type error: \(tag) wasn't expecting a body, but got one"
+            case .missingBody(let tag):
+                return "Type error: \(tag) was expecting a body, but didn't get one"
+            case .serializationError:
+                return "Serialization error"
+            }
     }
 
     /// Create a `LeafError` - only `reason` typically used as source locations are auto-grabbed
@@ -102,63 +152,3 @@ public struct LeafError: Error {
         self.reason = reason
     }
 }
-
-// MARK: - `LexerError` Summary (Wrapped by LeafError)
-
-/// `LexerError` reports errors during the stage.
-public struct LexerError: Error {
-    // MARK: - Public
-
-    public enum Reason {
-        // MARK: Errors occuring during Lexing
-        /// A character not usable in parameters is present when Lexer is not expecting it
-        case invalidParameterToken(Character)
-        /// A string was opened but never terminated by end of file
-        case unterminatedStringLiteral
-        /// Use in place of fatalError to indicate extreme issue
-        case unknownError(String)
-    }
-
-    /// Template source file line where error occured
-    public let line: Int
-    /// Template source column where error occured
-    public let column: Int
-    /// Name of template error occured in
-    public let name: String
-    /// Stated reason for error
-    public let reason: Reason
-
-    // MARK: - Internal Only
-
-    /// State of tokens already processed by Lexer prior to error
-    internal let lexed: [LeafToken]
-    /// Flag to true if lexing error is something that may be recoverable during parsing;
-    /// EG, `"#anhtmlanchor"` may lex as a tag name but fail to tokenize to tag because it isn't
-    /// followed by a left paren. Parser may be able to recover by decaying it to `.raw`.
-    internal let recoverable: Bool
-
-    /// Create a `LexerError`
-    /// - Parameters:
-    ///   - reason: The specific reason for the error
-    ///   - src: File being lexed
-    ///   - lexed: `LeafTokens` already lexed prior to error
-    ///   - recoverable: Flag to say whether the error can potentially be recovered during Parse
-    internal init(
-        _ reason: Reason,
-        src: LeafRawTemplate,
-        lexed: [LeafToken] = [],
-        recoverable: Bool = false
-    ) {
-        self.line = src.line
-        self.column = src.column
-        self.reason = reason
-        self.lexed = lexed
-        self.name = src.name
-        self.recoverable = recoverable
-    }
-
-    /// Convenience description of source file name, error reason, and location in file of error source
-    var localizedDescription: String {
-        return "\"\(name)\": \(reason) - \(line):\(column)"
-    }
-}