Implement newline and spaces matchers

Notes.txt

@@ -8,16 +8,16 @@ DONE cleanup repositories
     DONE implement a Rejection Reason
 
 [ Lexer ]
-    TODO support pre-made matchers
-        whitespaces
-        newlines
-        DONE "AnyOperator" should be renamed to something less misleading
-        DONE change "any" to "anything"
     TODO return a result accepting or rejecting the tokenized string, consider cases:
         Unexpected tokens
         No remaining tokens
         Trailing tokens
     TODO matching is done for every rule, for every token, it could probably be done more efficiently with a DFA
+    DONE support pre-made matchers
+        DONE whitespaces
+        DONE newlines
+        DONE "AnyOperator" should be renamed to something less misleading
+        DONE change "any" to "anything"
     DONE test rule order
     DONE bug: lexer is not taking the longest match
     DONE add a reference to the corresponding line text to each token

build.gradle.kts

@@ -1,5 +1,3 @@
-import org.jetbrains.kotlin.gradle.plugin.KotlinTargetHierarchy.SourceSetTree.Companion.test
-
 plugins {
     kotlin("jvm") version "1.9.22"
     id("maven-publish")
@@ -27,7 +25,7 @@ tasks.named<Test>("test") {
 }
 
 kotlin {
-    jvmToolchain(8)
+    jvmToolchain(19)
 }
 
 publishing {

src/main/kotlin/io/vexel/kobold/lexer/LexerDSL.kt

@@ -22,4 +22,6 @@ class LexerDSL(
     NotOperatorDSL by NotOperator(),
     OptionalOperatorDSL by OptionalOperator() {
     val anything = Anything()
+    val newline = "\n" or "\r\n"
+    val spaces = " " or "\t"
 }

src/test/kotlin/io/vexel/kobold/test/lexer/Lexer should.kt

@@ -6,6 +6,7 @@ import io.vexel.kobold.test.tokens.Number
 import org.junit.jupiter.api.Test
 import java.util.*
 import kotlin.test.assertEquals
+import kotlin.test.assertNotEquals
 
 class `Lexer should` {
     @Test
@@ -33,7 +34,7 @@ class `Lexer should` {
     fun `properly process line and column numbers` () {
         val lexer = lexer {
             "@".oneOrMore() with { Ats(it) }
-            ignore(" " or "\n")
+            ignore(newline or spaces)
         }
 
         val text = """
@@ -76,6 +77,36 @@ class `Lexer should` {
         assertEquals(expected, tokens)
     }
 
+    @Test
+    fun `match newlines from both conventions`() {
+        val text = "@@@\r\n@@@\n@@@".trimIndent()
+
+        val lexer = lexer {
+            "@" with { Ats(it) }
+            ignore(newline)
+        }
+
+        val tokens = lexer.tokenize(text)
+
+        assert(tokens.all { it.text == "@" })
+        assertEquals(9, tokens.count())
+    }
+
+    @Test
+    fun `match spaces and tabs`() {
+        val text = "@\t   \t@ @\t@".trimIndent()
+
+        val lexer = lexer {
+            "@" with { Ats(it) }
+            ignore(spaces)
+        }
+
+        val tokens = lexer.tokenize(text)
+
+        assert(tokens.all { it.text == "@" })
+        assertEquals(4, tokens.count())
+    }
+
     class Ats(
         text: String,
         lineText: String = "",