fix parse error caused by certain unicode characters (#270)

Author: PhilippImhof

classes/local/lexer.php

@@ -133,19 +133,19 @@ private function read_next_token(): ?token {
             return $this->read_operator();
         }
         // There are some single-character tokens...
-        if (preg_match('/[]\[(){},;π\\\]/', $currentchar)) {
-            $types = [
-                '[' => token::OPENING_BRACKET,
-                '(' => token::OPENING_PAREN,
-                '{' => token::OPENING_BRACE,
-                ']' => token::CLOSING_BRACKET,
-                ')' => token::CLOSING_PAREN,
-                '}' => token::CLOSING_BRACE,
-                ',' => token::ARG_SEPARATOR,
-                '\\' => token::PREFIX,
-                ';' => token::END_OF_STATEMENT,
-                'π' => token::CONSTANT,
-            ];
+        $types = [
+            '[' => token::OPENING_BRACKET,
+            '(' => token::OPENING_PAREN,
+            '{' => token::OPENING_BRACE,
+            ']' => token::CLOSING_BRACKET,
+            ')' => token::CLOSING_PAREN,
+            '}' => token::CLOSING_BRACE,
+            ',' => token::ARG_SEPARATOR,
+            '\\' => token::PREFIX,
+            ';' => token::END_OF_STATEMENT,
+            'π' => token::CONSTANT,
+        ];
+        if (in_array($currentchar, array_keys($types))) {
             return $this->read_single_char_token($types[$currentchar]);
         }
         // If we are still here, that's not good at all. We need to read the char (it is only peeked

tests/lexer_test.php

@@ -829,6 +829,16 @@ public function test_read_invalid_string($expected, $input): void {
         self::assertNotNull($e);
     }
 
+    public function test_unexpected_unicode_char_in_input(): void {
+        $e = null;
+        try {
+            new lexer('I’m');
+        } catch (Exception $e) {
+            self::assertEquals("1:2:Unexpected input: '’'", $e->getMessage());
+        }
+        self::assertNotNull($e);
+    }
+
     /**
      * Test whether the read() function of the tokenizer class correctly parses special
      * cases involving numbers.