temporary error catching for escaping single quote and unicode; tag 0…

….1.14

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "cirru_parser"
-version = "0.1.13"
+version = "0.1.14"
 authors = ["jiyinyiyong <[email protected]>"]
 edition = "2018"
 license = "MIT"
@@ -21,7 +21,7 @@ exclude = [
 [dependencies]
 
 [dev-dependencies]
-serde_json = "1.0.68"
+serde_json = "1.0.69"
 criterion = "0.3"
 
 

benches/parsing.rs

@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use criterion::{criterion_group, criterion_main, Criterion};
 use std::fs;
 
 use cirru_parser::parse;

examples/demo.rs

@@ -18,6 +18,8 @@ defn fib (n)
   );
 
   let large_demo = "/Users/chen/repo/calcit-lang/editor/compact.cirru";
+  // let large_demo = "/Users/chen/repo/calcit-lang/respo-calcit-workflow/js-out/program-ir.cirru";
+  // let large_demo = "/Users/chen/repo/calcit-lang/calcit_runner.rs/js-out/program-ir.cirru";
   let content = fs::read_to_string(large_demo).unwrap();
 
   match parse(&content) {

examples/escape.rs

@@ -0,0 +1,6 @@
+use cirru_parser::{lex, parse, Cirru};
+
+pub fn main() {
+  println!("{:?}", parse("a \"b\\u{87DF}\""));
+  println!("{:?}", parse("a \"b\\u{87DF}\" c d e f g f"));
+}

src/parser.rs

@@ -111,7 +111,7 @@ pub fn lex(initial_code: &str) -> Result<CirruLexItemList, String> {
   let mut buffer = String::from("");
   let code = initial_code;
 
-  for c in code.chars() {
+  for (idx, c) in code.chars().enumerate() {
     match state {
       CirruLexState::Space => match c {
         ' ' => {
@@ -196,6 +196,10 @@ pub fn lex(initial_code: &str) -> Result<CirruLexItemList, String> {
           state = CirruLexState::Str;
           buffer.push('"');
         }
+        '\'' => {
+          state = CirruLexState::Str;
+          buffer.push('\'');
+        }
         't' => {
           state = CirruLexState::Str;
           buffer.push('\t');
@@ -204,11 +208,29 @@ pub fn lex(initial_code: &str) -> Result<CirruLexItemList, String> {
           state = CirruLexState::Str;
           buffer.push('\n');
         }
+        'u' => {
+          // not supporting, but don't panic
+          let end = idx + 10;
+          let peek = if end >= code.len() {
+            &code[idx..]
+          } else {
+            &code[idx..end]
+          };
+          println!("Unicode escaping is not supported yet: {:?} ...", peek);
+          buffer.push('\\');
+          buffer.push('u');
+          state = CirruLexState::Str;
+        }
         '\\' => {
           state = CirruLexState::Str;
           buffer.push('\\');
         }
-        _ => return Err(String::from("unexpected character during string escaping")),
+        _ => {
+          return Err(format!(
+            "unexpected character during string escaping: {:?}",
+            c
+          ))
+        }
       },
       CirruLexState::Indent => match c {
         ' ' => {

tests/lexer_test.rs

@@ -103,3 +103,23 @@ fn lex_strings() -> Result<(), String> {
 
   Ok(())
 }
+
+#[test]
+fn escape_chars() -> Result<(), String> {
+  assert_eq!(
+    lex(r#""\u{6c49}""#)?,
+    vec![
+      CirruLexItem::Indent(0),
+      CirruLexItem::Str(String::from(r#"\u{6c49}"#))
+    ]
+  );
+
+  assert_eq!(
+    lex(r#""\'""#)?,
+    vec![
+      CirruLexItem::Indent(0),
+      CirruLexItem::Str(String::from(r#"'"#))
+    ]
+  );
+  Ok(())
+}