Merge pull request #348 from epage/custom

docs(examples): Add lexer/parser example

Cargo.toml

@@ -128,6 +128,7 @@ required-features = ["alloc"]
 name = "arithmetic"
 path = "examples/arithmetic/bench.rs"
 harness = false
+required-features = ["alloc"]
 
 [[bench]]
 name = "contains_token"

examples/arithmetic/bench.rs

@@ -1,19 +1,31 @@
 mod parser;
+mod parser_ast;
+mod parser_lexer;
 
 use winnow::prelude::*;
 
-use parser::expr;
-
 #[allow(clippy::eq_op, clippy::erasing_op)]
 fn arithmetic(c: &mut criterion::Criterion) {
-    let data = "  2*2 / ( 5 - 1) + 3 / 4 * (2 - 7 + 567 *12 /2) + 3*(1+2*( 45 /2));";
+    let data = "  2*2 / ( 5 - 1) + 3 / 4 * (2 - 7 + 567 *12 /2) + 3*(1+2*( 45 /2))";
+    let expected = 2 * 2 / (5 - 1) + 3 * (1 + 2 * (45 / 2));
 
+    assert_eq!(parser::expr.parse(data), Ok(expected));
+    assert_eq!(
+        parser_ast::expr.parse(data).map(|ast| ast.eval()),
+        Ok(expected)
+    );
     assert_eq!(
-        expr.parse_peek(data),
-        Ok((";", 2 * 2 / (5 - 1) + 3 * (1 + 2 * (45 / 2)),))
+        parser_lexer::expr2.parse(data).map(|ast| ast.eval()),
+        Ok(expected)
     );
-    c.bench_function("arithmetic", |b| {
-        b.iter(|| expr.parse_peek(data).unwrap());
+    c.bench_function("direct", |b| {
+        b.iter(|| parser::expr.parse(data).unwrap());
+    });
+    c.bench_function("ast", |b| {
+        b.iter(|| parser_ast::expr.parse(data).unwrap().eval());
+    });
+    c.bench_function("lexer", |b| {
+        b.iter(|| parser_lexer::expr2.parse_peek(data).unwrap());
     });
 }
 

examples/arithmetic/main.rs

@@ -2,32 +2,41 @@ use winnow::prelude::*;
 
 mod parser;
 mod parser_ast;
+mod parser_lexer;
 
 fn main() -> Result<(), lexopt::Error> {
     let args = Args::parse()?;
 
     let input = args.input.as_deref().unwrap_or("1 + 1");
+    if let Err(err) = calc(input, args.implementation) {
+        println!("FAILED");
+        println!("{}", err);
+    }
+
+    Ok(())
+}
 
+fn calc(
+    input: &str,
+    imp: Impl,
+) -> Result<(), winnow::error::ParseError<&str, winnow::error::ContextError>> {
     println!("{} =", input);
-    match args.implementation {
-        Impl::Eval => match parser::expr.parse(input) {
-            Ok(result) => {
-                println!("  {}", result);
-            }
-            Err(err) => {
-                println!("  {}", err);
-            }
-        },
-        Impl::Ast => match parser_ast::expr.parse(input) {
-            Ok(result) => {
-                println!("  {:#?}", result);
-            }
-            Err(err) => {
-                println!("  {}", err);
-            }
-        },
+    match imp {
+        Impl::Eval => {
+            let result = parser::expr.parse(input)?;
+            println!("  {}", result);
+        }
+        Impl::Ast => {
+            let result = parser_ast::expr.parse(input)?;
+            println!("  {:#?}={}", result, result.eval());
+        }
+        Impl::Lexer => {
+            let tokens = parser_lexer::lex.parse(input)?;
+            println!("  {:#?}", tokens);
+            let result = parser_lexer::expr.parse(tokens.as_slice()).unwrap();
+            println!("  {:#?}={}", result, result.eval());
+        }
     }
-
     Ok(())
 }
 
@@ -40,6 +49,7 @@ struct Args {
 enum Impl {
     Eval,
     Ast,
+    Lexer,
 }
 
 impl Default for Impl {
@@ -61,6 +71,7 @@ impl Args {
                     res.implementation = args.value()?.parse_with(|s| match s {
                         "eval" => Ok(Impl::Eval),
                         "ast" => Ok(Impl::Ast),
+                        "lexer" => Ok(Impl::Lexer),
                         _ => Err("expected `eval`, `ast`"),
                     })?;
                 }

examples/arithmetic/parser.rs

@@ -2,7 +2,7 @@ use std::str::FromStr;
 
 use winnow::prelude::*;
 use winnow::{
-    ascii::{digit1 as digits, space0 as spaces},
+    ascii::{digit1 as digits, multispace0 as multispaces},
     combinator::alt,
     combinator::delimited,
     combinator::fold_repeat,
@@ -56,13 +56,9 @@ fn term(i: &mut &str) -> PResult<i64> {
 // we fallback to the parens parser defined above
 fn factor(i: &mut &str) -> PResult<i64> {
     delimited(
-        spaces,
-        alt((
-            digits.try_map(FromStr::from_str),
-            delimited('(', expr, ')'),
-            parens,
-        )),
-        spaces,
+        multispaces,
+        alt((digits.try_map(FromStr::from_str), parens)),
+        multispaces,
     )
     .parse_next(i)
 }
@@ -74,29 +70,68 @@ fn parens(i: &mut &str) -> PResult<i64> {
 
 #[test]
 fn factor_test() {
-    assert_eq!(factor.parse_peek("3"), Ok(("", 3)));
-    assert_eq!(factor.parse_peek(" 12"), Ok(("", 12)));
-    assert_eq!(factor.parse_peek("537  "), Ok(("", 537)));
-    assert_eq!(factor.parse_peek("  24   "), Ok(("", 24)));
+    let input = "3";
+    let expected = Ok(("", 3));
+    assert_eq!(factor.parse_peek(input), expected);
+
+    let input = " 12";
+    let expected = Ok(("", 12));
+    assert_eq!(factor.parse_peek(input), expected);
+
+    let input = "537 ";
+    let expected = Ok(("", 537));
+    assert_eq!(factor.parse_peek(input), expected);
+
+    let input = "  24     ";
+    let expected = Ok(("", 24));
+    assert_eq!(factor.parse_peek(input), expected);
 }
 
 #[test]
 fn term_test() {
-    assert_eq!(term.parse_peek(" 12 *2 /  3"), Ok(("", 8)));
-    assert_eq!(term.parse_peek(" 2* 3  *2 *2 /  3"), Ok(("", 8)));
-    assert_eq!(term.parse_peek(" 48 /  3/2"), Ok(("", 8)));
+    let input = " 12 *2 /  3";
+    let expected = Ok(("", 8));
+    assert_eq!(term.parse_peek(input), expected);
+
+    let input = " 12 *2 /  3";
+    let expected = Ok(("", 8));
+    assert_eq!(term.parse_peek(input), expected);
+
+    let input = " 2* 3  *2 *2 /  3";
+    let expected = Ok(("", 8));
+    assert_eq!(term.parse_peek(input), expected);
+
+    let input = " 48 /  3/2";
+    let expected = Ok(("", 8));
+    assert_eq!(term.parse_peek(input), expected);
 }
 
 #[test]
 fn expr_test() {
-    assert_eq!(expr.parse_peek(" 1 +  2 "), Ok(("", 3)));
-    assert_eq!(expr.parse_peek(" 12 + 6 - 4+  3"), Ok(("", 17)));
-    assert_eq!(expr.parse_peek(" 1 + 2*3 + 4"), Ok(("", 11)));
+    let input = " 1 +  2 ";
+    let expected = Ok(("", 3));
+    assert_eq!(expr.parse_peek(input), expected);
+
+    let input = " 12 + 6 - 4+  3";
+    let expected = Ok(("", 17));
+    assert_eq!(expr.parse_peek(input), expected);
+
+    let input = " 1 + 2*3 + 4";
+    let expected = Ok(("", 11));
+    assert_eq!(expr.parse_peek(input), expected);
 }
 
 #[test]
 fn parens_test() {
-    assert_eq!(expr.parse_peek(" (  2 )"), Ok(("", 2)));
-    assert_eq!(expr.parse_peek(" 2* (  3 + 4 ) "), Ok(("", 14)));
-    assert_eq!(expr.parse_peek("  2*2 / ( 5 - 1) + 3"), Ok(("", 4)));
+    let input = " (  2 )";
+    let expected = Ok(("", 2));
+    assert_eq!(expr.parse_peek(input), expected);
+
+    let input = " 2* (  3 + 4 ) ";
+    let expected = Ok(("", 14));
+    assert_eq!(expr.parse_peek(input), expected);
+
+    let input = "  2*2 / ( 5 - 1) + 3";
+    let expected = Ok(("", 4));
+    assert_eq!(expr.parse_peek(input), expected);
 }