feature: unary prefix/suffix operator support via PrattParser (#710)

Closes #461

based on #344

Co-authored-by: Tomas Tauber <[email protected]>
Co-authored-by: Klas Segeljakt <[email protected]>

derive/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_derive"
 description = "pest's derive macro"
-version = "2.3.1"
+version = "2.4.0"
 edition = "2018"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest-parser.github.io/"
@@ -23,5 +23,5 @@ std = ["pest/std", "pest_generator/std"]
 
 [dependencies]
 # for tests, included transitively anyway
-pest = { path = "../pest", version = "2.3.1", default-features = false }
-pest_generator = { path = "../generator", version = "2.3.1", default-features = false }
+pest = { path = "../pest", version = "2.4.0", default-features = false }
+pest_generator = { path = "../generator", version = "2.4.0", default-features = false }

derive/examples/calc.pest

@@ -0,0 +1,16 @@
+WHITESPACE   =  _{ " " | "\t" | NEWLINE }
+
+ program      =   { SOI ~ expr ~ EOI }
+   expr       =   { prefix* ~ primary ~ postfix* ~ (infix ~ prefix* ~ primary ~ postfix* )* }
+     infix    =  _{ add | sub | mul | div | pow }
+       add    =   { "+" } // Addition
+       sub    =   { "-" } // Subtraction
+       mul    =   { "*" } // Multiplication
+       div    =   { "/" } // Division
+       pow    =   { "^" } // Exponentiation
+     prefix   =  _{ neg }
+       neg    =   { "-" } // Negation
+     postfix  =  _{ fac }
+       fac    =   { "!" } // Factorial
+     primary  =  _{ int | "(" ~ expr ~ ")" }
+       int    =  @{ (ASCII_NONZERO_DIGIT ~ ASCII_DIGIT+ | ASCII_DIGIT) }

derive/examples/calc.rs

@@ -0,0 +1,109 @@
+mod parser {
+    use pest_derive::Parser;
+
+    #[derive(Parser)]
+    #[grammar = "../examples/calc.pest"]
+    pub struct Parser;
+}
+
+use parser::Rule;
+use pest::{
+    iterators::Pairs,
+    pratt_parser::{Assoc::*, Op, PrattParser},
+    Parser,
+};
+use std::io::{stdin, stdout, Write};
+
+fn parse_to_str(pairs: Pairs<Rule>, pratt: &PrattParser<Rule>) -> String {
+    pratt
+        .map_primary(|primary| match primary.as_rule() {
+            Rule::int => primary.as_str().to_owned(),
+            Rule::expr => parse_to_str(primary.into_inner(), pratt),
+            _ => unreachable!(),
+        })
+        .map_prefix(|op, rhs| match op.as_rule() {
+            Rule::neg => format!("(-{})", rhs),
+            _ => unreachable!(),
+        })
+        .map_postfix(|lhs, op| match op.as_rule() {
+            Rule::fac => format!("({}!)", lhs),
+            _ => unreachable!(),
+        })
+        .map_infix(|lhs, op, rhs| match op.as_rule() {
+            Rule::add => format!("({}+{})", lhs, rhs),
+            Rule::sub => format!("({}-{})", lhs, rhs),
+            Rule::mul => format!("({}*{})", lhs, rhs),
+            Rule::div => format!("({}/{})", lhs, rhs),
+            Rule::pow => format!("({}^{})", lhs, rhs),
+            _ => unreachable!(),
+        })
+        .parse(pairs)
+}
+
+fn parse_to_i32(pairs: Pairs<Rule>, pratt: &PrattParser<Rule>) -> i128 {
+    pratt
+        .map_primary(|primary| match primary.as_rule() {
+            Rule::int => primary.as_str().parse().unwrap(),
+            Rule::expr => parse_to_i32(primary.into_inner(), pratt),
+            _ => unreachable!(),
+        })
+        .map_prefix(|op, rhs| match op.as_rule() {
+            Rule::neg => -rhs,
+            _ => unreachable!(),
+        })
+        .map_postfix(|lhs, op| match op.as_rule() {
+            Rule::fac => (1..lhs + 1).product(),
+            _ => unreachable!(),
+        })
+        .map_infix(|lhs, op, rhs| match op.as_rule() {
+            Rule::add => lhs + rhs,
+            Rule::sub => lhs - rhs,
+            Rule::mul => lhs * rhs,
+            Rule::div => lhs / rhs,
+            Rule::pow => (1..rhs + 1).map(|_| lhs).product(),
+            _ => unreachable!(),
+        })
+        .parse(pairs)
+}
+
+fn main() {
+    let pratt = PrattParser::new()
+        .op(Op::infix(Rule::add, Left) | Op::infix(Rule::sub, Left))
+        .op(Op::infix(Rule::mul, Left) | Op::infix(Rule::div, Left))
+        .op(Op::infix(Rule::pow, Right))
+        .op(Op::postfix(Rule::fac))
+        .op(Op::prefix(Rule::neg));
+
+    let stdin = stdin();
+    let mut stdout = stdout();
+
+    loop {
+        let source = {
+            print!("> ");
+            let _ = stdout.flush();
+            let mut input = String::new();
+            let _ = stdin.read_line(&mut input);
+            input.trim().to_string()
+        };
+
+        let pairs = match parser::Parser::parse(Rule::program, &source) {
+            Ok(mut parse_tree) => {
+                parse_tree
+                    .next()
+                    .unwrap()
+                    .into_inner() // inner of program
+                    .next()
+                    .unwrap()
+                    .into_inner() // inner of expr
+            }
+            Err(err) => {
+                println!("Failed parsing input: {:}", err);
+                continue;
+            }
+        };
+
+        print!("{} => ", source);
+        print!("{} => ", parse_to_str(pairs.clone(), &pratt));
+        println!("{}", parse_to_i32(pairs.clone(), &pratt));
+    }
+}

generator/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_generator"
 description = "pest code generator"
-version = "2.3.1"
+version = "2.4.0"
 edition = "2018"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest-parser.github.io/"
@@ -18,8 +18,8 @@ default = ["std"]
 std = ["pest/std"]
 
 [dependencies]
-pest = { path = "../pest", version = "2.3.1", default-features = false }
-pest_meta = { path = "../meta", version = "2.3.1" }
+pest = { path = "../pest", version = "2.4.0", default-features = false }
+pest_meta = { path = "../meta", version = "2.4.0" }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = "1.0"

grammars/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_grammars"
 description = "pest popular grammar implementations"
-version = "2.3.1"
+version = "2.4.0"
 edition = "2018"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest-parser.github.io/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.3.1" }
-pest_derive = { path = "../derive", version = "2.3.1" }
+pest = { path = "../pest", version = "2.4.0" }
+pest_derive = { path = "../derive", version = "2.4.0" }
 
 [dev-dependencies]
 criterion = "0.3"

meta/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_meta"
 description = "pest meta language parser and validator"
-version = "2.3.1"
+version = "2.4.0"
 edition = "2018"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest-parser.github.io/"
@@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.3.1" }
+pest = { path = "../pest", version = "2.4.0" }
 once_cell = "1.8.0"
 
 [build-dependencies]

meta/src/parser.rs

@@ -12,7 +12,7 @@ use std::iter::Peekable;
 
 use pest::error::{Error, ErrorVariant};
 use pest::iterators::{Pair, Pairs};
-use pest::prec_climber::{Assoc, Operator, PrecClimber};
+use pest::pratt_parser::{Assoc, Op, PrattParser};
 use pest::{Parser, Span};
 
 use crate::ast::{Expr, Rule as AstRule, RuleType};
@@ -175,10 +175,9 @@ pub fn consume_rules(pairs: Pairs<Rule>) -> Result<Vec<AstRule>, Vec<Error<Rule>
 }
 
 fn consume_rules_with_spans(pairs: Pairs<Rule>) -> Result<Vec<ParserRule>, Vec<Error<Rule>>> {
-    let climber = PrecClimber::new(vec![
-        Operator::new(Rule::choice_operator, Assoc::Left),
-        Operator::new(Rule::sequence_operator, Assoc::Left),
-    ]);
+    let pratt = PrattParser::new()
+        .op(Op::infix(Rule::choice_operator, Assoc::Left))
+        .op(Op::infix(Rule::sequence_operator, Assoc::Left));
 
     pairs
         .filter(|pair| pair.as_rule() == Rule::grammar_rule)
@@ -210,7 +209,7 @@ fn consume_rules_with_spans(pairs: Pairs<Rule>) -> Result<Vec<ParserRule>, Vec<E
                 inner_nodes.next().unwrap();
             }
 
-            let node = consume_expr(inner_nodes, &climber)?;
+            let node = consume_expr(inner_nodes, &pratt)?;
 
             Ok(ParserRule {
                 name,
@@ -224,17 +223,17 @@ fn consume_rules_with_spans(pairs: Pairs<Rule>) -> Result<Vec<ParserRule>, Vec<E
 
 fn consume_expr<'i>(
     pairs: Peekable<Pairs<'i, Rule>>,
-    climber: &PrecClimber<Rule>,
+    pratt: &PrattParser<Rule>,
 ) -> Result<ParserNode<'i>, Vec<Error<Rule>>> {
     fn unaries<'i>(
         mut pairs: Peekable<Pairs<'i, Rule>>,
-        climber: &PrecClimber<Rule>,
+        pratt: &PrattParser<Rule>,
     ) -> Result<ParserNode<'i>, Vec<Error<Rule>>> {
         let pair = pairs.next().unwrap();
 
         let node = match pair.as_rule() {
             Rule::opening_paren => {
-                let node = unaries(pairs, climber)?;
+                let node = unaries(pairs, pratt)?;
                 let end = node.span.end_pos();
 
                 ParserNode {
@@ -243,7 +242,7 @@ fn consume_expr<'i>(
                 }
             }
             Rule::positive_predicate_operator => {
-                let node = unaries(pairs, climber)?;
+                let node = unaries(pairs, pratt)?;
                 let end = node.span.end_pos();
 
                 ParserNode {
@@ -252,7 +251,7 @@ fn consume_expr<'i>(
                 }
             }
             Rule::negative_predicate_operator => {
-                let node = unaries(pairs, climber)?;
+                let node = unaries(pairs, pratt)?;
                 let end = node.span.end_pos();
 
                 ParserNode {
@@ -262,14 +261,14 @@ fn consume_expr<'i>(
             }
             other_rule => {
                 let node = match other_rule {
-                    Rule::expression => consume_expr(pair.into_inner().peekable(), climber)?,
+                    Rule::expression => consume_expr(pair.into_inner().peekable(), pratt)?,
                     Rule::_push => {
                         let start = pair.clone().as_span().start_pos();
                         let mut pairs = pair.into_inner();
                         pairs.next().unwrap(); // opening_paren
                         let pair = pairs.next().unwrap();
 
-                        let node = consume_expr(pair.into_inner().peekable(), climber)?;
+                        let node = consume_expr(pair.into_inner().peekable(), pratt)?;
                         let end = node.span.end_pos();
 
                         ParserNode {
@@ -529,7 +528,7 @@ fn consume_expr<'i>(
         Ok(node)
     }
 
-    let term = |pair: Pair<'i, Rule>| unaries(pair.into_inner().peekable(), climber);
+    let term = |pair: Pair<'i, Rule>| unaries(pair.into_inner().peekable(), pratt);
     let infix = |lhs: Result<ParserNode<'i>, Vec<Error<Rule>>>,
                  op: Pair<'i, Rule>,
                  rhs: Result<ParserNode<'i>, Vec<Error<Rule>>>| match op.as_rule() {
@@ -560,7 +559,7 @@ fn consume_expr<'i>(
         _ => unreachable!(),
     };
 
-    climber.climb(pairs, term, infix)
+    pratt.map_primary(term).map_infix(infix).parse(pairs)
 }
 
 fn unescape(string: &str) -> Option<String> {

meta/src/validator.rs

@@ -423,7 +423,7 @@ fn left_recursion<'a, 'i: 'a>(rules: HashMap<String, &'a ParserNode<'i>>) -> Vec
                     return Some(Error::new_from_span(
                         ErrorVariant::CustomError {
                             message: format!(
-                                "rule {} is left-recursive ({}); pest::prec_climber might be useful \
+                                "rule {} is left-recursive ({}); pest::pratt_parser might be useful \
                                  in this case",
                                 node.span.as_str(),
                                 chain
@@ -677,7 +677,7 @@ mod tests {
 1 | a = { a }
   |       ^
   |
-  = rule a is left-recursive (a -> a); pest::prec_climber might be useful in this case")]
+  = rule a is left-recursive (a -> a); pest::pratt_parser might be useful in this case")]
     fn simple_left_recursion() {
         let input = "a = { a }";
         unwrap_or_report(consume_rules(
@@ -693,14 +693,14 @@ mod tests {
 1 | a = { b } b = { a }
   |       ^
   |
-  = rule b is left-recursive (b -> a -> b); pest::prec_climber might be useful in this case
+  = rule b is left-recursive (b -> a -> b); pest::pratt_parser might be useful in this case
 
  --> 1:17
   |
 1 | a = { b } b = { a }
   |                 ^
   |
-  = rule a is left-recursive (a -> b -> a); pest::prec_climber might be useful in this case")]
+  = rule a is left-recursive (a -> b -> a); pest::pratt_parser might be useful in this case")]
     fn indirect_left_recursion() {
         let input = "a = { b } b = { a }";
         unwrap_or_report(consume_rules(
@@ -716,7 +716,7 @@ mod tests {
 1 | a = { \"\" ~ \"a\"? ~ \"a\"* ~ (\"a\" | \"\") ~ a }
   |                                       ^
   |
-  = rule a is left-recursive (a -> a); pest::prec_climber might be useful in this case")]
+  = rule a is left-recursive (a -> a); pest::pratt_parser might be useful in this case")]
     fn non_failing_left_recursion() {
         let input = "a = { \"\" ~ \"a\"? ~ \"a\"* ~ (\"a\" | \"\") ~ a }";
         unwrap_or_report(consume_rules(
@@ -732,7 +732,7 @@ mod tests {
 1 | a = { \"a\" | a }
   |             ^
   |
-  = rule a is left-recursive (a -> a); pest::prec_climber might be useful in this case")]
+  = rule a is left-recursive (a -> a); pest::pratt_parser might be useful in this case")]
     fn non_primary_choice_left_recursion() {
         let input = "a = { \"a\" | a }";
         unwrap_or_report(consume_rules(

pest/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest"
 description = "The Elegant Parser"
-version = "2.3.1"
+version = "2.4.0"
 edition = "2018"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest-parser.github.io/"

pest/src/lib.rs

@@ -90,6 +90,12 @@ mod macros;
 mod parser;
 mod parser_state;
 mod position;
+pub mod pratt_parser;
+#[deprecated(
+    since = "2.4.0",
+    note = "Use `pest::pratt_parser` instead (it is an equivalent which also supports unary prefix/suffix operators).
+While prec_climber is going to be kept in 2.x minor and patch releases, it may be removed in a future major release."
+)]
 pub mod prec_climber;
 mod span;
 mod stack;