fix: adjust tag association in optional and repeat expressions (#985)

fixes #984, fixes #982

debugger/Cargo.toml

@@ -1,9 +1,12 @@
 [package]
 name = "pest_debugger"
 description = "pest grammar debugger"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
-authors = ["Dragoș Tiselice <[email protected]>", "Tomas Tauber <[email protected]>"]
+authors = [
+    "Dragoș Tiselice <[email protected]>",
+    "Tomas Tauber <[email protected]>",
+]
 homepage = "https://pest.rs/"
 repository = "https://github.com/pest-parser/pest"
 documentation = "https://docs.rs/pest"
@@ -14,10 +17,14 @@ readme = "_README.md"
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.7" }
-pest_meta = { path = "../meta", version = "2.7.7" }
-pest_vm = { path = "../vm", version = "2.7.7" }
-reqwest = { version = "= 0.11.13", default-features = false, features = ["blocking", "json", "default-tls"] }
+pest = { path = "../pest", version = "2.7.8" }
+pest_meta = { path = "../meta", version = "2.7.8" }
+pest_vm = { path = "../vm", version = "2.7.8" }
+reqwest = { version = "= 0.11.13", default-features = false, features = [
+    "blocking",
+    "json",
+    "default-tls",
+] }
 rustyline = "10"
 serde_json = "1"
 thiserror = "1"

derive/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_derive"
 description = "pest's derive macro"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -25,5 +25,5 @@ grammar-extras = ["pest_generator/grammar-extras"]
 
 [dependencies]
 # for tests, included transitively anyway
-pest = { path = "../pest", version = "2.7.7", default-features = false }
-pest_generator = { path = "../generator", version = "2.7.7", default-features = false }
+pest = { path = "../pest", version = "2.7.8", default-features = false }
+pest_generator = { path = "../generator", version = "2.7.8", default-features = false }

derive/tests/implicit.rs

@@ -35,3 +35,12 @@ fn test_implicit_whitespace() {
     assert_eq!(pairs.clone().find_tagged("one_comp").count(), 2);
     assert_eq!(pairs.find_tagged("one_array").count(), 2);
 }
+
+#[test]
+#[cfg(feature = "grammar-extras")]
+fn test_implicit_whitespace_multitag() {
+    let successful_parse = TestImplicitParser::parse(Rule::program, "a a a");
+    assert!(successful_parse.is_ok());
+    let pairs = successful_parse.unwrap();
+    assert_eq!(pairs.clone().find_tagged("tail").count(), 2);
+}

derive/tests/opt.pest

@@ -0,0 +1,8 @@
+expr = {
+  SOI ~
+  #prefix=(STAR)? ~ #suffix=DOT?
+  ~ EOI
+}
+
+STAR={"*"}
+DOT={"."}

derive/tests/opt.rs

@@ -0,0 +1,42 @@
+// Licensed under the Apache License, Version 2.0
+// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
+// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. All files in the project carrying such notice may not be copied,
+// modified, or distributed except according to those terms.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+extern crate alloc;
+extern crate pest;
+extern crate pest_derive;
+
+#[cfg(feature = "grammar-extras")]
+use pest::Parser;
+use pest_derive::Parser;
+
+#[derive(Parser)]
+#[grammar = "../tests/opt.pest"]
+struct TestOptParser;
+
+#[test]
+#[cfg(feature = "grammar-extras")]
+fn test_opt_tag() {
+    let successful_parse = TestOptParser::parse(Rule::expr, "*");
+    assert!(successful_parse.is_ok());
+    let pairs = successful_parse.unwrap();
+    assert!(pairs.find_first_tagged("prefix").is_some());
+    assert!(pairs.find_first_tagged("suffix").is_none());
+
+    // Test with no STAR or DOT
+    let parse_no_components = TestOptParser::parse(Rule::expr, "");
+    assert!(parse_no_components.is_ok());
+    let pairs_no_components = parse_no_components.unwrap();
+    assert!(pairs_no_components.find_first_tagged("prefix").is_none());
+    assert!(pairs_no_components.find_first_tagged("suffix").is_none());
+
+    // Test with only DOT
+    let parse_only_dot = TestOptParser::parse(Rule::expr, ".");
+    assert!(parse_only_dot.is_ok());
+    let pairs_only_dot = parse_only_dot.unwrap();
+    assert!(pairs_only_dot.find_first_tagged("prefix").is_none());
+    assert!(pairs_only_dot.find_first_tagged("suffix").is_some());
+}

generator/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_generator"
 description = "pest code generator"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -22,8 +22,8 @@ grammar-extras = ["pest_meta/grammar-extras"]
 export-internal = []
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.7", default-features = false }
-pest_meta = { path = "../meta", version = "2.7.7" }
+pest = { path = "../pest", version = "2.7.8", default-features = false }
+pest_meta = { path = "../meta", version = "2.7.8" }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = "2.0"

generator/src/generator.rs

@@ -571,12 +571,42 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream {
             }
         }
         #[cfg(feature = "grammar-extras")]
-        OptimizedExpr::NodeTag(expr, tag) => {
-            let expr = generate_expr(*expr);
-            quote! {
-                #expr.and_then(|state| state.tag_node(#tag))
+        OptimizedExpr::NodeTag(expr, tag) => match *expr {
+            OptimizedExpr::Opt(expr) => {
+                let expr = generate_expr(*expr);
+                quote! {
+                    state.optional(|state| {
+                        #expr.and_then(|state| state.tag_node(#tag))
+                    })
+                }
             }
-        }
+            OptimizedExpr::Rep(expr) => {
+                let expr = generate_expr(*expr);
+                quote! {
+                    state.sequence(|state| {
+                        state.optional(|state| {
+                            #expr.and_then(|state| {
+                                state.repeat(|state| {
+                                    state.sequence(|state| {
+                                        super::hidden::skip(
+                                            state
+                                        ).and_then(|state| {
+                                            #expr.and_then(|state| state.tag_node(#tag))
+                                        })
+                                    })
+                                })
+                            }).and_then(|state| state.tag_node(#tag))
+                        })
+                    })
+                }
+            }
+            expr => {
+                let expr = generate_expr(expr);
+                quote! {
+                    #expr.and_then(|state| state.tag_node(#tag))
+                }
+            }
+        },
     }
 }
 
@@ -726,12 +756,32 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream {
             }
         }
         #[cfg(feature = "grammar-extras")]
-        OptimizedExpr::NodeTag(expr, tag) => {
-            let expr = generate_expr_atomic(*expr);
-            quote! {
-                #expr.and_then(|state| state.tag_node(#tag))
+        OptimizedExpr::NodeTag(expr, tag) => match *expr {
+            OptimizedExpr::Opt(expr) => {
+                let expr = generate_expr_atomic(*expr);
+
+                quote! {
+                    state.optional(|state| {
+                        #expr.and_then(|state| state.tag_node(#tag))
+                    })
+                }
             }
-        }
+            OptimizedExpr::Rep(expr) => {
+                let expr = generate_expr_atomic(*expr);
+
+                quote! {
+                    state.repeat(|state| {
+                        #expr.and_then(|state| state.tag_node(#tag))
+                    })
+                }
+            }
+            expr => {
+                let expr = generate_expr_atomic(expr);
+                quote! {
+                    #expr.and_then(|state| state.tag_node(#tag))
+                }
+            }
+        },
     }
 }
 

grammars/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_grammars"
 description = "pest popular grammar implementations"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.7" }
-pest_derive = { path = "../derive", version = "2.7.7" }
+pest = { path = "../pest", version = "2.7.8" }
+pest_derive = { path = "../derive", version = "2.7.8" }
 
 [dev-dependencies]
 criterion = "0.5"

meta/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_meta"
 description = "pest meta language parser and validator"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -12,11 +12,17 @@ categories = ["parsing"]
 license = "MIT OR Apache-2.0"
 readme = "_README.md"
 exclude = ["src/grammar.pest"]
-include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"]
+include = [
+    "Cargo.toml",
+    "src/**/*",
+    "src/grammar.rs",
+    "_README.md",
+    "LICENSE-*",
+]
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.7" }
+pest = { path = "../pest", version = "2.7.8" }
 once_cell = "1.8.0"
 
 [build-dependencies]

pest/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest"
 description = "The Elegant Parser"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"

vm/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "pest_vm"
 description = "pest grammar virtual machine"
-version = "2.7.7"
+version = "2.7.8"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.61"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.7" }
-pest_meta = { path = "../meta", version = "2.7.7" }
+pest = { path = "../pest", version = "2.7.8" }
+pest_meta = { path = "../meta", version = "2.7.8" }
 
 [features]
 grammar-extras = ["pest_meta/grammar-extras"]