chore: added docs for semver changes in grammar + remove fast-col-line (

#787) using line_col from pairs is recommended: #785 (comment) as it's faster or similar performance after the line-index patch Co-authored-by: Tomas Tauber <[email protected]>
pest-parser · Feb 8, 2023 · 2562356 · 2562356
1 parent 14d3ce5
commit 2562356
Show file tree

Hide file tree

Showing 11 changed files with 76 additions and 97 deletions.
diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_debugger"
 description = "pest grammar debugger"
-version = "2.5.4"
+version = "2.5.5"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>", "Tomas Tauber <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -14,9 +14,9 @@ readme = "_README.md"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.5.4" }
-pest_meta = { path = "../meta", version = "2.5.4" }
-pest_vm = { path = "../vm", version = "2.5.4" }
+pest = { path = "../pest", version = "2.5.5" }
+pest_meta = { path = "../meta", version = "2.5.5" }
+pest_vm = { path = "../vm", version = "2.5.5" }
 reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "default-tls"] }
 rustyline = "10"
 serde_json = "1"

diff --git a/derive/Cargo.toml b/derive/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_derive"
 description = "pest's derive macro"
-version = "2.5.4"
+version = "2.5.5"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -23,5 +23,5 @@ std = ["pest/std", "pest_generator/std"]
 
 [dependencies]
 # for tests, included transitively anyway
-pest = { path = "../pest", version = "2.5.4", default-features = false }
-pest_generator = { path = "../generator", version = "2.5.4", default-features = false }
+pest = { path = "../pest", version = "2.5.5", default-features = false }
+pest_generator = { path = "../generator", version = "2.5.5", default-features = false }
diff --git a/generator/Cargo.toml b/generator/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_generator"
 description = "pest code generator"
-version = "2.5.4"
+version = "2.5.5"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -18,8 +18,8 @@ default = ["std"]
 std = ["pest/std"]
 
 [dependencies]
-pest = { path = "../pest", version = "2.5.4", default-features = false }
-pest_meta = { path = "../meta", version = "2.5.4" }
+pest = { path = "../pest", version = "2.5.5", default-features = false }
+pest_meta = { path = "../meta", version = "2.5.5" }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = "1.0"
diff --git a/grammars/Cargo.toml b/grammars/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_grammars"
 description = "pest popular grammar implementations"
-version = "2.5.4"
+version = "2.5.5"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.5.4" }
-pest_derive = { path = "../derive", version = "2.5.4" }
+pest = { path = "../pest", version = "2.5.5" }
+pest_derive = { path = "../derive", version = "2.5.5" }
 
 [dev-dependencies]
 criterion = "0.3"

diff --git a/meta/Cargo.toml b/meta/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_meta"
 description = "pest meta language parser and validator"
-version = "2.5.4"
+version = "2.5.5"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.5.4" }
+pest = { path = "../pest", version = "2.5.5" }
 once_cell = "1.8.0"
 
 [build-dependencies]

diff --git a/meta/src/ast.rs b/meta/src/ast.rs
@@ -49,6 +49,13 @@ pub enum RuleType {
 }
 
 /// All possible rule expressions
+///
+/// # Warning: Semantic Versioning
+/// There may be non-breaking changes to the meta-grammar
+/// between minor versions. Those non-breaking changes, however,
+/// may translate into semver-breaking changes due to the additional variants
+/// propaged from the `Rule` enum. This is a known issue and will be fixed in the
+/// future (e.g. by increasing MSRV and non_exhaustive annotations).
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum Expr {
     /// Matches an exact string, e.g. `"a"`

diff --git a/meta/src/grammar.pest b/meta/src/grammar.pest
@@ -7,6 +7,13 @@
 // option. All files in the project carrying such notice may not be copied,
 // modified, or distributed except according to those terms.
 //! Pest meta-grammar
+//!
+//! # Warning: Semantic Versioning
+//! There may be non-breaking changes to the meta-grammar
+//! between minor versions. Those non-breaking changes, however,
+//! may translate into semver-breaking changes due to the additional variants
+//! added to the `Rule` enum. This is a known issue and will be fixed in the
+//! future (e.g. by increasing MSRV and non_exhaustive annotations).
 
 /// The top-level rule of a grammar.
 grammar_rules = _{ SOI ~ grammar_doc* ~ (grammar_rule)+ ~ EOI }

diff --git a/meta/src/optimizer/mod.rs b/meta/src/optimizer/mod.rs
@@ -102,6 +102,13 @@ pub struct OptimizedRule {
 }
 
 /// The optimized version of the pest AST's `Expr`.
+///
+/// # Warning: Semantic Versioning
+/// There may be non-breaking changes to the meta-grammar
+/// between minor versions. Those non-breaking changes, however,
+/// may translate into semver-breaking changes due to the additional variants
+/// propaged from the `Rule` enum. This is a known issue and will be fixed in the
+/// future (e.g. by increasing MSRV and non_exhaustive annotations).
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum OptimizedExpr {
     /// Matches an exact string, e.g. `"a"`

diff --git a/pest/Cargo.toml b/pest/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest"
 description = "The Elegant Parser"
-version = "2.5.4"
+version = "2.5.5"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -21,14 +21,10 @@ std = ["ucd-trie/std", "thiserror"]
 pretty-print = ["serde", "serde_json"]
 # Enable const fn constructor for `PrecClimber`
 const_prec_climber = []
-# Enable faster `Position::line_col` calculation using SIMD
-# (note that this may have extra overhead for small inputs)
-fast-line-col = ["memchr", "bytecount"]
 
 [dependencies]
 ucd-trie = { version = "0.1.5", default-features = false }
 serde = { version = "1.0.145", optional = true }
 serde_json = { version = "1.0.85", optional = true}
 thiserror = { version = "1.0.37", optional = true }
 memchr = { version = "2", optional = true }
-bytecount = { version = "0.6", optional = true }
diff --git a/pest/src/position.rs b/pest/src/position.rs
@@ -138,8 +138,43 @@ impl<'i> Position<'i> {
         if self.pos > self.input.len() {
             panic!("position out of bounds");
         }
+        let mut pos = self.pos;
+        let slice = &self.input[..pos];
+        let mut chars = slice.chars().peekable();
+
+        let mut line_col = (1, 1);
+
+        while pos != 0 {
+            match chars.next() {
+                Some('\r') => {
+                    if let Some(&'\n') = chars.peek() {
+                        chars.next();
+
+                        if pos == 1 {
+                            pos -= 1;
+                        } else {
+                            pos -= 2;
+                        }
+
+                        line_col = (line_col.0 + 1, 1);
+                    } else {
+                        pos -= 1;
+                        line_col = (line_col.0, line_col.1 + 1);
+                    }
+                }
+                Some('\n') => {
+                    pos -= 1;
+                    line_col = (line_col.0 + 1, 1);
+                }
+                Some(c) => {
+                    pos -= c.len_utf8();
+                    line_col = (line_col.0, line_col.1 + 1);
+                }
+                None => unreachable!(),
+            }
+        }
 
-        line_col(self.input, self.pos, (1, 1))
+        line_col
     }
 
     /// Returns the entire line of the input that contains this `Position`.
@@ -452,79 +487,6 @@ impl<'i> Hash for Position<'i> {
     }
 }
 
-/// Returns the line and column of the given `pos` in `input`.
-pub(crate) fn line_col(input: &str, pos: usize, start: (usize, usize)) -> (usize, usize) {
-    #[cfg(feature = "fast-line-col")]
-    {
-        fast_line_col(input, pos, start)
-    }
-    #[cfg(not(feature = "fast-line-col"))]
-    {
-        original_line_col(input, pos, start)
-    }
-}
-
-#[inline]
-#[cfg(not(feature = "fast-line-col"))]
-pub(crate) fn original_line_col(
-    input: &str,
-    mut pos: usize,
-    start: (usize, usize),
-) -> (usize, usize) {
-    // Position's pos is always a UTF-8 border.
-    let slice = &input[..pos];
-    let mut chars = slice.chars().peekable();
-
-    let mut line_col = start;
-
-    while pos != 0 {
-        match chars.next() {
-            Some('\r') => {
-                if let Some(&'\n') = chars.peek() {
-                    chars.next();
-
-                    if pos == 1 {
-                        pos -= 1;
-                    } else {
-                        pos -= 2;
-                    }
-
-                    line_col = (line_col.0 + 1, 1);
-                } else {
-                    pos -= 1;
-                    line_col = (line_col.0, line_col.1 + 1);
-                }
-            }
-            Some('\n') => {
-                pos -= 1;
-                line_col = (line_col.0 + 1, 1);
-            }
-            Some(c) => {
-                pos -= c.len_utf8();
-                line_col = (line_col.0, line_col.1 + 1);
-            }
-            None => unreachable!(),
-        }
-    }
-
-    line_col
-}
-
-#[inline]
-#[cfg(feature = "fast-line-col")]
-fn fast_line_col(input: &str, pos: usize, start: (usize, usize)) -> (usize, usize) {
-    // Position's pos is always a UTF-8 border.
-    let slice = &input[..pos];
-
-    let prec_ln = memchr::memrchr(b'\n', slice.as_bytes());
-    if let Some(prec_nl_pos) = prec_ln {
-        let lines = bytecount::count(slice[..=prec_nl_pos].as_bytes(), b'\n') + start.0;
-        (lines, slice[prec_nl_pos..].chars().count())
-    } else {
-        (start.0, slice.chars().count() + start.1)
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

diff --git a/vm/Cargo.toml b/vm/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_vm"
 description = "pest grammar virtual machine"
-version = "2.5.4"
+version = "2.5.5"
 edition = "2021"
 authors = ["Dragoș Tiselice <[email protected]>"]
 homepage = "https://pest.rs/"
@@ -14,5 +14,5 @@ readme = "_README.md"
 rust-version = "1.56"
 
 [dependencies]
-pest = { path = "../pest", version = "2.5.4" }
-pest_meta = { path = "../meta", version = "2.5.4" }
+pest = { path = "../pest", version = "2.5.5" }
+pest_meta = { path = "../meta", version = "2.5.5" }