From 5bfa99ef3caaa93e649a7d8825a983d54458352d Mon Sep 17 00:00:00 2001 From: Tomas Tauber <2410580+tomtau@users.noreply.github.com> Date: Wed, 26 Jul 2023 08:36:35 +0800 Subject: [PATCH] propagate non_exhaustive attribute to the generated Rule enums (#901) --- .github/workflows/release.yml | 2 +- debugger/Cargo.toml | 8 ++--- derive/Cargo.toml | 6 ++-- generator/Cargo.toml | 6 ++-- generator/src/generator.rs | 64 ++++++++++++++++++++++------------- generator/src/lib.rs | 50 ++++++++++++++++++++++----- grammars/Cargo.toml | 6 ++-- meta/Cargo.toml | 4 +-- pest/Cargo.toml | 2 +- vm/Cargo.toml | 6 ++-- 10 files changed, 102 insertions(+), 52 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e86ea4d..7141048d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: - name: Install jq run: sudo apt-get update && sudo apt-get install -y jq - name: Install toolchain - uses: dtolnay/rust-toolchain@1.62.0 + uses: dtolnay/rust-toolchain@1.63.0 # needed for pest_debugger (linux-raw-sys v0.4.3 requires it) - name: Bootstraping Grammars - Building run: cargo build --package pest_bootstrap - name: Bootstraping Grammars - Executing diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml index 543622ec..e11f0a28 100644 --- a/debugger/Cargo.toml +++ b/debugger/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_debugger" description = "pest grammar debugger" -version = "2.7.1" +version = "2.7.2" edition = "2021" authors = ["Dragoș Tiselice ", "Tomas Tauber "] homepage = "https://pest.rs/" @@ -14,9 +14,9 @@ readme = "_README.md" rust-version = "1.60" [dependencies] -pest = { path = "../pest", version = "2.7.1" } -pest_meta = { path = "../meta", version = "2.7.1" } -pest_vm = { path = "../vm", version = "2.7.1" } +pest = { path = "../pest", version = "2.7.2" } +pest_meta = { path = "../meta", version = "2.7.2" } +pest_vm = { path = "../vm", version = "2.7.2" } reqwest = { version = "= 0.11.13", default-features = false, features = ["blocking", "json", "default-tls"] } rustyline = "10" serde_json = "1" diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 5f6110bf..8de0f14e 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_derive" description = "pest's derive macro" -version = "2.7.1" +version = "2.7.2" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -25,5 +25,5 @@ grammar-extras = ["pest_generator/grammar-extras"] [dependencies] # for tests, included transitively anyway -pest = { path = "../pest", version = "2.7.1", default-features = false } -pest_generator = { path = "../generator", version = "2.7.1", default-features = false } +pest = { path = "../pest", version = "2.7.2", default-features = false } +pest_generator = { path = "../generator", version = "2.7.2", default-features = false } diff --git a/generator/Cargo.toml b/generator/Cargo.toml index 018bd927..1763657e 100644 --- a/generator/Cargo.toml +++ b/generator/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_generator" description = "pest code generator" -version = "2.7.1" +version = "2.7.2" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -20,8 +20,8 @@ not-bootstrap-in-src = ["pest_meta/not-bootstrap-in-src"] grammar-extras = ["pest_meta/grammar-extras"] [dependencies] -pest = { path = "../pest", version = "2.7.1", default-features = false } -pest_meta = { path = "../meta", version = "2.7.1" } +pest = { path = "../pest", version = "2.7.2", default-features = false } +pest_meta = { path = "../meta", version = "2.7.2" } proc-macro2 = "1.0" quote = "1.0" syn = "2.0" diff --git a/generator/src/generator.rs b/generator/src/generator.rs index 7677c35a..ac0b9397 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -11,17 +11,17 @@ use std::path::PathBuf; use proc_macro2::TokenStream; use quote::{ToTokens, TokenStreamExt}; -use syn::{self, Generics, Ident}; +use syn::{self, Ident}; use pest::unicode::unicode_property_names; use pest_meta::ast::*; use pest_meta::optimizer::*; use crate::docs::DocComment; +use crate::ParsedDerive; pub(crate) fn generate( - name: Ident, - generics: &Generics, + parsed_derive: ParsedDerive, paths: Vec, rules: Vec, defaults: Vec<&str>, @@ -29,14 +29,14 @@ pub(crate) fn generate( include_grammar: bool, ) -> TokenStream { let uses_eoi = defaults.iter().any(|name| *name == "EOI"); - + let name = parsed_derive.name; let builtins = generate_builtin_rules(); let include_fix = if include_grammar { generate_include(&name, paths) } else { quote!() }; - let rule_enum = generate_enum(&rules, doc_comment, uses_eoi); + let rule_enum = generate_enum(&rules, doc_comment, uses_eoi, parsed_derive.non_exhaustive); let patterns = generate_patterns(&rules, uses_eoi); let skip = generate_skip(&rules); @@ -49,7 +49,7 @@ pub(crate) fn generate( } })); - let (impl_generics, ty_generics, where_clause) = generics.split_for_impl(); + let (impl_generics, ty_generics, where_clause) = parsed_derive.generics.split_for_impl(); let result = result_type(); @@ -197,7 +197,12 @@ fn generate_include(name: &Ident, paths: Vec) -> TokenStream { } } -fn generate_enum(rules: &[OptimizedRule], doc_comment: &DocComment, uses_eoi: bool) -> TokenStream { +fn generate_enum( + rules: &[OptimizedRule], + doc_comment: &DocComment, + uses_eoi: bool, + non_exhaustive: bool, +) -> TokenStream { let rules = rules.iter().map(|rule| { let rule_name = format_ident!("r#{}", rule.name); @@ -213,26 +218,34 @@ fn generate_enum(rules: &[OptimizedRule], doc_comment: &DocComment, uses_eoi: bo }); let grammar_doc = &doc_comment.grammar_doc; + let mut result = quote! { + #[doc = #grammar_doc] + #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] + #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + }; + if non_exhaustive { + result.append_all(quote! { + #[non_exhaustive] + }); + } + result.append_all(quote! { + pub enum Rule + }); if uses_eoi { - quote! { - #[doc = #grammar_doc] - #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] - #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - pub enum Rule { + result.append_all(quote! { + { EOI, #( #rules ),* } - } + }); } else { - quote! { - #[doc = #grammar_doc] - #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] - #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - pub enum Rule { + result.append_all(quote! { + { #( #rules ),* } - } - } + }) + }; + result } fn generate_patterns(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream { @@ -756,6 +769,7 @@ mod tests { use proc_macro2::Span; use std::collections::HashMap; + use syn::Generics; #[test] fn rule_enum_simple() { @@ -774,7 +788,7 @@ mod tests { }; assert_eq!( - generate_enum(&rules, doc_comment, false).to_string(), + generate_enum(&rules, doc_comment, false, false).to_string(), quote! { #[doc = "Rule doc\nhello"] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] @@ -1095,9 +1109,13 @@ mod tests { let base_path = current_dir.join("base.pest").to_str().unwrap().to_string(); let test_path = current_dir.join("test.pest").to_str().unwrap().to_string(); - + let parsed_derive = ParsedDerive { + name, + generics, + non_exhaustive: false, + }; assert_eq!( - generate(name, &generics, vec![PathBuf::from("base.pest"), PathBuf::from("test.pest")], rules, defaults, doc_comment, true).to_string(), + generate(parsed_derive, vec![PathBuf::from("base.pest"), PathBuf::from("test.pest")], rules, defaults, doc_comment, true).to_string(), quote! { #[allow(non_upper_case_globals)] const _PEST_GRAMMAR_MyParser: [&'static str; 2usize] = [include_str!(#base_path), include_str!(#test_path)]; diff --git a/generator/src/lib.rs b/generator/src/lib.rs index 98c72652..cbd13eaf 100644 --- a/generator/src/lib.rs +++ b/generator/src/lib.rs @@ -42,7 +42,7 @@ use pest_meta::{optimizer, unwrap_or_report, validator}; /// "include_str" statement (done in pest_derive, but turned off in the local bootstrap). pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { let ast: DeriveInput = syn::parse2(input).unwrap(); - let (name, generics, contents) = parse_derive(ast); + let (parsed_derive, contents) = parse_derive(ast); let mut data = String::new(); let mut paths = vec![]; @@ -97,8 +97,7 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { let optimized = optimizer::optimize(ast); generator::generate( - name, - &generics, + parsed_derive, paths, optimized, defaults, @@ -120,7 +119,13 @@ enum GrammarSource { Inline(String), } -fn parse_derive(ast: DeriveInput) -> (Ident, Generics, Vec) { +struct ParsedDerive { + pub(crate) name: Ident, + pub(crate) generics: Generics, + pub(crate) non_exhaustive: bool, +} + +fn parse_derive(ast: DeriveInput) -> (ParsedDerive, Vec) { let name = ast.ident; let generics = ast.generics; @@ -142,7 +147,19 @@ fn parse_derive(ast: DeriveInput) -> (Ident, Generics, Vec) { grammar_sources.push(get_attribute(attr)) } - (name, generics, grammar_sources) + let non_exhaustive = ast + .attrs + .iter() + .any(|attr| attr.meta.path().is_ident("non_exhaustive")); + + ( + ParsedDerive { + name, + generics, + non_exhaustive, + }, + grammar_sources, + ) } fn get_attribute(attr: &Attribute) -> GrammarSource { @@ -177,7 +194,7 @@ mod tests { pub struct MyParser<'a, T>; "; let ast = syn::parse_str(definition).unwrap(); - let (_, _, filenames) = parse_derive(ast); + let (_, filenames) = parse_derive(ast); assert_eq!(filenames, [GrammarSource::Inline("GRAMMAR".to_string())]); } @@ -189,8 +206,9 @@ mod tests { pub struct MyParser<'a, T>; "; let ast = syn::parse_str(definition).unwrap(); - let (_, _, filenames) = parse_derive(ast); + let (parsed_derive, filenames) = parse_derive(ast); assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]); + assert!(!parsed_derive.non_exhaustive); } #[test] @@ -202,7 +220,7 @@ mod tests { pub struct MyParser<'a, T>; "; let ast = syn::parse_str(definition).unwrap(); - let (_, _, filenames) = parse_derive(ast); + let (_, filenames) = parse_derive(ast); assert_eq!( filenames, [ @@ -212,6 +230,19 @@ mod tests { ); } + #[test] + fn derive_nonexhaustive() { + let definition = " + #[non_exhaustive] + #[grammar = \"myfile.pest\"] + pub struct MyParser<'a, T>; + "; + let ast = syn::parse_str(definition).unwrap(); + let (parsed_derive, filenames) = parse_derive(ast); + assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]); + assert!(parsed_derive.non_exhaustive); + } + #[test] #[should_panic(expected = "grammar attribute must be a string")] fn derive_wrong_arg() { @@ -242,6 +273,7 @@ mod tests { fn test_generate_doc() { let input = quote! { #[derive(Parser)] + #[non_exhaustive] #[grammar = "../tests/test.pest"] pub struct TestParser; }; @@ -252,7 +284,7 @@ mod tests { #[doc = "A parser for JSON file.\nAnd this is a example for JSON parser.\n\n indent-4-space\n"] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - + #[non_exhaustive] pub enum Rule { #[doc = "Matches foo str, e.g.: `foo`"] r#foo, diff --git a/grammars/Cargo.toml b/grammars/Cargo.toml index 51a4a95d..9cc543be 100644 --- a/grammars/Cargo.toml +++ b/grammars/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_grammars" description = "pest popular grammar implementations" -version = "2.7.1" +version = "2.7.2" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -14,8 +14,8 @@ readme = "_README.md" rust-version = "1.60" [dependencies] -pest = { path = "../pest", version = "2.7.1" } -pest_derive = { path = "../derive", version = "2.7.1" } +pest = { path = "../pest", version = "2.7.2" } +pest_derive = { path = "../derive", version = "2.7.2" } [dev-dependencies] criterion = "0.5" diff --git a/meta/Cargo.toml b/meta/Cargo.toml index 08f91ed4..5aa93d15 100644 --- a/meta/Cargo.toml +++ b/meta/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_meta" description = "pest meta language parser and validator" -version = "2.7.1" +version = "2.7.2" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*" rust-version = "1.60" [dependencies] -pest = { path = "../pest", version = "2.7.1" } +pest = { path = "../pest", version = "2.7.2" } once_cell = "1.8.0" [build-dependencies] diff --git a/pest/Cargo.toml b/pest/Cargo.toml index 9c80930a..34bec34d 100644 --- a/pest/Cargo.toml +++ b/pest/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest" description = "The Elegant Parser" -version = "2.7.1" +version = "2.7.2" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index e50c5ca0..2ce176b8 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pest_vm" description = "pest grammar virtual machine" -version = "2.7.1" +version = "2.7.2" edition = "2021" authors = ["Dragoș Tiselice "] homepage = "https://pest.rs/" @@ -14,8 +14,8 @@ readme = "_README.md" rust-version = "1.60" [dependencies] -pest = { path = "../pest", version = "2.7.1" } -pest_meta = { path = "../meta", version = "2.7.1" } +pest = { path = "../pest", version = "2.7.2" } +pest_meta = { path = "../meta", version = "2.7.2" } [features] grammar-extras = ["pest_meta/grammar-extras"] \ No newline at end of file