From 98ec9638825f1d74ad441dfdf2087c5fd5ea60d7 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Mon, 9 Jan 2023 22:24:10 +0800 Subject: [PATCH 1/5] Add to support `///` and `//!` syntax for add doc comment for rules. Resolve #748 For example: ```rust //! A parser for JSON file. /// Matches object, e.g.: `{ "foo": "bar" }` object = { "{" ~ pair ~ ("," ~ pair)* ~ "}" | "{" ~ "}" } ``` should generate: ```rust /// A parser for JSON file. enum Rule { /// Matches object, e.g.: `{ "foo": "bar" }` object, } ``` --- generator/src/generator.rs | 36 ++++++++++-- generator/src/lib.rs | 24 +++++++- grammars/src/grammars/json.pest | 3 + meta/src/ast.rs | 3 + meta/src/grammar.pest | 18 ++++-- meta/src/optimizer/concatenator.rs | 8 ++- meta/src/optimizer/factorizer.rs | 8 ++- meta/src/optimizer/lister.rs | 8 ++- meta/src/optimizer/mod.rs | 27 +++++++++ meta/src/optimizer/restorer.rs | 19 ++++++- meta/src/optimizer/rotater.rs | 8 ++- meta/src/optimizer/skipper.rs | 8 ++- meta/src/optimizer/unroller.rs | 8 ++- meta/src/parser.rs | 89 +++++++++++++++++++++++++++--- 14 files changed, 240 insertions(+), 27 deletions(-) diff --git a/generator/src/generator.rs b/generator/src/generator.rs index fc1263d8..2f4173e9 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -23,6 +23,7 @@ pub fn generate( path: Option, rules: Vec, defaults: Vec<&str>, + grammar_docs: Vec<&str>, include_grammar: bool, ) -> TokenStream { let uses_eoi = defaults.iter().any(|name| *name == "EOI"); @@ -36,7 +37,7 @@ pub fn generate( } else { quote!() }; - let rule_enum = generate_enum(&rules, uses_eoi); + let rule_enum = generate_enum(&rules, grammar_docs, uses_eoi); let patterns = generate_patterns(&rules, uses_eoi); let skip = generate_skip(&rules); @@ -181,10 +182,26 @@ fn generate_include(name: &Ident, path: &str) -> TokenStream { } } -fn generate_enum(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream { - let rules = rules.iter().map(|rule| format_ident!("r#{}", rule.name)); +fn generate_enum(rules: &[OptimizedRule], grammar_docs: Vec<&str>, uses_eoi: bool) -> TokenStream { + let rules = rules.iter().map(|rule| { + let rule_name = format_ident!("r#{}", rule.name); + if rule.comments.is_empty() { + quote! { + #rule_name + } + } else { + let comments = rule.comments.join("\n"); + quote! { + #[doc = #comments] + #rule_name + } + } + }); + + let grammar_docs = grammar_docs.join("\n"); if uses_eoi { quote! { + #[doc = #grammar_docs] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum Rule { @@ -194,6 +211,7 @@ fn generate_enum(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream { } } else { quote! { + #[doc = #grammar_docs] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum Rule { @@ -208,6 +226,7 @@ fn generate_patterns(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream { .iter() .map(|rule| { let rule = format_ident!("r#{}", rule.name); + quote! { Rule::#rule => rules::#rule(state) } @@ -667,14 +686,17 @@ mod tests { name: "f".to_owned(), ty: RuleType::Normal, expr: OptimizedExpr::Ident("g".to_owned()), + comments: vec!["This is rule comment".to_owned()], }]; assert_eq!( - generate_enum(&rules, false).to_string(), + generate_enum(&rules, vec!["Rule doc", "hello"], false).to_string(), quote! { + #[doc = "Rule doc\nhello"] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum Rule { + #[doc = "This is rule comment"] r#f } } @@ -966,11 +988,13 @@ mod tests { name: "a".to_owned(), ty: RuleType::Silent, expr: OptimizedExpr::Str("b".to_owned()), + comments: vec![], }, OptimizedRule { name: "if".to_owned(), ty: RuleType::Silent, expr: OptimizedExpr::Ident("a".to_owned()), + comments: vec!["If statement".to_owned()], }, ]; @@ -981,15 +1005,17 @@ mod tests { current_dir.push("test.pest"); let test_path = current_dir.to_str().expect("path contains invalid unicode"); assert_eq!( - generate(name, &generics, Some(PathBuf::from("test.pest")), rules, defaults, true).to_string(), + generate(name, &generics, Some(PathBuf::from("test.pest")), rules, defaults, vec!["This is Rule doc", "This is second line"], true).to_string(), quote! { #[allow(non_upper_case_globals)] const _PEST_GRAMMAR_MyParser: &'static str = include_str!(#test_path); + #[doc = "This is Rule doc\nThis is second line"] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum Rule { r#a, + #[doc = "If statement"] r#if } diff --git a/generator/src/lib.rs b/generator/src/lib.rs index 938bd168..87818a6d 100644 --- a/generator/src/lib.rs +++ b/generator/src/lib.rs @@ -26,6 +26,7 @@ use std::fs::File; use std::io::{self, Read}; use std::path::Path; +use pest::iterators::Pairs; use proc_macro2::TokenStream; use syn::{Attribute, DeriveInput, Generics, Ident, Lit, Meta}; @@ -90,11 +91,32 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { Err(error) => panic!("error parsing \n{}", error.renamed_rules(rename_meta_rule)), }; + let grammar_docs = consume_grammar_doc(pairs.clone()); + let defaults = unwrap_or_report(validator::validate_pairs(pairs.clone())); let ast = unwrap_or_report(parser::consume_rules(pairs)); let optimized = optimizer::optimize(ast); - generator::generate(name, &generics, path, optimized, defaults, include_grammar) + generator::generate( + name, + &generics, + path, + optimized, + defaults, + grammar_docs, + include_grammar, + ) +} + +fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> Vec<&'_ str> { + let mut docs = vec![]; + for pair in pairs { + if pair.as_rule() == Rule::grammar_doc { + docs.push(pair.as_str()[3..pair.as_str().len()].trim()); + } + } + + docs } fn read_file>(path: P) -> io::Result { diff --git a/grammars/src/grammars/json.pest b/grammars/src/grammars/json.pest index f8b423a5..2692e8ab 100644 --- a/grammars/src/grammars/json.pest +++ b/grammars/src/grammars/json.pest @@ -7,8 +7,11 @@ // option. All files in the project carrying such notice may not be copied, // modified, or distributed except according to those terms. +//! A parser for JSON file. +//! And this is a example for JSON parser. json = { SOI ~ (object | array) ~ EOI } +/// Matches object, e.g.: `{ "foo": "bar" }` object = { "{" ~ pair ~ ("," ~ pair)* ~ "}" | "{" ~ "}" } pair = { string ~ ":" ~ value } diff --git a/meta/src/ast.rs b/meta/src/ast.rs index ffac8ea7..781855d1 100644 --- a/meta/src/ast.rs +++ b/meta/src/ast.rs @@ -10,6 +10,7 @@ //! Types for the pest's abstract syntax tree. /// A grammar rule +#[non_exhaustive] #[derive(Clone, Debug, Eq, PartialEq)] pub struct Rule { /// The name of the rule @@ -18,6 +19,8 @@ pub struct Rule { pub ty: RuleType, /// The rule's expression pub expr: Expr, + /// Doc comments of the rule + pub(crate) comments: Vec, } /// All possible rule types diff --git a/meta/src/grammar.pest b/meta/src/grammar.pest index 282ca35b..817588bd 100644 --- a/meta/src/grammar.pest +++ b/meta/src/grammar.pest @@ -7,11 +7,12 @@ // option. All files in the project carrying such notice may not be copied, // modified, or distributed except according to those terms. -grammar_rules = _{ SOI ~ grammar_rule+ ~ EOI } +grammar_rules = _{ SOI ~ grammar_doc* ~ (grammar_rule)+ ~ EOI } grammar_rule = { identifier ~ assignment_operator ~ modifier? ~ - opening_brace ~ expression ~ closing_brace + opening_brace ~ expression ~ closing_brace | + line_doc } assignment_operator = { "=" } @@ -92,7 +93,12 @@ quote = { "\"" } single_quote = { "'" } range_operator = { ".." } -newline = _{ "\n" | "\r\n" } -WHITESPACE = _{ " " | "\t" | newline } -block_comment = _{ "/*" ~ (block_comment | !"*/" ~ ANY)* ~ "*/" } -COMMENT = _{ block_comment | ("//" ~ (!newline ~ ANY)*) } +newline = _{ "\n" | "\r\n" } +WHITESPACE = _{ " " | "\t" | newline } +line_comment = _{ ("//" ~ !("/" | "!") ~ (!newline ~ ANY)*) } +block_comment = _{ "/*" ~ (block_comment | !"*/" ~ ANY)* ~ "*/" } +COMMENT = _{ block_comment | line_comment } + +// ref: https://doc.rust-lang.org/reference/comments.html +grammar_doc = ${ "//!" ~ (!newline ~ ANY)* } +line_doc = ${ "///" ~ !"/" ~ (!newline ~ ANY)* } diff --git a/meta/src/optimizer/concatenator.rs b/meta/src/optimizer/concatenator.rs index 31d3aa53..3e991987 100644 --- a/meta/src/optimizer/concatenator.rs +++ b/meta/src/optimizer/concatenator.rs @@ -10,7 +10,12 @@ use crate::ast::*; pub fn concatenate(rule: Rule) -> Rule { - let Rule { name, ty, expr } = rule; + let Rule { + name, + ty, + expr, + comments, + } = rule; Rule { name, ty, @@ -29,5 +34,6 @@ pub fn concatenate(rule: Rule) -> Rule { expr } }), + comments, } } diff --git a/meta/src/optimizer/factorizer.rs b/meta/src/optimizer/factorizer.rs index cff018b6..c807549d 100644 --- a/meta/src/optimizer/factorizer.rs +++ b/meta/src/optimizer/factorizer.rs @@ -10,7 +10,12 @@ use crate::ast::*; pub fn factor(rule: Rule) -> Rule { - let Rule { name, ty, expr } = rule; + let Rule { + name, + ty, + expr, + comments, + } = rule; Rule { name, ty, @@ -51,5 +56,6 @@ pub fn factor(rule: Rule) -> Rule { expr => expr, } }), + comments, } } diff --git a/meta/src/optimizer/lister.rs b/meta/src/optimizer/lister.rs index e1988503..b9f7ae82 100644 --- a/meta/src/optimizer/lister.rs +++ b/meta/src/optimizer/lister.rs @@ -10,7 +10,12 @@ use crate::ast::*; pub fn list(rule: Rule) -> Rule { - let Rule { name, ty, expr } = rule; + let Rule { + name, + ty, + expr, + comments, + } = rule; Rule { name, ty, @@ -38,5 +43,6 @@ pub fn list(rule: Rule) -> Rule { expr => expr, } }), + comments, } } diff --git a/meta/src/optimizer/mod.rs b/meta/src/optimizer/mod.rs index b1fa05ff..0ebbbe90 100644 --- a/meta/src/optimizer/mod.rs +++ b/meta/src/optimizer/mod.rs @@ -80,6 +80,7 @@ fn rule_to_optimized_rule(rule: Rule) -> OptimizedRule { name: rule.name, ty: rule.ty, expr: to_optimized(rule.expr), + comments: rule.comments, } } @@ -99,6 +100,8 @@ pub struct OptimizedRule { pub ty: RuleType, /// The optimized expression of the rule. pub expr: OptimizedExpr, + /// The doc comments of the rule. + pub comments: Vec, } /// The optimized version of the pest AST's `Expr`. @@ -319,6 +322,7 @@ mod tests { ), Str(String::from("d")) )), + comments: vec![], }] }; let rotated = { @@ -333,6 +337,7 @@ mod tests { Choice(Str(String::from("c")), Str(String::from("d"))) ) )), + comments: vec![], }] }; @@ -350,12 +355,14 @@ mod tests { NegPred(Choice(Str(String::from("a")), Str(String::from("b")))), Ident(String::from("ANY")) ))), + comments: vec![], }] }; let skipped = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, expr: OptimizedExpr::Skip(vec![String::from("a"), String::from("b")]), + comments: vec![], }]; assert_eq!(optimize(rules), skipped); @@ -372,12 +379,14 @@ mod tests { Seq(Str(String::from("a")), Str(String::from("b"))), Seq(Str(String::from("c")), Str(String::from("d"))) )), + comments: vec![], }] }; let concatenated = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, expr: OptimizedExpr::Str(String::from("abcd")), + comments: vec![], }]; assert_eq!(optimize(rules), concatenated); @@ -389,6 +398,7 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepExact(Box::new(Expr::Ident(String::from("a"))), 3), + comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -399,6 +409,7 @@ mod tests { Ident(String::from("a")), Seq(Ident(String::from("a")), Ident(String::from("a"))) )), + comments: vec![], }] }; @@ -411,6 +422,7 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepMax(Box::new(Expr::Str("a".to_owned())), 3), + comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -421,6 +433,7 @@ mod tests { Opt(Str(String::from("a"))), Seq(Opt(Str(String::from("a"))), Opt(Str(String::from("a")))) )), + comments: vec![], }] }; @@ -433,6 +446,7 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepMin(Box::new(Expr::Str("a".to_owned())), 2), + comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -443,6 +457,7 @@ mod tests { Str(String::from("a")), Seq(Str(String::from("a")), Rep(Str(String::from("a")))) )), + comments: vec![], }] }; @@ -455,6 +470,7 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepMinMax(Box::new(Expr::Str("a".to_owned())), 2, 3), + comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -472,6 +488,7 @@ mod tests { Str(String::from("a")), Seq(Str(String::from("a")), Opt(Str(String::from("a")))) )), + comments: vec![], }] }; @@ -489,12 +506,14 @@ mod tests { Seq(Insens(String::from("a")), Insens(String::from("b"))), Seq(Insens(String::from("c")), Insens(String::from("d"))) )), + comments: vec![], }] }; let concatenated = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, expr: OptimizedExpr::Insens(String::from("abcd")), + comments: vec![], }]; assert_eq!(optimize(rules), concatenated); @@ -517,6 +536,7 @@ mod tests { Ident(String::from("d")) ) )), + comments: vec![], }] }; let optimized = { @@ -531,6 +551,7 @@ mod tests { Choice(Ident(String::from("c")), Ident(String::from("d"))) ) )), + comments: vec![], }] }; @@ -548,6 +569,7 @@ mod tests { Seq(Ident(String::from("a")), Ident(String::from("b"))), Ident(String::from("a")) )), + comments: vec![], }] }; let optimized = { @@ -556,6 +578,7 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: box_tree!(Seq(Ident(String::from("a")), Opt(Ident(String::from("b"))))), + comments: vec![], }] }; @@ -573,6 +596,7 @@ mod tests { Ident(String::from("a")), Seq(Ident(String::from("a")), Ident(String::from("b"))) )), + comments: vec![], }] }; let optimized = { @@ -581,6 +605,7 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Silent, expr: box_tree!(Ident(String::from("a"))), + comments: vec![], }] }; @@ -598,6 +623,7 @@ mod tests { Rep(Seq(Ident(String::from("a")), Ident(String::from("b")))), Ident(String::from("a")) )), + comments: vec![], }] }; let optimized = { @@ -609,6 +635,7 @@ mod tests { Ident(String::from("a")), Rep(Seq(Ident(String::from("b")), Ident(String::from("a")))) )), + comments: vec![], }] }; diff --git a/meta/src/optimizer/restorer.rs b/meta/src/optimizer/restorer.rs index e128e03f..cccf9ebb 100644 --- a/meta/src/optimizer/restorer.rs +++ b/meta/src/optimizer/restorer.rs @@ -14,9 +14,19 @@ pub fn restore_on_err( rule: OptimizedRule, rules: &HashMap, ) -> OptimizedRule { - let OptimizedRule { name, ty, expr } = rule; + let OptimizedRule { + name, + ty, + expr, + comments, + } = rule; let expr = expr.map_bottom_up(|expr| wrap_branching_exprs(expr, rules)); - OptimizedRule { name, ty, expr } + OptimizedRule { + name, + ty, + expr, + comments, + } } fn wrap_branching_exprs( @@ -100,6 +110,7 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Opt(Str("a".to_string()))), + comments: vec![], }]; assert_eq!( @@ -114,12 +125,14 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Rep(Push(Str("a".to_string())))), + comments: vec![], }]; let restored = OptimizedRule { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Rep(RestoreOnErr(Push(Str("a".to_string()))))), + comments: vec![], }; assert_eq!( @@ -134,6 +147,7 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Choice(Push(Str("a".to_string())), Str("a".to_string()))), + comments: vec![], }]; let restored = OptimizedRule { @@ -143,6 +157,7 @@ mod tests { RestoreOnErr(Push(Str("a".to_string()))), Str("a".to_string()) )), + comments: vec![], }; assert_eq!( diff --git a/meta/src/optimizer/rotater.rs b/meta/src/optimizer/rotater.rs index 7a7d8fba..23176d4e 100644 --- a/meta/src/optimizer/rotater.rs +++ b/meta/src/optimizer/rotater.rs @@ -35,10 +35,16 @@ pub fn rotate(rule: Rule) -> Rule { } } - let Rule { name, ty, expr } = rule; + let Rule { + name, + ty, + expr, + comments, + } = rule; Rule { name, ty, expr: expr.map_top_down(rotate_internal), + comments, } } diff --git a/meta/src/optimizer/skipper.rs b/meta/src/optimizer/skipper.rs index 40bc5a16..4458af04 100644 --- a/meta/src/optimizer/skipper.rs +++ b/meta/src/optimizer/skipper.rs @@ -28,7 +28,12 @@ pub fn skip(rule: Rule) -> Rule { } } - let Rule { name, ty, expr } = rule; + let Rule { + name, + ty, + expr, + comments, + } = rule; Rule { name, ty, @@ -52,5 +57,6 @@ pub fn skip(rule: Rule) -> Rule { } else { expr }, + comments, } } diff --git a/meta/src/optimizer/unroller.rs b/meta/src/optimizer/unroller.rs index e3c360d9..419a4469 100644 --- a/meta/src/optimizer/unroller.rs +++ b/meta/src/optimizer/unroller.rs @@ -10,7 +10,12 @@ use crate::ast::*; pub fn unroll(rule: Rule) -> Rule { - let Rule { name, ty, expr } = rule; + let Rule { + name, + ty, + expr, + comments, + } = rule; Rule { name, ty, @@ -62,5 +67,6 @@ pub fn unroll(rule: Rule) -> Rule { .unwrap(), expr => expr, }), + comments, } } diff --git a/meta/src/parser.rs b/meta/src/parser.rs index fc0224b3..b71e171c 100644 --- a/meta/src/parser.rs +++ b/meta/src/parser.rs @@ -11,6 +11,7 @@ use std::char; use std::iter::Peekable; +use std::sync::Mutex; use pest::error::{Error, ErrorVariant}; use pest::iterators::{Pair, Pairs}; @@ -50,6 +51,8 @@ pub struct ParserRule<'i> { pub ty: RuleType, /// The rule's parser node pub node: ParserNode<'i>, + /// Doc comments of the rule + pub(crate) comments: Vec, } /// The pest grammar node @@ -167,9 +170,20 @@ pub enum ParserExpr<'i> { } fn convert_rule(rule: ParserRule<'_>) -> AstRule { - let ParserRule { name, ty, node, .. } = rule; + let ParserRule { + name, + ty, + node, + comments, + .. + } = rule; let expr = convert_node(node); - AstRule { name, ty, expr } + AstRule { + name, + ty, + expr, + comments, + } } fn convert_node(node: ParserNode<'_>) -> Expr { @@ -243,10 +257,23 @@ pub fn rename_meta_rule(rule: &Rule) -> String { Rule::insensitive_string => "`^`".to_owned(), Rule::range_operator => "`..`".to_owned(), Rule::single_quote => "`'`".to_owned(), + Rule::grammar_doc => "//!".to_owned(), + Rule::line_doc => "///".to_owned(), other_rule => format!("{:?}", other_rule), } } +fn filter_line_docs(pair: &Pair<'_, Rule>, line_docs: &mut Vec) -> bool { + let mut pairs = pair.clone().into_inner(); + let pair = pairs.next().unwrap(); + if pair.as_rule() == Rule::line_doc { + line_docs.push(pair.as_str()[3..pair.as_str().len()].trim().to_string()); + false + } else { + true + } +} + fn consume_rules_with_spans( pairs: Pairs<'_, Rule>, ) -> Result>, Vec>> { @@ -254,8 +281,11 @@ fn consume_rules_with_spans( .op(Op::infix(Rule::choice_operator, Assoc::Left)) .op(Op::infix(Rule::sequence_operator, Assoc::Left)); + let line_docs: Mutex> = Mutex::new(vec![]); + pairs .filter(|pair| pair.as_rule() == Rule::grammar_rule) + .filter(|pair| filter_line_docs(pair, &mut line_docs.lock().unwrap())) .map(|pair| { let mut pairs = pair.into_inner().peekable(); @@ -286,11 +316,17 @@ fn consume_rules_with_spans( let node = consume_expr(inner_nodes, &pratt)?; + // consume doc comments + let mut line_docs = line_docs.lock().unwrap(); + let comments = line_docs.clone(); + line_docs.clear(); + Ok(ParserRule { name, span, ty, node, + comments, }) }) .collect() @@ -1093,13 +1129,44 @@ mod tests { }; } + #[test] + fn grammar_doc_and_line_doc() { + let input = "//! hello\n/// world\na = { \"a\" }"; + parses_to! { + parser: PestParser, + input: input, + rule: Rule::grammar_rules, + tokens: [ + grammar_doc(0, 9), + grammar_rule(10, 19, [ + line_doc(10, 19), + ]), + grammar_rule(20, 31, [ + identifier(20, 21), + assignment_operator(22, 23), + opening_brace(24, 25), + expression(26, 30, [ + term(26, 30, [ + string(26, 29, [ + quote(26, 27), + inner_str(27, 28), + quote(28, 29) + ]) + ]) + ]), + closing_brace(30, 31), + ]) + ] + }; + } + #[test] fn wrong_identifier() { fails_with! { parser: PestParser, input: "0", rule: Rule::grammar_rules, - positives: vec![Rule::identifier], + positives: vec![Rule::grammar_rule, Rule::grammar_doc], negatives: vec![], pos: 0 }; @@ -1315,8 +1382,11 @@ mod tests { #[test] fn ast() { - let input = - "rule = _{ a{1} ~ \"a\"{3,} ~ b{, 2} ~ \"b\"{1, 2} | !(^\"c\" | PUSH('d'..'e'))?* }"; + let input = r##" + /// This is line comment + /// This is rule + rule = _{ a{1} ~ "a"{3,} ~ b{, 2} ~ "b"{1, 2} | !(^"c" | PUSH('d'..'e'))?* } + "##; let pairs = PestParser::parse(Rule::grammar_rules, input).unwrap(); let ast = consume_rules_with_spans(pairs).unwrap(); @@ -1347,7 +1417,11 @@ mod tests { )))) )) )))))) - ) + ), + comments: vec![ + "This is line comment".to_string(), + "This is rule".to_string(), + ], },] ); } @@ -1368,7 +1442,8 @@ mod tests { expr: Expr::Seq( Box::new(Expr::PeekSlice(-4, None)), Box::new(Expr::PeekSlice(0, Some(3))), - ) + ), + comments: vec![], }], ); } From 65ffea6bc3e81fc90ed196e8a929178191c6dc25 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 10 Jan 2023 02:13:22 +0800 Subject: [PATCH 2/5] Rewrite line_doc extract in pest_generator, avoid change AST. --- generator/src/generator.rs | 69 ++++++++++++++++++---- generator/src/lib.rs | 95 +++++++++++++++++++++++++++++- generator/tests/test.pest | 18 ++++++ grammars/src/grammars/json.pest | 1 + meta/src/ast.rs | 3 - meta/src/optimizer/concatenator.rs | 8 +-- meta/src/optimizer/factorizer.rs | 8 +-- meta/src/optimizer/lister.rs | 8 +-- meta/src/optimizer/mod.rs | 27 --------- meta/src/optimizer/restorer.rs | 19 +----- meta/src/optimizer/rotater.rs | 8 +-- meta/src/optimizer/skipper.rs | 8 +-- meta/src/optimizer/unroller.rs | 8 +-- meta/src/parser.rs | 52 ++++------------ 14 files changed, 187 insertions(+), 145 deletions(-) create mode 100644 generator/tests/test.pest diff --git a/generator/src/generator.rs b/generator/src/generator.rs index 2f4173e9..8a0a8e9e 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -17,13 +17,28 @@ use pest::unicode::unicode_property_names; use pest_meta::ast::*; use pest_meta::optimizer::*; -pub fn generate( +#[derive(Debug)] +pub(crate) struct DocComment<'a> { + pub(crate) grammar_docs: Vec<&'a str>, + pub(crate) line_docs: Vec>, + pub(crate) rules: Vec, +} + +impl DocComment<'_> { + fn line_docs_for_rule(&self, rule_name: &str) -> Option { + let idx = self.rules.iter().position(|r| r.name == rule_name)?; + + self.line_docs.get(idx).map(|comments| comments.join("\n")) + } +} + +pub(crate) fn generate( name: Ident, generics: &Generics, path: Option, rules: Vec, defaults: Vec<&str>, - grammar_docs: Vec<&str>, + doc_comment: &DocComment<'_>, include_grammar: bool, ) -> TokenStream { let uses_eoi = defaults.iter().any(|name| *name == "EOI"); @@ -37,7 +52,7 @@ pub fn generate( } else { quote!() }; - let rule_enum = generate_enum(&rules, grammar_docs, uses_eoi); + let rule_enum = generate_enum(&rules, doc_comment, uses_eoi); let patterns = generate_patterns(&rules, uses_eoi); let skip = generate_skip(&rules); @@ -182,15 +197,21 @@ fn generate_include(name: &Ident, path: &str) -> TokenStream { } } -fn generate_enum(rules: &[OptimizedRule], grammar_docs: Vec<&str>, uses_eoi: bool) -> TokenStream { +fn generate_enum( + rules: &[OptimizedRule], + doc_comment: &DocComment<'_>, + uses_eoi: bool, +) -> TokenStream { let rules = rules.iter().map(|rule| { let rule_name = format_ident!("r#{}", rule.name); - if rule.comments.is_empty() { + + let comments = doc_comment.line_docs_for_rule(&rule.name); + let comments = comments.unwrap_or_else(|| "".to_owned()); + if comments.is_empty() { quote! { #rule_name } } else { - let comments = rule.comments.join("\n"); quote! { #[doc = #comments] #rule_name @@ -198,7 +219,7 @@ fn generate_enum(rules: &[OptimizedRule], grammar_docs: Vec<&str>, uses_eoi: boo } }); - let grammar_docs = grammar_docs.join("\n"); + let grammar_docs = doc_comment.grammar_docs.join("\n"); if uses_eoi { quote! { #[doc = #grammar_docs] @@ -686,11 +707,20 @@ mod tests { name: "f".to_owned(), ty: RuleType::Normal, expr: OptimizedExpr::Ident("g".to_owned()), - comments: vec!["This is rule comment".to_owned()], }]; + let doc_comment = &DocComment { + grammar_docs: vec!["Rule doc", "hello"], + line_docs: vec![vec!["This is rule comment"]], + rules: vec![Rule { + name: "f".to_owned(), + ty: RuleType::Normal, + expr: Expr::Ident("g".to_owned()), + }], + }; + assert_eq!( - generate_enum(&rules, vec!["Rule doc", "hello"], false).to_string(), + generate_enum(&rules, doc_comment, false).to_string(), quote! { #[doc = "Rule doc\nhello"] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] @@ -988,16 +1018,31 @@ mod tests { name: "a".to_owned(), ty: RuleType::Silent, expr: OptimizedExpr::Str("b".to_owned()), - comments: vec![], }, OptimizedRule { name: "if".to_owned(), ty: RuleType::Silent, expr: OptimizedExpr::Ident("a".to_owned()), - comments: vec!["If statement".to_owned()], }, ]; + let doc_comment = &DocComment { + line_docs: vec![vec![], vec!["If statement"]], + grammar_docs: vec!["This is Rule doc", "This is second line"], + rules: vec![ + Rule { + name: "a".to_owned(), + ty: RuleType::Silent, + expr: Expr::Str("b".to_owned()), + }, + Rule { + name: "if".to_owned(), + ty: RuleType::Silent, + expr: Expr::Str("b".to_owned()), + }, + ], + }; + let defaults = vec!["ANY"]; let result = result_type(); let box_ty = box_type(); @@ -1005,7 +1050,7 @@ mod tests { current_dir.push("test.pest"); let test_path = current_dir.to_str().expect("path contains invalid unicode"); assert_eq!( - generate(name, &generics, Some(PathBuf::from("test.pest")), rules, defaults, vec!["This is Rule doc", "This is second line"], true).to_string(), + generate(name, &generics, Some(PathBuf::from("test.pest")), rules, defaults, doc_comment, true).to_string(), quote! { #[allow(non_upper_case_globals)] const _PEST_GRAMMAR_MyParser: &'static str = include_str!(#test_path); diff --git a/generator/src/lib.rs b/generator/src/lib.rs index 87818a6d..d431c0b9 100644 --- a/generator/src/lib.rs +++ b/generator/src/lib.rs @@ -37,6 +37,8 @@ mod generator; use pest_meta::parser::{self, rename_meta_rule, Rule}; use pest_meta::{optimizer, unwrap_or_report, validator}; +use generator::DocComment; + /// Processes the derive/proc macro input and generates the corresponding parser based /// on the parsed grammar. If `include_grammar` is set to true, it'll generate an explicit /// "include_str" statement (done in pest_derive, but turned off in the local bootstrap). @@ -92,10 +94,17 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { }; let grammar_docs = consume_grammar_doc(pairs.clone()); + let line_docs = consume_line_docs(pairs.clone()); let defaults = unwrap_or_report(validator::validate_pairs(pairs.clone())); let ast = unwrap_or_report(parser::consume_rules(pairs)); - let optimized = optimizer::optimize(ast); + let optimized = optimizer::optimize(ast.clone()); + + let doc_comment = DocComment { + grammar_docs, + line_docs, + rules: ast, + }; generator::generate( name, @@ -103,7 +112,7 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { path, optimized, defaults, - grammar_docs, + &doc_comment, include_grammar, ) } @@ -119,6 +128,32 @@ fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> Vec<&'_ str> { docs } +fn consume_line_docs(pairs: Pairs<'_, Rule>) -> Vec> { + let mut docs = vec![]; + let mut comments = vec![]; + + for pair in pairs { + if pair.as_rule() == Rule::grammar_rule { + if let Some(inner) = pair.into_inner().next() { + if inner.as_rule() == Rule::line_doc { + comments.push(inner.as_str()[3..inner.as_str().len()].trim()); + continue; + } else { + docs.push(comments); + comments = vec![]; + } + } + } + + if !comments.is_empty() { + docs.push(comments); + comments = vec![]; + } + } + + docs +} + fn read_file>(path: P) -> io::Result { let mut file = File::open(path.as_ref())?; let mut string = String::new(); @@ -177,9 +212,12 @@ fn get_attribute(attr: &Attribute) -> GrammarSource { #[cfg(test)] mod tests { + use super::consume_line_docs; use super::parse_derive; use super::GrammarSource; + use pest_meta::parser::{self, Rule}; + #[test] fn derive_inline_file() { let definition = " @@ -247,4 +285,57 @@ mod tests { let ast = syn::parse_str(definition).unwrap(); parse_derive(ast); } + + #[test] + fn test_consume_line_docs() { + let pairs = match parser::parse(Rule::grammar_rules, include_str!("../tests/test.pest")) { + Ok(pairs) => pairs, + Err(_) => panic!("error parsing tests/test.pest"), + }; + + let line_docs = consume_line_docs(pairs); + assert_eq!( + vec![ + vec!["Matches foo str, e.g.: `foo`"], + vec!["Matches bar str,", "e.g: `bar` or `foobar`"], + vec![], + vec!["Matches dar", "Match dar description"] + ], + line_docs + ); + } + + #[test] + fn test_generate_doc() { + let input = quote! { + #[derive(Parser)] + #[grammar = "../tests/test.pest"] + pub struct TestParser; + }; + + let token = super::derive_parser(input, true); + + let expected = quote! { + #[doc = "A parser for JSON file.\nAnd this is a example for JSON parser."] + #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] + #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + + pub enum Rule { + #[doc = "Matches foo str, e.g.: `foo`"] + r#foo, + #[doc = "Matches bar str,\ne.g: `bar` or `foobar`"] + r#bar, + r#bar1, + #[doc = "Matches dar\nMatch dar description"] + r#dar + } + }; + + assert!( + token.to_string().contains(expected.to_string().as_str()), + "{}\n\nExpected to contains:\n{}", + token, + expected + ); + } } diff --git a/generator/tests/test.pest b/generator/tests/test.pest new file mode 100644 index 00000000..28f6b88a --- /dev/null +++ b/generator/tests/test.pest @@ -0,0 +1,18 @@ +//! A parser for JSON file. +//! And this is a example for JSON parser. + +/// Matches foo str, e.g.: `foo` +foo = { "foo" } + +/// Matches bar str, +/// e.g: `bar` or `foobar` + +bar = { "bar" | "foobar" } + +bar1 = { "bar1" } + +/// Matches dar + +/// Match dar description + +dar = { "da" } \ No newline at end of file diff --git a/grammars/src/grammars/json.pest b/grammars/src/grammars/json.pest index 2692e8ab..4d0d300f 100644 --- a/grammars/src/grammars/json.pest +++ b/grammars/src/grammars/json.pest @@ -12,6 +12,7 @@ json = { SOI ~ (object | array) ~ EOI } /// Matches object, e.g.: `{ "foo": "bar" }` +/// Foobar object = { "{" ~ pair ~ ("," ~ pair)* ~ "}" | "{" ~ "}" } pair = { string ~ ":" ~ value } diff --git a/meta/src/ast.rs b/meta/src/ast.rs index 781855d1..ffac8ea7 100644 --- a/meta/src/ast.rs +++ b/meta/src/ast.rs @@ -10,7 +10,6 @@ //! Types for the pest's abstract syntax tree. /// A grammar rule -#[non_exhaustive] #[derive(Clone, Debug, Eq, PartialEq)] pub struct Rule { /// The name of the rule @@ -19,8 +18,6 @@ pub struct Rule { pub ty: RuleType, /// The rule's expression pub expr: Expr, - /// Doc comments of the rule - pub(crate) comments: Vec, } /// All possible rule types diff --git a/meta/src/optimizer/concatenator.rs b/meta/src/optimizer/concatenator.rs index 3e991987..31d3aa53 100644 --- a/meta/src/optimizer/concatenator.rs +++ b/meta/src/optimizer/concatenator.rs @@ -10,12 +10,7 @@ use crate::ast::*; pub fn concatenate(rule: Rule) -> Rule { - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, @@ -34,6 +29,5 @@ pub fn concatenate(rule: Rule) -> Rule { expr } }), - comments, } } diff --git a/meta/src/optimizer/factorizer.rs b/meta/src/optimizer/factorizer.rs index c807549d..cff018b6 100644 --- a/meta/src/optimizer/factorizer.rs +++ b/meta/src/optimizer/factorizer.rs @@ -10,12 +10,7 @@ use crate::ast::*; pub fn factor(rule: Rule) -> Rule { - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, @@ -56,6 +51,5 @@ pub fn factor(rule: Rule) -> Rule { expr => expr, } }), - comments, } } diff --git a/meta/src/optimizer/lister.rs b/meta/src/optimizer/lister.rs index b9f7ae82..e1988503 100644 --- a/meta/src/optimizer/lister.rs +++ b/meta/src/optimizer/lister.rs @@ -10,12 +10,7 @@ use crate::ast::*; pub fn list(rule: Rule) -> Rule { - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, @@ -43,6 +38,5 @@ pub fn list(rule: Rule) -> Rule { expr => expr, } }), - comments, } } diff --git a/meta/src/optimizer/mod.rs b/meta/src/optimizer/mod.rs index 0ebbbe90..b1fa05ff 100644 --- a/meta/src/optimizer/mod.rs +++ b/meta/src/optimizer/mod.rs @@ -80,7 +80,6 @@ fn rule_to_optimized_rule(rule: Rule) -> OptimizedRule { name: rule.name, ty: rule.ty, expr: to_optimized(rule.expr), - comments: rule.comments, } } @@ -100,8 +99,6 @@ pub struct OptimizedRule { pub ty: RuleType, /// The optimized expression of the rule. pub expr: OptimizedExpr, - /// The doc comments of the rule. - pub comments: Vec, } /// The optimized version of the pest AST's `Expr`. @@ -322,7 +319,6 @@ mod tests { ), Str(String::from("d")) )), - comments: vec![], }] }; let rotated = { @@ -337,7 +333,6 @@ mod tests { Choice(Str(String::from("c")), Str(String::from("d"))) ) )), - comments: vec![], }] }; @@ -355,14 +350,12 @@ mod tests { NegPred(Choice(Str(String::from("a")), Str(String::from("b")))), Ident(String::from("ANY")) ))), - comments: vec![], }] }; let skipped = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, expr: OptimizedExpr::Skip(vec![String::from("a"), String::from("b")]), - comments: vec![], }]; assert_eq!(optimize(rules), skipped); @@ -379,14 +372,12 @@ mod tests { Seq(Str(String::from("a")), Str(String::from("b"))), Seq(Str(String::from("c")), Str(String::from("d"))) )), - comments: vec![], }] }; let concatenated = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, expr: OptimizedExpr::Str(String::from("abcd")), - comments: vec![], }]; assert_eq!(optimize(rules), concatenated); @@ -398,7 +389,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepExact(Box::new(Expr::Ident(String::from("a"))), 3), - comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -409,7 +399,6 @@ mod tests { Ident(String::from("a")), Seq(Ident(String::from("a")), Ident(String::from("a"))) )), - comments: vec![], }] }; @@ -422,7 +411,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepMax(Box::new(Expr::Str("a".to_owned())), 3), - comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -433,7 +421,6 @@ mod tests { Opt(Str(String::from("a"))), Seq(Opt(Str(String::from("a"))), Opt(Str(String::from("a")))) )), - comments: vec![], }] }; @@ -446,7 +433,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepMin(Box::new(Expr::Str("a".to_owned())), 2), - comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -457,7 +443,6 @@ mod tests { Str(String::from("a")), Seq(Str(String::from("a")), Rep(Str(String::from("a")))) )), - comments: vec![], }] }; @@ -470,7 +455,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: Expr::RepMinMax(Box::new(Expr::Str("a".to_owned())), 2, 3), - comments: vec![], }]; let unrolled = { use crate::optimizer::OptimizedExpr::*; @@ -488,7 +472,6 @@ mod tests { Str(String::from("a")), Seq(Str(String::from("a")), Opt(Str(String::from("a")))) )), - comments: vec![], }] }; @@ -506,14 +489,12 @@ mod tests { Seq(Insens(String::from("a")), Insens(String::from("b"))), Seq(Insens(String::from("c")), Insens(String::from("d"))) )), - comments: vec![], }] }; let concatenated = vec![OptimizedRule { name: "rule".to_owned(), ty: RuleType::Atomic, expr: OptimizedExpr::Insens(String::from("abcd")), - comments: vec![], }]; assert_eq!(optimize(rules), concatenated); @@ -536,7 +517,6 @@ mod tests { Ident(String::from("d")) ) )), - comments: vec![], }] }; let optimized = { @@ -551,7 +531,6 @@ mod tests { Choice(Ident(String::from("c")), Ident(String::from("d"))) ) )), - comments: vec![], }] }; @@ -569,7 +548,6 @@ mod tests { Seq(Ident(String::from("a")), Ident(String::from("b"))), Ident(String::from("a")) )), - comments: vec![], }] }; let optimized = { @@ -578,7 +556,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Atomic, expr: box_tree!(Seq(Ident(String::from("a")), Opt(Ident(String::from("b"))))), - comments: vec![], }] }; @@ -596,7 +573,6 @@ mod tests { Ident(String::from("a")), Seq(Ident(String::from("a")), Ident(String::from("b"))) )), - comments: vec![], }] }; let optimized = { @@ -605,7 +581,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Silent, expr: box_tree!(Ident(String::from("a"))), - comments: vec![], }] }; @@ -623,7 +598,6 @@ mod tests { Rep(Seq(Ident(String::from("a")), Ident(String::from("b")))), Ident(String::from("a")) )), - comments: vec![], }] }; let optimized = { @@ -635,7 +609,6 @@ mod tests { Ident(String::from("a")), Rep(Seq(Ident(String::from("b")), Ident(String::from("a")))) )), - comments: vec![], }] }; diff --git a/meta/src/optimizer/restorer.rs b/meta/src/optimizer/restorer.rs index cccf9ebb..e128e03f 100644 --- a/meta/src/optimizer/restorer.rs +++ b/meta/src/optimizer/restorer.rs @@ -14,19 +14,9 @@ pub fn restore_on_err( rule: OptimizedRule, rules: &HashMap, ) -> OptimizedRule { - let OptimizedRule { - name, - ty, - expr, - comments, - } = rule; + let OptimizedRule { name, ty, expr } = rule; let expr = expr.map_bottom_up(|expr| wrap_branching_exprs(expr, rules)); - OptimizedRule { - name, - ty, - expr, - comments, - } + OptimizedRule { name, ty, expr } } fn wrap_branching_exprs( @@ -110,7 +100,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Opt(Str("a".to_string()))), - comments: vec![], }]; assert_eq!( @@ -125,14 +114,12 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Rep(Push(Str("a".to_string())))), - comments: vec![], }]; let restored = OptimizedRule { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Rep(RestoreOnErr(Push(Str("a".to_string()))))), - comments: vec![], }; assert_eq!( @@ -147,7 +134,6 @@ mod tests { name: "rule".to_owned(), ty: RuleType::Normal, expr: box_tree!(Choice(Push(Str("a".to_string())), Str("a".to_string()))), - comments: vec![], }]; let restored = OptimizedRule { @@ -157,7 +143,6 @@ mod tests { RestoreOnErr(Push(Str("a".to_string()))), Str("a".to_string()) )), - comments: vec![], }; assert_eq!( diff --git a/meta/src/optimizer/rotater.rs b/meta/src/optimizer/rotater.rs index 23176d4e..7a7d8fba 100644 --- a/meta/src/optimizer/rotater.rs +++ b/meta/src/optimizer/rotater.rs @@ -35,16 +35,10 @@ pub fn rotate(rule: Rule) -> Rule { } } - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, expr: expr.map_top_down(rotate_internal), - comments, } } diff --git a/meta/src/optimizer/skipper.rs b/meta/src/optimizer/skipper.rs index 4458af04..40bc5a16 100644 --- a/meta/src/optimizer/skipper.rs +++ b/meta/src/optimizer/skipper.rs @@ -28,12 +28,7 @@ pub fn skip(rule: Rule) -> Rule { } } - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, @@ -57,6 +52,5 @@ pub fn skip(rule: Rule) -> Rule { } else { expr }, - comments, } } diff --git a/meta/src/optimizer/unroller.rs b/meta/src/optimizer/unroller.rs index 419a4469..e3c360d9 100644 --- a/meta/src/optimizer/unroller.rs +++ b/meta/src/optimizer/unroller.rs @@ -10,12 +10,7 @@ use crate::ast::*; pub fn unroll(rule: Rule) -> Rule { - let Rule { - name, - ty, - expr, - comments, - } = rule; + let Rule { name, ty, expr } = rule; Rule { name, ty, @@ -67,6 +62,5 @@ pub fn unroll(rule: Rule) -> Rule { .unwrap(), expr => expr, }), - comments, } } diff --git a/meta/src/parser.rs b/meta/src/parser.rs index b71e171c..fabc5bdd 100644 --- a/meta/src/parser.rs +++ b/meta/src/parser.rs @@ -11,7 +11,6 @@ use std::char; use std::iter::Peekable; -use std::sync::Mutex; use pest::error::{Error, ErrorVariant}; use pest::iterators::{Pair, Pairs}; @@ -51,8 +50,6 @@ pub struct ParserRule<'i> { pub ty: RuleType, /// The rule's parser node pub node: ParserNode<'i>, - /// Doc comments of the rule - pub(crate) comments: Vec, } /// The pest grammar node @@ -170,20 +167,9 @@ pub enum ParserExpr<'i> { } fn convert_rule(rule: ParserRule<'_>) -> AstRule { - let ParserRule { - name, - ty, - node, - comments, - .. - } = rule; + let ParserRule { name, ty, node, .. } = rule; let expr = convert_node(node); - AstRule { - name, - ty, - expr, - comments, - } + AstRule { name, ty, expr } } fn convert_node(node: ParserNode<'_>) -> Expr { @@ -263,17 +249,6 @@ pub fn rename_meta_rule(rule: &Rule) -> String { } } -fn filter_line_docs(pair: &Pair<'_, Rule>, line_docs: &mut Vec) -> bool { - let mut pairs = pair.clone().into_inner(); - let pair = pairs.next().unwrap(); - if pair.as_rule() == Rule::line_doc { - line_docs.push(pair.as_str()[3..pair.as_str().len()].trim().to_string()); - false - } else { - true - } -} - fn consume_rules_with_spans( pairs: Pairs<'_, Rule>, ) -> Result>, Vec>> { @@ -281,11 +256,15 @@ fn consume_rules_with_spans( .op(Op::infix(Rule::choice_operator, Assoc::Left)) .op(Op::infix(Rule::sequence_operator, Assoc::Left)); - let line_docs: Mutex> = Mutex::new(vec![]); - pairs .filter(|pair| pair.as_rule() == Rule::grammar_rule) - .filter(|pair| filter_line_docs(pair, &mut line_docs.lock().unwrap())) + .filter(|pair| { + // To ignore `grammar_rule > line_doc` pairs + let mut pairs = pair.clone().into_inner(); + let pair = pairs.next().unwrap(); + + pair.as_rule() != Rule::line_doc + }) .map(|pair| { let mut pairs = pair.into_inner().peekable(); @@ -316,17 +295,11 @@ fn consume_rules_with_spans( let node = consume_expr(inner_nodes, &pratt)?; - // consume doc comments - let mut line_docs = line_docs.lock().unwrap(); - let comments = line_docs.clone(); - line_docs.clear(); - Ok(ParserRule { name, span, ty, node, - comments, }) }) .collect() @@ -1417,11 +1390,7 @@ mod tests { )))) )) )))))) - ), - comments: vec![ - "This is line comment".to_string(), - "This is rule".to_string(), - ], + ) },] ); } @@ -1443,7 +1412,6 @@ mod tests { Box::new(Expr::PeekSlice(-4, None)), Box::new(Expr::PeekSlice(0, Some(3))), ), - comments: vec![], }], ); } From f14a9577b986913b5925deabfcc7577cedbde94e Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Wed, 18 Jan 2023 11:45:30 +0800 Subject: [PATCH 3/5] Improve grammar_doc, line_doc parse for keep whitespaces in head and tail. --- generator/src/lib.rs | 12 +++++++----- generator/tests/test.pest | 4 +++- meta/src/grammar.pest | 6 ++++-- meta/src/parser.rs | 8 ++++++-- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/generator/src/lib.rs b/generator/src/lib.rs index d431c0b9..7418a37c 100644 --- a/generator/src/lib.rs +++ b/generator/src/lib.rs @@ -121,7 +121,8 @@ fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> Vec<&'_ str> { let mut docs = vec![]; for pair in pairs { if pair.as_rule() == Rule::grammar_doc { - docs.push(pair.as_str()[3..pair.as_str().len()].trim()); + let inner_doc = pair.into_inner().next().unwrap(); + docs.push(inner_doc.as_str()); } } @@ -136,7 +137,8 @@ fn consume_line_docs(pairs: Pairs<'_, Rule>) -> Vec> { if pair.as_rule() == Rule::grammar_rule { if let Some(inner) = pair.into_inner().next() { if inner.as_rule() == Rule::line_doc { - comments.push(inner.as_str()[3..inner.as_str().len()].trim()); + let inner_doc = inner.into_inner().next().unwrap(); + comments.push(inner_doc.as_str()); continue; } else { docs.push(comments); @@ -297,7 +299,7 @@ mod tests { assert_eq!( vec![ vec!["Matches foo str, e.g.: `foo`"], - vec!["Matches bar str,", "e.g: `bar` or `foobar`"], + vec!["Matches bar str,", " Indent 2, e.g: `bar` or `foobar`"], vec![], vec!["Matches dar", "Match dar description"] ], @@ -316,14 +318,14 @@ mod tests { let token = super::derive_parser(input, true); let expected = quote! { - #[doc = "A parser for JSON file.\nAnd this is a example for JSON parser."] + #[doc = "A parser for JSON file.\nAnd this is a example for JSON parser.\n\n indent-4-space"] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum Rule { #[doc = "Matches foo str, e.g.: `foo`"] r#foo, - #[doc = "Matches bar str,\ne.g: `bar` or `foobar`"] + #[doc = "Matches bar str,\n Indent 2, e.g: `bar` or `foobar`"] r#bar, r#bar1, #[doc = "Matches dar\nMatch dar description"] diff --git a/generator/tests/test.pest b/generator/tests/test.pest index 28f6b88a..c86a65ff 100644 --- a/generator/tests/test.pest +++ b/generator/tests/test.pest @@ -1,11 +1,13 @@ //! A parser for JSON file. //! And this is a example for JSON parser. +//! +//! indent-4-space /// Matches foo str, e.g.: `foo` foo = { "foo" } /// Matches bar str, -/// e.g: `bar` or `foobar` +/// Indent 2, e.g: `bar` or `foobar` bar = { "bar" | "foobar" } diff --git a/meta/src/grammar.pest b/meta/src/grammar.pest index 817588bd..e09bf61b 100644 --- a/meta/src/grammar.pest +++ b/meta/src/grammar.pest @@ -100,5 +100,7 @@ block_comment = _{ "/*" ~ (block_comment | !"*/" ~ ANY)* ~ "*/" } COMMENT = _{ block_comment | line_comment } // ref: https://doc.rust-lang.org/reference/comments.html -grammar_doc = ${ "//!" ~ (!newline ~ ANY)* } -line_doc = ${ "///" ~ !"/" ~ (!newline ~ ANY)* } +space = _{ " " | "\t" } +grammar_doc = ${ "//!" ~ space? ~ inner_doc } +line_doc = ${ "///" ~ space? ~ !"/" ~ inner_doc } +inner_doc = @{ (!newline ~ ANY)* } diff --git a/meta/src/parser.rs b/meta/src/parser.rs index fabc5bdd..eb957a16 100644 --- a/meta/src/parser.rs +++ b/meta/src/parser.rs @@ -1110,9 +1110,13 @@ mod tests { input: input, rule: Rule::grammar_rules, tokens: [ - grammar_doc(0, 9), + grammar_doc(0, 9, [ + inner_doc(4, 9), + ]), grammar_rule(10, 19, [ - line_doc(10, 19), + line_doc(10, 19, [ + inner_doc(14, 19), + ]), ]), grammar_rule(20, 31, [ identifier(20, 21), From 63985fccc118f1b63feffdcc65de4bf64aba98a1 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Wed, 18 Jan 2023 16:31:21 +0800 Subject: [PATCH 4/5] Rewrite line_docs generator by use HashMap with rule_name. --- generator/src/generator.rs | 95 ++++++++++++++++----------------- generator/src/lib.rs | 88 ++++++++++++++++++------------ grammars/src/grammars/json.pest | 1 + 3 files changed, 102 insertions(+), 82 deletions(-) diff --git a/generator/src/generator.rs b/generator/src/generator.rs index 8a0a8e9e..2a1aa87d 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -7,6 +7,7 @@ // option. All files in the project carrying such notice may not be copied, // modified, or distributed except according to those terms. +use std::collections::HashMap; use std::path::PathBuf; use proc_macro2::TokenStream; @@ -18,17 +19,31 @@ use pest_meta::ast::*; use pest_meta::optimizer::*; #[derive(Debug)] -pub(crate) struct DocComment<'a> { - pub(crate) grammar_docs: Vec<&'a str>, - pub(crate) line_docs: Vec>, - pub(crate) rules: Vec, +pub(crate) struct DocComment { + /// Multi-line grammar doc, (joined with `\n`) + /// + /// e.g. + /// + /// ```ignore + /// "grammar doc 1\ngrammar doc 2" + /// ``` + grammar_doc: String, + /// HashMap rule name and doc comments (joined with `\n`) + /// + /// e.g. + /// + /// ```ignore + /// { "foo": "line doc 1\nline doc 2", "bar": "line doc 3" } + /// ``` + line_docs: HashMap, } -impl DocComment<'_> { - fn line_docs_for_rule(&self, rule_name: &str) -> Option { - let idx = self.rules.iter().position(|r| r.name == rule_name)?; - - self.line_docs.get(idx).map(|comments| comments.join("\n")) +impl DocComment { + pub fn new(grammar_doc: String, line_docs: HashMap) -> Self { + Self { + grammar_doc, + line_docs, + } } } @@ -38,7 +53,7 @@ pub(crate) fn generate( path: Option, rules: Vec, defaults: Vec<&str>, - doc_comment: &DocComment<'_>, + doc_comment: &DocComment, include_grammar: bool, ) -> TokenStream { let uses_eoi = defaults.iter().any(|name| *name == "EOI"); @@ -197,32 +212,25 @@ fn generate_include(name: &Ident, path: &str) -> TokenStream { } } -fn generate_enum( - rules: &[OptimizedRule], - doc_comment: &DocComment<'_>, - uses_eoi: bool, -) -> TokenStream { +fn generate_enum(rules: &[OptimizedRule], doc_comment: &DocComment, uses_eoi: bool) -> TokenStream { let rules = rules.iter().map(|rule| { let rule_name = format_ident!("r#{}", rule.name); - let comments = doc_comment.line_docs_for_rule(&rule.name); - let comments = comments.unwrap_or_else(|| "".to_owned()); - if comments.is_empty() { - quote! { + match doc_comment.line_docs.get(&rule.name) { + Some(doc) => quote! { + #[doc = #doc] #rule_name - } - } else { - quote! { - #[doc = #comments] + }, + None => quote! { #rule_name - } + }, } }); - let grammar_docs = doc_comment.grammar_docs.join("\n"); + let grammar_doc = &doc_comment.grammar_doc; if uses_eoi { quote! { - #[doc = #grammar_docs] + #[doc = #grammar_doc] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum Rule { @@ -232,7 +240,7 @@ fn generate_enum( } } else { quote! { - #[doc = #grammar_docs] + #[doc = #grammar_doc] #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum Rule { @@ -709,14 +717,12 @@ mod tests { expr: OptimizedExpr::Ident("g".to_owned()), }]; + let mut line_docs = HashMap::new(); + line_docs.insert("f".to_owned(), "This is rule comment".to_owned()); + let doc_comment = &DocComment { - grammar_docs: vec!["Rule doc", "hello"], - line_docs: vec![vec!["This is rule comment"]], - rules: vec![Rule { - name: "f".to_owned(), - ty: RuleType::Normal, - expr: Expr::Ident("g".to_owned()), - }], + grammar_doc: "Rule doc\nhello".to_owned(), + line_docs, }; assert_eq!( @@ -1009,7 +1015,7 @@ mod tests { } #[test] - fn generate_complete() { + fn test_generate_complete() { let name = Ident::new("MyParser", Span::call_site()); let generics = Generics::default(); @@ -1026,21 +1032,12 @@ mod tests { }, ]; + let mut line_docs = HashMap::new(); + line_docs.insert("if".to_owned(), "If statement".to_owned()); + let doc_comment = &DocComment { - line_docs: vec![vec![], vec!["If statement"]], - grammar_docs: vec!["This is Rule doc", "This is second line"], - rules: vec![ - Rule { - name: "a".to_owned(), - ty: RuleType::Silent, - expr: Expr::Str("b".to_owned()), - }, - Rule { - name: "if".to_owned(), - ty: RuleType::Silent, - expr: Expr::Str("b".to_owned()), - }, - ], + line_docs, + grammar_doc: "This is Rule doc\nThis is second line".to_owned(), }; let defaults = vec!["ANY"]; diff --git a/generator/src/lib.rs b/generator/src/lib.rs index 7418a37c..ebb3fdc1 100644 --- a/generator/src/lib.rs +++ b/generator/src/lib.rs @@ -21,6 +21,7 @@ #[macro_use] extern crate quote; +use std::collections::HashMap; use std::env; use std::fs::File; use std::io::{self, Read}; @@ -93,18 +94,14 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { Err(error) => panic!("error parsing \n{}", error.renamed_rules(rename_meta_rule)), }; - let grammar_docs = consume_grammar_doc(pairs.clone()); + let grammar_doc = consume_grammar_doc(pairs.clone()); let line_docs = consume_line_docs(pairs.clone()); let defaults = unwrap_or_report(validator::validate_pairs(pairs.clone())); let ast = unwrap_or_report(parser::consume_rules(pairs)); - let optimized = optimizer::optimize(ast.clone()); + let optimized = optimizer::optimize(ast); - let doc_comment = DocComment { - grammar_docs, - line_docs, - rules: ast, - }; + let doc_comment = &DocComment::new(grammar_doc, line_docs); generator::generate( name, @@ -112,12 +109,13 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { path, optimized, defaults, - &doc_comment, + doc_comment, include_grammar, ) } -fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> Vec<&'_ str> { +/// Consume grammar doc into String, multi-line joined with `\n` +fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> String { let mut docs = vec![]; for pair in pairs { if pair.as_rule() == Rule::grammar_doc { @@ -126,31 +124,50 @@ fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> Vec<&'_ str> { } } - docs + docs.join("\n") } -fn consume_line_docs(pairs: Pairs<'_, Rule>) -> Vec> { - let mut docs = vec![]; +/// Consume line docs into HashMap +/// +/// Example a `test.pest`: +/// +/// ```ignore +/// /// Line doc 1 +/// foo = {} +/// +/// /// Line doc 2 +/// /// Line doc 3 +/// bar = {} +/// ``` +/// +/// Will returns `{ "foo": "This is line comment", "bar": "Line doc 2\n/// Line doc 3" }` +fn consume_line_docs(pairs: Pairs<'_, Rule>) -> HashMap { + let mut docs: HashMap = HashMap::new(); let mut comments = vec![]; for pair in pairs { - if pair.as_rule() == Rule::grammar_rule { + let rule = pair.as_rule(); + + if rule == Rule::grammar_rule { if let Some(inner) = pair.into_inner().next() { - if inner.as_rule() == Rule::line_doc { - let inner_doc = inner.into_inner().next().unwrap(); - comments.push(inner_doc.as_str()); - continue; - } else { - docs.push(comments); - comments = vec![]; + // grammar_rule > line_doc | identifier + match inner.as_rule() { + Rule::line_doc => { + if let Some(inner_doc) = inner.into_inner().next() { + comments.push(inner_doc.as_str()) + } + } + Rule::identifier => { + if !comments.is_empty() { + let rule_name = inner.as_str().to_owned(); + docs.insert(rule_name, comments.join("\n")); + comments = vec![]; + } + } + _ => (), } } } - - if !comments.is_empty() { - docs.push(comments); - comments = vec![]; - } } docs @@ -214,6 +231,8 @@ fn get_attribute(attr: &Attribute) -> GrammarSource { #[cfg(test)] mod tests { + use std::collections::HashMap; + use super::consume_line_docs; use super::parse_derive; use super::GrammarSource; @@ -296,15 +315,18 @@ mod tests { }; let line_docs = consume_line_docs(pairs); - assert_eq!( - vec![ - vec!["Matches foo str, e.g.: `foo`"], - vec!["Matches bar str,", " Indent 2, e.g: `bar` or `foobar`"], - vec![], - vec!["Matches dar", "Match dar description"] - ], - line_docs + + let mut expected = HashMap::new(); + expected.insert("foo".to_owned(), "Matches foo str, e.g.: `foo`".to_owned()); + expected.insert( + "bar".to_owned(), + "Matches bar str,\n Indent 2, e.g: `bar` or `foobar`".to_owned(), + ); + expected.insert( + "dar".to_owned(), + "Matches dar\nMatch dar description".to_owned(), ); + assert_eq!(expected, line_docs); } #[test] diff --git a/grammars/src/grammars/json.pest b/grammars/src/grammars/json.pest index 4d0d300f..413aa7b3 100644 --- a/grammars/src/grammars/json.pest +++ b/grammars/src/grammars/json.pest @@ -8,6 +8,7 @@ // modified, or distributed except according to those terms. //! A parser for JSON file. +//! //! And this is a example for JSON parser. json = { SOI ~ (object | array) ~ EOI } From 279e18d2d021524f30474f99fb07079ceda54ae0 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Wed, 18 Jan 2023 19:14:42 +0800 Subject: [PATCH 5/5] Move DocComment methods into docs.rs --- generator/src/docs.rs | 122 +++++++++++++++++++++++++++++++++++++ generator/src/generator.rs | 35 ++--------- generator/src/lib.rs | 99 +----------------------------- 3 files changed, 129 insertions(+), 127 deletions(-) create mode 100644 generator/src/docs.rs diff --git a/generator/src/docs.rs b/generator/src/docs.rs new file mode 100644 index 00000000..8660e538 --- /dev/null +++ b/generator/src/docs.rs @@ -0,0 +1,122 @@ +use pest::iterators::Pairs; +use pest_meta::parser::Rule; +use std::collections::HashMap; + +#[derive(Debug)] +pub(crate) struct DocComment { + pub grammar_doc: String, + + /// HashMap for store all doc_comments for rules. + /// key is rule name, value is doc_comment. + pub line_docs: HashMap, +} + +/// Consume pairs to matches `Rule::grammar_doc`, `Rule::line_doc` into `DocComment` +/// +/// e.g. +/// +/// a pest file: +/// +/// ```ignore +/// //! This is a grammar doc +/// /// line doc 1 +/// /// line doc 2 +/// foo = {} +/// +/// /// line doc 3 +/// bar = {} +/// ``` +/// +/// Then will get: +/// +/// ```ignore +/// grammar_doc = "This is a grammar doc" +/// line_docs = { "foo": "line doc 1\nline doc 2", "bar": "line doc 3" } +/// ``` +pub(crate) fn consume(pairs: Pairs<'_, Rule>) -> DocComment { + let mut grammar_doc = String::new(); + + let mut line_docs: HashMap = HashMap::new(); + let mut line_doc = String::new(); + + for pair in pairs { + match pair.as_rule() { + Rule::grammar_doc => { + // grammar_doc > inner_doc + let inner_doc = pair.into_inner().next().unwrap(); + grammar_doc.push_str(inner_doc.as_str()); + grammar_doc.push('\n'); + } + Rule::grammar_rule => { + if let Some(inner) = pair.into_inner().next() { + // grammar_rule > line_doc | identifier + match inner.as_rule() { + Rule::line_doc => { + if let Some(inner_doc) = inner.into_inner().next() { + line_doc.push_str(inner_doc.as_str()); + line_doc.push('\n'); + } + } + Rule::identifier => { + if !line_doc.is_empty() { + let rule_name = inner.as_str().to_owned(); + + // Remove last \n + line_doc.pop(); + line_docs.insert(rule_name, line_doc.clone()); + line_doc.clear(); + } + } + _ => (), + } + } + } + _ => (), + } + } + + if !grammar_doc.is_empty() { + // Remove last \n + grammar_doc.pop(); + } + + DocComment { + grammar_doc, + line_docs, + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use pest_meta::parser; + use pest_meta::parser::Rule; + + #[test] + fn test_doc_comment() { + let pairs = match parser::parse(Rule::grammar_rules, include_str!("../tests/test.pest")) { + Ok(pairs) => pairs, + Err(_) => panic!("error parsing tests/test.pest"), + }; + + let doc_comment = super::consume(pairs); + + let mut expected = HashMap::new(); + expected.insert("foo".to_owned(), "Matches foo str, e.g.: `foo`".to_owned()); + expected.insert( + "bar".to_owned(), + "Matches bar str,\n Indent 2, e.g: `bar` or `foobar`".to_owned(), + ); + expected.insert( + "dar".to_owned(), + "Matches dar\nMatch dar description".to_owned(), + ); + assert_eq!(expected, doc_comment.line_docs); + + assert_eq!( + "A parser for JSON file.\nAnd this is a example for JSON parser.\n\n indent-4-space", + doc_comment.grammar_doc + ); + } +} diff --git a/generator/src/generator.rs b/generator/src/generator.rs index 2a1aa87d..87d1f00d 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -7,7 +7,6 @@ // option. All files in the project carrying such notice may not be copied, // modified, or distributed except according to those terms. -use std::collections::HashMap; use std::path::PathBuf; use proc_macro2::TokenStream; @@ -18,34 +17,7 @@ use pest::unicode::unicode_property_names; use pest_meta::ast::*; use pest_meta::optimizer::*; -#[derive(Debug)] -pub(crate) struct DocComment { - /// Multi-line grammar doc, (joined with `\n`) - /// - /// e.g. - /// - /// ```ignore - /// "grammar doc 1\ngrammar doc 2" - /// ``` - grammar_doc: String, - /// HashMap rule name and doc comments (joined with `\n`) - /// - /// e.g. - /// - /// ```ignore - /// { "foo": "line doc 1\nline doc 2", "bar": "line doc 3" } - /// ``` - line_docs: HashMap, -} - -impl DocComment { - pub fn new(grammar_doc: String, line_docs: HashMap) -> Self { - Self { - grammar_doc, - line_docs, - } - } -} +use crate::docs::DocComment; pub(crate) fn generate( name: Ident, @@ -705,10 +677,11 @@ fn option_type() -> TokenStream { #[cfg(test)] mod tests { - use proc_macro2::Span; - use super::*; + use proc_macro2::Span; + use std::collections::HashMap; + #[test] fn rule_enum_simple() { let rules = vec![OptimizedRule { diff --git a/generator/src/lib.rs b/generator/src/lib.rs index ebb3fdc1..f9c118f8 100644 --- a/generator/src/lib.rs +++ b/generator/src/lib.rs @@ -21,25 +21,22 @@ #[macro_use] extern crate quote; -use std::collections::HashMap; use std::env; use std::fs::File; use std::io::{self, Read}; use std::path::Path; -use pest::iterators::Pairs; use proc_macro2::TokenStream; use syn::{Attribute, DeriveInput, Generics, Ident, Lit, Meta}; #[macro_use] mod macros; +mod docs; mod generator; use pest_meta::parser::{self, rename_meta_rule, Rule}; use pest_meta::{optimizer, unwrap_or_report, validator}; -use generator::DocComment; - /// Processes the derive/proc macro input and generates the corresponding parser based /// on the parsed grammar. If `include_grammar` is set to true, it'll generate an explicit /// "include_str" statement (done in pest_derive, but turned off in the local bootstrap). @@ -94,85 +91,22 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { Err(error) => panic!("error parsing \n{}", error.renamed_rules(rename_meta_rule)), }; - let grammar_doc = consume_grammar_doc(pairs.clone()); - let line_docs = consume_line_docs(pairs.clone()); - let defaults = unwrap_or_report(validator::validate_pairs(pairs.clone())); + let doc_comment = docs::consume(pairs.clone()); let ast = unwrap_or_report(parser::consume_rules(pairs)); let optimized = optimizer::optimize(ast); - let doc_comment = &DocComment::new(grammar_doc, line_docs); - generator::generate( name, &generics, path, optimized, defaults, - doc_comment, + &doc_comment, include_grammar, ) } -/// Consume grammar doc into String, multi-line joined with `\n` -fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> String { - let mut docs = vec![]; - for pair in pairs { - if pair.as_rule() == Rule::grammar_doc { - let inner_doc = pair.into_inner().next().unwrap(); - docs.push(inner_doc.as_str()); - } - } - - docs.join("\n") -} - -/// Consume line docs into HashMap -/// -/// Example a `test.pest`: -/// -/// ```ignore -/// /// Line doc 1 -/// foo = {} -/// -/// /// Line doc 2 -/// /// Line doc 3 -/// bar = {} -/// ``` -/// -/// Will returns `{ "foo": "This is line comment", "bar": "Line doc 2\n/// Line doc 3" }` -fn consume_line_docs(pairs: Pairs<'_, Rule>) -> HashMap { - let mut docs: HashMap = HashMap::new(); - let mut comments = vec![]; - - for pair in pairs { - let rule = pair.as_rule(); - - if rule == Rule::grammar_rule { - if let Some(inner) = pair.into_inner().next() { - // grammar_rule > line_doc | identifier - match inner.as_rule() { - Rule::line_doc => { - if let Some(inner_doc) = inner.into_inner().next() { - comments.push(inner_doc.as_str()) - } - } - Rule::identifier => { - if !comments.is_empty() { - let rule_name = inner.as_str().to_owned(); - docs.insert(rule_name, comments.join("\n")); - comments = vec![]; - } - } - _ => (), - } - } - } - } - - docs -} - fn read_file>(path: P) -> io::Result { let mut file = File::open(path.as_ref())?; let mut string = String::new(); @@ -231,14 +165,9 @@ fn get_attribute(attr: &Attribute) -> GrammarSource { #[cfg(test)] mod tests { - use std::collections::HashMap; - - use super::consume_line_docs; use super::parse_derive; use super::GrammarSource; - use pest_meta::parser::{self, Rule}; - #[test] fn derive_inline_file() { let definition = " @@ -307,28 +236,6 @@ mod tests { parse_derive(ast); } - #[test] - fn test_consume_line_docs() { - let pairs = match parser::parse(Rule::grammar_rules, include_str!("../tests/test.pest")) { - Ok(pairs) => pairs, - Err(_) => panic!("error parsing tests/test.pest"), - }; - - let line_docs = consume_line_docs(pairs); - - let mut expected = HashMap::new(); - expected.insert("foo".to_owned(), "Matches foo str, e.g.: `foo`".to_owned()); - expected.insert( - "bar".to_owned(), - "Matches bar str,\n Indent 2, e.g: `bar` or `foobar`".to_owned(), - ); - expected.insert( - "dar".to_owned(), - "Matches dar\nMatch dar description".to_owned(), - ); - assert_eq!(expected, line_docs); - } - #[test] fn test_generate_doc() { let input = quote! {