Rewrite line_doc extract in pest_generator, avoid change AST.

generator/src/generator.rs

@@ -17,13 +17,28 @@ use pest::unicode::unicode_property_names;
 use pest_meta::ast::*;
 use pest_meta::optimizer::*;
 
-pub fn generate(
+#[derive(Debug)]
+pub(crate) struct DocComment<'a> {
+    pub(crate) grammar_docs: Vec<&'a str>,
+    pub(crate) line_docs: Vec<Vec<&'a str>>,
+    pub(crate) rules: Vec<Rule>,
+}
+
+impl DocComment<'_> {
+    fn line_docs_for_rule(&self, rule_name: &str) -> Option<String> {
+        let idx = self.rules.iter().position(|r| r.name == rule_name)?;
+
+        self.line_docs.get(idx).map(|comments| comments.join("\n"))
+    }
+}
+
+pub(crate) fn generate(
     name: Ident,
     generics: &Generics,
     path: Option<PathBuf>,
     rules: Vec<OptimizedRule>,
     defaults: Vec<&str>,
-    grammar_docs: Vec<&str>,
+    doc_comment: &DocComment<'_>,
     include_grammar: bool,
 ) -> TokenStream {
     let uses_eoi = defaults.iter().any(|name| *name == "EOI");
@@ -37,7 +52,7 @@ pub fn generate(
     } else {
         quote!()
     };
-    let rule_enum = generate_enum(&rules, grammar_docs, uses_eoi);
+    let rule_enum = generate_enum(&rules, doc_comment, uses_eoi);
     let patterns = generate_patterns(&rules, uses_eoi);
     let skip = generate_skip(&rules);
 
@@ -182,23 +197,29 @@ fn generate_include(name: &Ident, path: &str) -> TokenStream {
     }
 }
 
-fn generate_enum(rules: &[OptimizedRule], grammar_docs: Vec<&str>, uses_eoi: bool) -> TokenStream {
+fn generate_enum(
+    rules: &[OptimizedRule],
+    doc_comment: &DocComment<'_>,
+    uses_eoi: bool,
+) -> TokenStream {
     let rules = rules.iter().map(|rule| {
         let rule_name = format_ident!("r#{}", rule.name);
-        if rule.comments.is_empty() {
+
+        let comments = doc_comment.line_docs_for_rule(&rule.name);
+        let comments = comments.unwrap_or_else(|| "".to_owned());
+        if comments.is_empty() {
             quote! {
                 #rule_name
             }
         } else {
-            let comments = rule.comments.join("\n");
             quote! {
                 #[doc = #comments]
                 #rule_name
             }
         }
     });
 
-    let grammar_docs = grammar_docs.join("\n");
+    let grammar_docs = doc_comment.grammar_docs.join("\n");
     if uses_eoi {
         quote! {
             #[doc = #grammar_docs]
@@ -686,11 +707,20 @@ mod tests {
             name: "f".to_owned(),
             ty: RuleType::Normal,
             expr: OptimizedExpr::Ident("g".to_owned()),
-            comments: vec!["This is rule comment".to_owned()],
         }];
 
+        let doc_comment = &DocComment {
+            grammar_docs: vec!["Rule doc", "hello"],
+            line_docs: vec![vec!["This is rule comment"]],
+            rules: vec![Rule {
+                name: "f".to_owned(),
+                ty: RuleType::Normal,
+                expr: Expr::Ident("g".to_owned()),
+            }],
+        };
+
         assert_eq!(
-            generate_enum(&rules, vec!["Rule doc", "hello"], false).to_string(),
+            generate_enum(&rules, doc_comment, false).to_string(),
             quote! {
                 #[doc = "Rule doc\nhello"]
                 #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
@@ -988,24 +1018,39 @@ mod tests {
                 name: "a".to_owned(),
                 ty: RuleType::Silent,
                 expr: OptimizedExpr::Str("b".to_owned()),
-                comments: vec![],
             },
             OptimizedRule {
                 name: "if".to_owned(),
                 ty: RuleType::Silent,
                 expr: OptimizedExpr::Ident("a".to_owned()),
-                comments: vec!["If statement".to_owned()],
             },
         ];
 
+        let doc_comment = &DocComment {
+            line_docs: vec![vec![], vec!["If statement"]],
+            grammar_docs: vec!["This is Rule doc", "This is second line"],
+            rules: vec![
+                Rule {
+                    name: "a".to_owned(),
+                    ty: RuleType::Silent,
+                    expr: Expr::Str("b".to_owned()),
+                },
+                Rule {
+                    name: "if".to_owned(),
+                    ty: RuleType::Silent,
+                    expr: Expr::Str("b".to_owned()),
+                },
+            ],
+        };
+
         let defaults = vec!["ANY"];
         let result = result_type();
         let box_ty = box_type();
         let mut current_dir = std::env::current_dir().expect("Unable to get current directory");
         current_dir.push("test.pest");
         let test_path = current_dir.to_str().expect("path contains invalid unicode");
         assert_eq!(
-            generate(name, &generics, Some(PathBuf::from("test.pest")), rules, defaults, vec!["This is Rule doc", "This is second line"], true).to_string(),
+            generate(name, &generics, Some(PathBuf::from("test.pest")), rules, defaults, doc_comment, true).to_string(),
             quote! {
                 #[allow(non_upper_case_globals)]
                 const _PEST_GRAMMAR_MyParser: &'static str = include_str!(#test_path);

generator/src/lib.rs

@@ -37,6 +37,8 @@ mod generator;
 use pest_meta::parser::{self, rename_meta_rule, Rule};
 use pest_meta::{optimizer, unwrap_or_report, validator};
 
+use generator::DocComment;
+
 /// Processes the derive/proc macro input and generates the corresponding parser based
 /// on the parsed grammar. If `include_grammar` is set to true, it'll generate an explicit
 /// "include_str" statement (done in pest_derive, but turned off in the local bootstrap).
@@ -92,18 +94,25 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream {
     };
 
     let grammar_docs = consume_grammar_doc(pairs.clone());
+    let line_docs = consume_line_docs(pairs.clone());
 
     let defaults = unwrap_or_report(validator::validate_pairs(pairs.clone()));
     let ast = unwrap_or_report(parser::consume_rules(pairs));
-    let optimized = optimizer::optimize(ast);
+    let optimized = optimizer::optimize(ast.clone());
+
+    let doc_comment = DocComment {
+        grammar_docs,
+        line_docs,
+        rules: ast,
+    };
 
     generator::generate(
         name,
         &generics,
         path,
         optimized,
         defaults,
-        grammar_docs,
+        &doc_comment,
         include_grammar,
     )
 }
@@ -119,6 +128,32 @@ fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> Vec<&'_ str> {
     docs
 }
 
+fn consume_line_docs(pairs: Pairs<'_, Rule>) -> Vec<Vec<&'_ str>> {
+    let mut docs = vec![];
+    let mut comments = vec![];
+
+    for pair in pairs {
+        if pair.as_rule() == Rule::grammar_rule {
+            if let Some(inner) = pair.into_inner().next() {
+                if inner.as_rule() == Rule::line_doc {
+                    comments.push(inner.as_str()[3..inner.as_str().len()].trim());
+                    continue;
+                } else {
+                    docs.push(comments);
+                    comments = vec![];
+                }
+            }
+        }
+
+        if !comments.is_empty() {
+            docs.push(comments);
+            comments = vec![];
+        }
+    }
+
+    docs
+}
+
 fn read_file<P: AsRef<Path>>(path: P) -> io::Result<String> {
     let mut file = File::open(path.as_ref())?;
     let mut string = String::new();
@@ -177,9 +212,12 @@ fn get_attribute(attr: &Attribute) -> GrammarSource {
 
 #[cfg(test)]
 mod tests {
+    use super::consume_line_docs;
     use super::parse_derive;
     use super::GrammarSource;
 
+    use pest_meta::parser::{self, Rule};
+
     #[test]
     fn derive_inline_file() {
         let definition = "
@@ -247,4 +285,57 @@ mod tests {
         let ast = syn::parse_str(definition).unwrap();
         parse_derive(ast);
     }
+
+    #[test]
+    fn test_consume_line_docs() {
+        let pairs = match parser::parse(Rule::grammar_rules, include_str!("../tests/test.pest")) {
+            Ok(pairs) => pairs,
+            Err(_) => panic!("error parsing tests/test.pest"),
+        };
+
+        let line_docs = consume_line_docs(pairs);
+        assert_eq!(
+            vec![
+                vec!["Matches foo str, e.g.: `foo`"],
+                vec!["Matches bar str,", "e.g: `bar` or `foobar`"],
+                vec![],
+                vec!["Matches dar", "Match dar description"]
+            ],
+            line_docs
+        );
+    }
+
+    #[test]
+    fn test_generate_doc() {
+        let input = quote! {
+            #[derive(Parser)]
+            #[grammar = "../tests/test.pest"]
+            pub struct TestParser;
+        };
+
+        let token = super::derive_parser(input, true);
+
+        let expected = quote! {
+            #[doc = "A parser for JSON file.\nAnd this is a example for JSON parser."]
+            #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
+            #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
+
+            pub enum Rule {
+                #[doc = "Matches foo str, e.g.: `foo`"]
+                r#foo,
+                #[doc = "Matches bar str,\ne.g: `bar` or `foobar`"]
+                r#bar,
+                r#bar1,
+                #[doc = "Matches dar\nMatch dar description"]
+                r#dar
+            }
+        };
+
+        assert!(
+            token.to_string().contains(expected.to_string().as_str()),
+            "{}\n\nExpected to contains:\n{}",
+            token,
+            expected
+        );
+    }
 }

generator/tests/test.pest

@@ -0,0 +1,18 @@
+//! A parser for JSON file.
+//! And this is a example for JSON parser.
+
+/// Matches foo str, e.g.: `foo`
+foo = { "foo" }
+
+/// Matches bar str,
+/// e.g: `bar` or `foobar`
+
+bar = { "bar" | "foobar" }
+
+bar1 = { "bar1" }
+
+/// Matches dar
+
+/// Match dar description
+
+dar = { "da" }

grammars/src/grammars/json.pest

@@ -12,6 +12,7 @@
 json = { SOI ~ (object | array) ~ EOI }
 
 /// Matches object, e.g.: `{ "foo": "bar" }`
+/// Foobar
 object = { "{" ~ pair ~ ("," ~ pair)* ~ "}" | "{" ~ "}" }
 pair   = { string ~ ":" ~ value }
 

meta/src/ast.rs

@@ -10,7 +10,6 @@
 //! Types for the pest's abstract syntax tree.
 
 /// A grammar rule
-#[non_exhaustive]
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Rule {
     /// The name of the rule
@@ -19,8 +18,6 @@ pub struct Rule {
     pub ty: RuleType,
     /// The rule's expression
     pub expr: Expr,
-    /// Doc comments of the rule
-    pub(crate) comments: Vec<String>,
 }
 
 /// All possible rule types

meta/src/optimizer/concatenator.rs

@@ -10,12 +10,7 @@
 use crate::ast::*;
 
 pub fn concatenate(rule: Rule) -> Rule {
-    let Rule {
-        name,
-        ty,
-        expr,
-        comments,
-    } = rule;
+    let Rule { name, ty, expr } = rule;
     Rule {
         name,
         ty,
@@ -34,6 +29,5 @@ pub fn concatenate(rule: Rule) -> Rule {
                 expr
             }
         }),
-        comments,
     }
 }

meta/src/optimizer/factorizer.rs

@@ -10,12 +10,7 @@
 use crate::ast::*;
 
 pub fn factor(rule: Rule) -> Rule {
-    let Rule {
-        name,
-        ty,
-        expr,
-        comments,
-    } = rule;
+    let Rule { name, ty, expr } = rule;
     Rule {
         name,
         ty,
@@ -52,6 +47,5 @@ pub fn factor(rule: Rule) -> Rule {
                 expr => expr,
             }
         }),
-        comments,
     }
 }

meta/src/optimizer/lister.rs

@@ -10,12 +10,7 @@
 use crate::ast::*;
 
 pub fn list(rule: Rule) -> Rule {
-    let Rule {
-        name,
-        ty,
-        expr,
-        comments,
-    } = rule;
+    let Rule { name, ty, expr } = rule;
     Rule {
         name,
         ty,
@@ -43,6 +38,5 @@ pub fn list(rule: Rule) -> Rule {
                 expr => expr,
             }
         }),
-        comments,
     }
 }