Improve grammar_doc, line_doc parse for keep whitespaces in head and …

…tail.
pest-parser · Jan 18, 2023 · 9043679 · 9043679
1 parent 34385e0
commit 9043679
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 10 deletions.
diff --git a/generator/src/lib.rs b/generator/src/lib.rs
@@ -121,7 +121,8 @@ fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> Vec<&'_ str> {
     let mut docs = vec![];
     for pair in pairs {
         if pair.as_rule() == Rule::grammar_doc {
-            docs.push(pair.as_str()[3..pair.as_str().len()].trim());
+            let inner_doc = pair.into_inner().next().unwrap();
+            docs.push(inner_doc.as_str());
         }
     }
 
@@ -136,7 +137,8 @@ fn consume_line_docs(pairs: Pairs<'_, Rule>) -> Vec<Vec<&'_ str>> {
         if pair.as_rule() == Rule::grammar_rule {
             if let Some(inner) = pair.into_inner().next() {
                 if inner.as_rule() == Rule::line_doc {
-                    comments.push(inner.as_str()[3..inner.as_str().len()].trim());
+                    let inner_doc = inner.into_inner().next().unwrap();
+                    comments.push(inner_doc.as_str());
                     continue;
                 } else {
                     docs.push(comments);
@@ -297,7 +299,7 @@ mod tests {
         assert_eq!(
             vec![
                 vec!["Matches foo str, e.g.: `foo`"],
-                vec!["Matches bar str,", "e.g: `bar` or `foobar`"],
+                vec!["Matches bar str,", "  Indent 2, e.g: `bar` or `foobar`"],
                 vec![],
                 vec!["Matches dar", "Match dar description"]
             ],
@@ -316,14 +318,14 @@ mod tests {
         let token = super::derive_parser(input, true);
 
         let expected = quote! {
-            #[doc = "A parser for JSON file.\nAnd this is a example for JSON parser."]
+            #[doc = "A parser for JSON file.\nAnd this is a example for JSON parser.\n\n    indent-4-space"]
             #[allow(dead_code, non_camel_case_types, clippy::upper_case_acronyms)]
             #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
 
             pub enum Rule {
                 #[doc = "Matches foo str, e.g.: `foo`"]
                 r#foo,
-                #[doc = "Matches bar str,\ne.g: `bar` or `foobar`"]
+                #[doc = "Matches bar str,\n  Indent 2, e.g: `bar` or `foobar`"]
                 r#bar,
                 r#bar1,
                 #[doc = "Matches dar\nMatch dar description"]

diff --git a/generator/tests/test.pest b/generator/tests/test.pest
@@ -1,11 +1,13 @@
 //! A parser for JSON file.
 //! And this is a example for JSON parser.
+//!
+//!     indent-4-space
 
 /// Matches foo str, e.g.: `foo`
 foo = { "foo" }
 
 /// Matches bar str,
-/// e.g: `bar` or `foobar`
+///   Indent 2, e.g: `bar` or `foobar`
 
 bar = { "bar" | "foobar" }
 

diff --git a/meta/src/grammar.pest b/meta/src/grammar.pest
@@ -100,5 +100,7 @@ block_comment  = _{ "/*" ~ (block_comment | !"*/" ~ ANY)* ~ "*/" }
 COMMENT        = _{ block_comment | line_comment }
 
 // ref: https://doc.rust-lang.org/reference/comments.html
-grammar_doc = ${ "//!" ~ (!newline ~ ANY)* }
-line_doc    = ${ "///" ~ !"/" ~ (!newline ~ ANY)* }
+space       = _{ " " | "\t" }
+grammar_doc = ${ "//!" ~ space? ~ inner_doc }
+line_doc    = ${ "///" ~ space? ~ !"/" ~ inner_doc }
+inner_doc   = @{ (!newline ~ ANY)* }
diff --git a/meta/src/parser.rs b/meta/src/parser.rs
@@ -1110,9 +1110,13 @@ mod tests {
             input: input,
             rule: Rule::grammar_rules,
             tokens: [
-                grammar_doc(0, 9),
+                grammar_doc(0, 9, [
+                    inner_doc(4, 9),
+                ]),
                 grammar_rule(10, 19, [
-                    line_doc(10, 19),
+                    line_doc(10, 19, [
+                        inner_doc(14, 19),
+                    ]),
                 ]),
                 grammar_rule(20, 31, [
                     identifier(20, 21),