From 45722326feadc10d687e203db50bd1e38462eaa9 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Wed, 18 Jan 2023 18:19:22 +0800 Subject: [PATCH] Move DocComment methods into docs.rs --- generator/src/docs.rs | 127 +++++++++++++++++++++++++++++++++++++ generator/src/generator.rs | 35 ++-------- generator/src/lib.rs | 99 +---------------------------- 3 files changed, 134 insertions(+), 127 deletions(-) create mode 100644 generator/src/docs.rs diff --git a/generator/src/docs.rs b/generator/src/docs.rs new file mode 100644 index 00000000..b2d19177 --- /dev/null +++ b/generator/src/docs.rs @@ -0,0 +1,127 @@ +use pest::iterators::Pairs; +use pest_meta::parser::Rule; +use std::collections::HashMap; + +#[derive(Debug)] +pub(crate) struct DocComment { + /// Multi-line grammar doc, (joined with `\n`) + /// + /// e.g. + /// + /// ```ignore + /// "grammar doc 1\ngrammar doc 2" + /// ``` + pub grammar_doc: String, + /// HashMap rule name and doc comments (joined with `\n`) + /// + /// e.g. + /// + /// ```ignore + /// { "foo": "line doc 1\nline doc 2", "bar": "line doc 3" } + /// ``` + pub line_docs: HashMap, +} + +pub(crate) fn consume(pairs: Pairs<'_, Rule>) -> DocComment { + let grammar_doc = consume_grammar_doc(pairs.clone()); + let line_docs = consume_line_docs(pairs); + + DocComment { + grammar_doc, + line_docs, + } +} + +/// Consume grammar doc into String, multi-line joined with `\n` +fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> String { + let mut docs = vec![]; + for pair in pairs { + if pair.as_rule() == Rule::grammar_doc { + let inner_doc = pair.into_inner().next().unwrap(); + docs.push(inner_doc.as_str()); + } + } + + docs.join("\n") +} + +/// Consume line docs into HashMap +/// +/// Example a `test.pest`: +/// +/// ```ignore +/// /// Line doc 1 +/// foo = {} +/// +/// /// Line doc 2 +/// /// Line doc 3 +/// bar = {} +/// ``` +/// +/// Will returns `{ "foo": "This is line comment", "bar": "Line doc 2\n/// Line doc 3" }` +fn consume_line_docs(pairs: Pairs<'_, Rule>) -> HashMap { + let mut docs: HashMap = HashMap::new(); + let mut comments = vec![]; + + for pair in pairs { + let rule = pair.as_rule(); + + if rule == Rule::grammar_rule { + if let Some(inner) = pair.into_inner().next() { + // grammar_rule > line_doc | identifier + match inner.as_rule() { + Rule::line_doc => { + if let Some(inner_doc) = inner.into_inner().next() { + comments.push(inner_doc.as_str()) + } + } + Rule::identifier => { + if !comments.is_empty() { + let rule_name = inner.as_str().to_owned(); + docs.insert(rule_name, comments.join("\n")); + comments = vec![]; + } + } + _ => (), + } + } + } + } + + docs +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use pest_meta::parser; + use pest_meta::parser::Rule; + + #[test] + fn test_doc_comment() { + let pairs = match parser::parse(Rule::grammar_rules, include_str!("../tests/test.pest")) { + Ok(pairs) => pairs, + Err(_) => panic!("error parsing tests/test.pest"), + }; + + let doc_comment = super::consume(pairs); + + let mut expected = HashMap::new(); + expected.insert("foo".to_owned(), "Matches foo str, e.g.: `foo`".to_owned()); + expected.insert( + "bar".to_owned(), + "Matches bar str,\n Indent 2, e.g: `bar` or `foobar`".to_owned(), + ); + expected.insert( + "dar".to_owned(), + "Matches dar\nMatch dar description".to_owned(), + ); + assert_eq!(expected, doc_comment.line_docs); + + assert_eq!( + "A parser for JSON file.\nAnd this is a example for JSON parser.\n\n indent-4-space", + doc_comment.grammar_doc + ); + } +} diff --git a/generator/src/generator.rs b/generator/src/generator.rs index 2a1aa87d..87d1f00d 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -7,7 +7,6 @@ // option. All files in the project carrying such notice may not be copied, // modified, or distributed except according to those terms. -use std::collections::HashMap; use std::path::PathBuf; use proc_macro2::TokenStream; @@ -18,34 +17,7 @@ use pest::unicode::unicode_property_names; use pest_meta::ast::*; use pest_meta::optimizer::*; -#[derive(Debug)] -pub(crate) struct DocComment { - /// Multi-line grammar doc, (joined with `\n`) - /// - /// e.g. - /// - /// ```ignore - /// "grammar doc 1\ngrammar doc 2" - /// ``` - grammar_doc: String, - /// HashMap rule name and doc comments (joined with `\n`) - /// - /// e.g. - /// - /// ```ignore - /// { "foo": "line doc 1\nline doc 2", "bar": "line doc 3" } - /// ``` - line_docs: HashMap, -} - -impl DocComment { - pub fn new(grammar_doc: String, line_docs: HashMap) -> Self { - Self { - grammar_doc, - line_docs, - } - } -} +use crate::docs::DocComment; pub(crate) fn generate( name: Ident, @@ -705,10 +677,11 @@ fn option_type() -> TokenStream { #[cfg(test)] mod tests { - use proc_macro2::Span; - use super::*; + use proc_macro2::Span; + use std::collections::HashMap; + #[test] fn rule_enum_simple() { let rules = vec![OptimizedRule { diff --git a/generator/src/lib.rs b/generator/src/lib.rs index ebb3fdc1..f9c118f8 100644 --- a/generator/src/lib.rs +++ b/generator/src/lib.rs @@ -21,25 +21,22 @@ #[macro_use] extern crate quote; -use std::collections::HashMap; use std::env; use std::fs::File; use std::io::{self, Read}; use std::path::Path; -use pest::iterators::Pairs; use proc_macro2::TokenStream; use syn::{Attribute, DeriveInput, Generics, Ident, Lit, Meta}; #[macro_use] mod macros; +mod docs; mod generator; use pest_meta::parser::{self, rename_meta_rule, Rule}; use pest_meta::{optimizer, unwrap_or_report, validator}; -use generator::DocComment; - /// Processes the derive/proc macro input and generates the corresponding parser based /// on the parsed grammar. If `include_grammar` is set to true, it'll generate an explicit /// "include_str" statement (done in pest_derive, but turned off in the local bootstrap). @@ -94,85 +91,22 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream { Err(error) => panic!("error parsing \n{}", error.renamed_rules(rename_meta_rule)), }; - let grammar_doc = consume_grammar_doc(pairs.clone()); - let line_docs = consume_line_docs(pairs.clone()); - let defaults = unwrap_or_report(validator::validate_pairs(pairs.clone())); + let doc_comment = docs::consume(pairs.clone()); let ast = unwrap_or_report(parser::consume_rules(pairs)); let optimized = optimizer::optimize(ast); - let doc_comment = &DocComment::new(grammar_doc, line_docs); - generator::generate( name, &generics, path, optimized, defaults, - doc_comment, + &doc_comment, include_grammar, ) } -/// Consume grammar doc into String, multi-line joined with `\n` -fn consume_grammar_doc(pairs: Pairs<'_, Rule>) -> String { - let mut docs = vec![]; - for pair in pairs { - if pair.as_rule() == Rule::grammar_doc { - let inner_doc = pair.into_inner().next().unwrap(); - docs.push(inner_doc.as_str()); - } - } - - docs.join("\n") -} - -/// Consume line docs into HashMap -/// -/// Example a `test.pest`: -/// -/// ```ignore -/// /// Line doc 1 -/// foo = {} -/// -/// /// Line doc 2 -/// /// Line doc 3 -/// bar = {} -/// ``` -/// -/// Will returns `{ "foo": "This is line comment", "bar": "Line doc 2\n/// Line doc 3" }` -fn consume_line_docs(pairs: Pairs<'_, Rule>) -> HashMap { - let mut docs: HashMap = HashMap::new(); - let mut comments = vec![]; - - for pair in pairs { - let rule = pair.as_rule(); - - if rule == Rule::grammar_rule { - if let Some(inner) = pair.into_inner().next() { - // grammar_rule > line_doc | identifier - match inner.as_rule() { - Rule::line_doc => { - if let Some(inner_doc) = inner.into_inner().next() { - comments.push(inner_doc.as_str()) - } - } - Rule::identifier => { - if !comments.is_empty() { - let rule_name = inner.as_str().to_owned(); - docs.insert(rule_name, comments.join("\n")); - comments = vec![]; - } - } - _ => (), - } - } - } - } - - docs -} - fn read_file>(path: P) -> io::Result { let mut file = File::open(path.as_ref())?; let mut string = String::new(); @@ -231,14 +165,9 @@ fn get_attribute(attr: &Attribute) -> GrammarSource { #[cfg(test)] mod tests { - use std::collections::HashMap; - - use super::consume_line_docs; use super::parse_derive; use super::GrammarSource; - use pest_meta::parser::{self, Rule}; - #[test] fn derive_inline_file() { let definition = " @@ -307,28 +236,6 @@ mod tests { parse_derive(ast); } - #[test] - fn test_consume_line_docs() { - let pairs = match parser::parse(Rule::grammar_rules, include_str!("../tests/test.pest")) { - Ok(pairs) => pairs, - Err(_) => panic!("error parsing tests/test.pest"), - }; - - let line_docs = consume_line_docs(pairs); - - let mut expected = HashMap::new(); - expected.insert("foo".to_owned(), "Matches foo str, e.g.: `foo`".to_owned()); - expected.insert( - "bar".to_owned(), - "Matches bar str,\n Indent 2, e.g: `bar` or `foobar`".to_owned(), - ); - expected.insert( - "dar".to_owned(), - "Matches dar\nMatch dar description".to_owned(), - ); - assert_eq!(expected, line_docs); - } - #[test] fn test_generate_doc() { let input = quote! {