From d81d057cb5c07d5f916caab1ac284f9f1daef731 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 28 Oct 2019 14:40:49 +0100 Subject: [PATCH 1/3] Update pulldown-cmark version --- Cargo.lock | 2 +- src/librustdoc/Cargo.toml | 2 +- src/librustdoc/html/markdown.rs | 92 +++++++------------ .../passes/check_code_block_syntax.rs | 1 - 4 files changed, 37 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f135a94ef7d77..5649ae5a13083 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3902,7 +3902,7 @@ name = "rustdoc" version = "0.0.0" dependencies = [ "minifier", - "pulldown-cmark 0.5.3", + "pulldown-cmark 0.6.1", "rustc-rayon 0.3.0", "tempfile", ] diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index e3de7fe20493e..ea1a86cc3c98f 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -9,7 +9,7 @@ name = "rustdoc" path = "lib.rs" [dependencies] -pulldown-cmark = { version = "0.5.3", default-features = false } +pulldown-cmark = { version = "0.6.1", default-features = false } minifier = "0.0.33" rayon = { version = "0.3.0", package = "rustc-rayon" } tempfile = "3" diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index 9ff1e1d31197d..f35ea0f9c62ca 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -368,11 +368,11 @@ impl<'a, 'b, 'ids, I: Iterator>> Iterator for HeadingLinks<'a, } let event = self.inner.next(); - if let Some(Event::Start(Tag::Header(level))) = event { + if let Some(Event::Start(Tag::Heading(level))) = event { let mut id = String::new(); for event in &mut self.inner { match &event { - Event::End(Tag::Header(..)) => break, + Event::End(Tag::Heading(..)) => break, Event::Text(text) | Event::Code(text) => { id.extend(text.chars().filter_map(slugify)); } @@ -386,16 +386,16 @@ impl<'a, 'b, 'ids, I: Iterator>> Iterator for HeadingLinks<'a, let mut html_header = String::new(); html::push_html(&mut html_header, self.buf.iter().cloned()); let sec = builder.push(level as u32, html_header, id.clone()); - self.buf.push_front(Event::InlineHtml(format!("{} ", sec).into())); + self.buf.push_front(Event::Html(format!("{} ", sec).into())); } - self.buf.push_back(Event::InlineHtml(format!("", level).into())); + self.buf.push_back(Event::Html(format!("", level).into())); let start_tags = format!("\ ", id = id, level = level); - return Some(Event::InlineHtml(start_tags.into())); + return Some(Event::Html(start_tags.into())); } event } @@ -553,15 +553,13 @@ impl<'a, I: Iterator>> Iterator for Footnotes<'a, I> { pub fn find_testable_code(doc: &str, tests: &mut T, error_codes: ErrorCodes, enable_per_target_ignores: bool) { - let mut parser = Parser::new(doc); - let mut prev_offset = 0; + let mut parser = Parser::new(doc).into_offset_iter(); let mut nb_lines = 0; let mut register_header = None; - while let Some(event) = parser.next() { + let mut prev_offset = 0; + while let Some((event, offset)) = parser.next() { match event { Event::Start(Tag::CodeBlock(s)) => { - let offset = parser.get_offset(); - let block_info = if s.is_empty() { LangString::all_false() } else { @@ -571,8 +569,7 @@ pub fn find_testable_code(doc: &str, tests: &mut T, error_codes continue; } let mut test_s = String::new(); - - while let Some(Event::Text(s)) = parser.next() { + while let Some((Event::Text(s), _)) = parser.next() { test_s.push_str(&s); } @@ -581,12 +578,12 @@ pub fn find_testable_code(doc: &str, tests: &mut T, error_codes .map(|l| map_line(l).for_code()) .collect::>>() .join("\n"); - nb_lines += doc[prev_offset..offset].lines().count(); + nb_lines += doc[prev_offset..offset.end].lines().count(); let line = tests.get_line() + nb_lines; tests.add_test(text, block_info, line); - prev_offset = offset; + prev_offset = offset.start; } - Event::Start(Tag::Header(level)) => { + Event::Start(Tag::Heading(level)) => { register_header = Some(level as u32); } Event::Text(ref s) if register_header.is_some() => { @@ -766,7 +763,7 @@ impl MarkdownHtml<'_> { // Treat inline HTML as plain text. let p = p.map(|event| match event { - Event::Html(text) | Event::InlineHtml(text) => Event::Text(text), + Event::Html(text) => Event::Text(text), _ => event }); @@ -823,10 +820,10 @@ pub fn plain_summary_line(md: &str) -> String { let next_event = next_event.unwrap(); let (ret, is_in) = match next_event { Event::Start(Tag::Paragraph) => (None, 1), - Event::Start(Tag::Header(_)) => (None, 1), + Event::Start(Tag::Heading(_)) => (None, 1), Event::Code(code) => (Some(format!("`{}`", code)), 0), Event::Text(ref s) if self.is_in > 0 => (Some(s.as_ref().to_owned()), 0), - Event::End(Tag::Paragraph) | Event::End(Tag::Header(_)) => (None, -1), + Event::End(Tag::Paragraph) | Event::End(Tag::Heading(_)) => (None, -1), _ => (None, 0), }; if is_in > 0 || (is_in < 0 && self.is_in > 0) { @@ -925,16 +922,14 @@ crate fn rust_code_blocks(md: &str) -> Vec { return code_blocks; } - let mut p = Parser::new_ext(md, opts()); + let mut p = Parser::new_ext(md, opts()).into_offset_iter(); - let mut code_block_start = 0; + let mut code_block = None; let mut code_start = 0; let mut is_fenced = false; - let mut previous_offset = 0; + let mut previous_offset = Range { start: 0, end: 0 }; let mut in_rust_code_block = false; - while let Some(event) = p.next() { - let offset = p.get_offset(); - + while let Some((event, offset_range)) = p.next() { match event { Event::Start(Tag::CodeBlock(syntax)) => { let lang_string = if syntax.is_empty() { @@ -945,55 +940,40 @@ crate fn rust_code_blocks(md: &str) -> Vec { if lang_string.rust { in_rust_code_block = true; + code_block = Some(offset_range.clone()); - code_start = offset; - code_block_start = match md[previous_offset..offset].find("```") { - Some(fence_idx) => { + code_start = match md[offset_range.clone()].find("```") { + Some(_) => { is_fenced = true; - previous_offset + fence_idx + offset_range.start + md[offset_range.clone()] + .lines() + .next() + .map_or(0, |x| x.len() + 1) } None => { is_fenced = false; - offset + offset_range.start } }; + previous_offset = Range { start: code_start, end: offset_range.end }; } } Event::End(Tag::CodeBlock(syntax)) if in_rust_code_block => { in_rust_code_block = false; - let code_block_end = if is_fenced { - let fence_str = &md[previous_offset..offset] - .chars() - .rev() - .collect::(); - fence_str - .find("```") - .map(|fence_idx| offset - fence_idx) - .unwrap_or_else(|| offset) - } else if md - .as_bytes() - .get(offset) - .map(|b| *b == b'\n') - .unwrap_or_default() - { - offset - 1 - } else { - offset - }; - let code_end = if is_fenced { - previous_offset + let last_len = md[previous_offset.clone()] + .lines() + .last() + .map_or(0, |l| l.len()); + previous_offset.end - last_len } else { - code_block_end + previous_offset.end }; code_blocks.push(RustCodeBlock { is_fenced, - range: Range { - start: code_block_start, - end: code_block_end, - }, + range: code_block.clone().unwrap(), code: Range { start: code_start, end: code_end, @@ -1007,8 +987,6 @@ crate fn rust_code_blocks(md: &str) -> Vec { } _ => (), } - - previous_offset = offset; } code_blocks diff --git a/src/librustdoc/passes/check_code_block_syntax.rs b/src/librustdoc/passes/check_code_block_syntax.rs index 5c4c975cec051..da0de0a26a599 100644 --- a/src/librustdoc/passes/check_code_block_syntax.rs +++ b/src/librustdoc/passes/check_code_block_syntax.rs @@ -33,7 +33,6 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> { FileName::Custom(String::from("doctest")), dox[code_block.code].to_owned(), ); - let validation_status = { let mut has_syntax_errors = false; let mut only_whitespace = true; From 7fe3b218f9edc483b905237fa516ead1174df17d Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 13 Nov 2019 16:23:27 +0100 Subject: [PATCH 2/3] Add some debug implementations --- src/librustdoc/passes/mod.rs | 1 + src/librustdoc/test.rs | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/librustdoc/passes/mod.rs b/src/librustdoc/passes/mod.rs index f6560218a78c8..417aaabcfa630 100644 --- a/src/librustdoc/passes/mod.rs +++ b/src/librustdoc/passes/mod.rs @@ -322,6 +322,7 @@ pub fn look_for_tests<'tcx>( } }; + #[derive(Debug)] struct Tests { found_tests: usize, } diff --git a/src/librustdoc/test.rs b/src/librustdoc/test.rs index c483f6fb4770f..1c1d71aa3bfbf 100644 --- a/src/librustdoc/test.rs +++ b/src/librustdoc/test.rs @@ -26,7 +26,7 @@ use crate::clean::Attributes; use crate::config::Options; use crate::html::markdown::{self, ErrorCodes, LangString, Ignore}; -#[derive(Clone, Default)] +#[derive(Clone, Default, Debug)] pub struct TestOptions { /// Whether to disable the default `extern crate my_crate;` when creating doctests. pub no_crate_inject: bool, @@ -640,6 +640,19 @@ pub struct Collector { filename: Option, } +impl ::std::fmt::Debug for Collector { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + f.debug_struct("Collector") + .field("tests", &self.tests) + .field("options", &self.options) + .field("use_headers", &self.use_headers) + .field("enable_per_target_ignores", &self.enable_per_target_ignores) + .field("cratename", &self.cratename) + .field("opts", &self.opts) + .finish() + } +} + impl Collector { pub fn new(cratename: String, options: Options, use_headers: bool, opts: TestOptions, source_map: Option>, filename: Option, From 339e29ac91f332556ad109a42660fd56c9514996 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 13 Nov 2019 17:56:33 +0100 Subject: [PATCH 3/3] Fix tests line --- Cargo.lock | 19 +++-------- src/librustdoc/html/markdown.rs | 39 ++++++++++------------- src/test/rustdoc-ui/invalid-syntax.stderr | 3 +- 3 files changed, 23 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5649ae5a13083..bf6753b017a73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -480,7 +480,7 @@ dependencies = [ "itertools 0.8.0", "lazy_static 1.3.0", "matches", - "pulldown-cmark 0.6.1", + "pulldown-cmark", "quine-mc_cluskey", "regex-syntax", "semver", @@ -1972,7 +1972,7 @@ dependencies = [ "log", "memchr", "open", - "pulldown-cmark 0.6.1", + "pulldown-cmark", "regex", "serde", "serde_derive", @@ -1999,7 +1999,7 @@ dependencies = [ "log", "mdbook", "percent-encoding 2.1.0", - "pulldown-cmark 0.6.1", + "pulldown-cmark", "rayon", "regex", "reqwest", @@ -2641,17 +2641,6 @@ dependencies = [ "url 2.1.0", ] -[[package]] -name = "pulldown-cmark" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77043da1282374688ee212dc44b3f37ff929431de9c9adc3053bd3cee5630357" -dependencies = [ - "bitflags", - "memchr", - "unicase", -] - [[package]] name = "pulldown-cmark" version = "0.6.1" @@ -3902,7 +3891,7 @@ name = "rustdoc" version = "0.0.0" dependencies = [ "minifier", - "pulldown-cmark 0.6.1", + "pulldown-cmark", "rustc-rayon 0.3.0", "tempfile", ] diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index f35ea0f9c62ca..d10b131af8ec6 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -554,9 +554,7 @@ impl<'a, I: Iterator>> Iterator for Footnotes<'a, I> { pub fn find_testable_code(doc: &str, tests: &mut T, error_codes: ErrorCodes, enable_per_target_ignores: bool) { let mut parser = Parser::new(doc).into_offset_iter(); - let mut nb_lines = 0; let mut register_header = None; - let mut prev_offset = 0; while let Some((event, offset)) = parser.next() { match event { Event::Start(Tag::CodeBlock(s)) => { @@ -578,10 +576,10 @@ pub fn find_testable_code(doc: &str, tests: &mut T, error_codes .map(|l| map_line(l).for_code()) .collect::>>() .join("\n"); - nb_lines += doc[prev_offset..offset.end].lines().count(); + // "+ 1" is to take into account the start of the code block + let nb_lines = doc[..offset.start].lines().count() + 1; let line = tests.get_line() + nb_lines; tests.add_test(text, block_info, line); - prev_offset = offset.start; } Event::Start(Tag::Heading(level)) => { register_header = Some(level as u32); @@ -927,8 +925,8 @@ crate fn rust_code_blocks(md: &str) -> Vec { let mut code_block = None; let mut code_start = 0; let mut is_fenced = false; - let mut previous_offset = Range { start: 0, end: 0 }; let mut in_rust_code_block = false; + let mut previous_offset = Range { start: 0, end: 0 }; while let Some((event, offset_range)) = p.next() { match event { Event::Start(Tag::CodeBlock(syntax)) => { @@ -942,35 +940,31 @@ crate fn rust_code_blocks(md: &str) -> Vec { in_rust_code_block = true; code_block = Some(offset_range.clone()); - code_start = match md[offset_range.clone()].find("```") { - Some(_) => { - is_fenced = true; - offset_range.start + md[offset_range.clone()] - .lines() - .next() - .map_or(0, |x| x.len() + 1) - } - None => { - is_fenced = false; - offset_range.start - } + code_start = if !md[previous_offset.end..offset_range.start].ends_with(" ") { + is_fenced = true; + offset_range.start + md[offset_range.clone()] + .lines() + .next() + .map_or(0, |x| x.len() + 1) + } else { + is_fenced = false; + offset_range.start }; - previous_offset = Range { start: code_start, end: offset_range.end }; } } Event::End(Tag::CodeBlock(syntax)) if in_rust_code_block => { in_rust_code_block = false; let code_end = if is_fenced { - let last_len = md[previous_offset.clone()] + let last_len = md[offset_range.clone()] .lines() .last() + .filter(|l| l.ends_with("```")) .map_or(0, |l| l.len()); - previous_offset.end - last_len + offset_range.end - last_len } else { - previous_offset.end + offset_range.end }; - code_blocks.push(RustCodeBlock { is_fenced, range: code_block.clone().unwrap(), @@ -987,6 +981,7 @@ crate fn rust_code_blocks(md: &str) -> Vec { } _ => (), } + previous_offset = offset_range; } code_blocks diff --git a/src/test/rustdoc-ui/invalid-syntax.stderr b/src/test/rustdoc-ui/invalid-syntax.stderr index fe5442163ea72..df507e8bd18ef 100644 --- a/src/test/rustdoc-ui/invalid-syntax.stderr +++ b/src/test/rustdoc-ui/invalid-syntax.stderr @@ -115,7 +115,8 @@ warning: could not parse code block as Rust code LL | /// code with bad syntax | _________^ LL | | /// \_ - | |__________^ +LL | | /// + | |_ error: unknown start of token: ` --> :1:1