From d8589de1f05fdf84016ad8b6fa1d14a076385b90 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 13 Feb 2020 18:39:40 +0100 Subject: [PATCH] Update pulldown-cmark dependency --- Cargo.lock | 30 +++--- src/librustdoc/Cargo.toml | 2 +- src/librustdoc/html/markdown.rs | 170 ++++++++++++++++++-------------- 3 files changed, 108 insertions(+), 94 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 149f02b4b8c94..f3b2f41a09b1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1564,7 +1564,7 @@ dependencies = [ "rand_xoshiro", "sized-chunks", "typenum", - "version_check 0.9.1", + "version_check", ] [[package]] @@ -2014,9 +2014,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.2.0" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39" +checksum = "53445de381a1f436797497c61d851644d0e8e88e6140f22872ad33a704933978" [[package]] name = "memmap" @@ -2604,23 +2604,23 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.5.3" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77043da1282374688ee212dc44b3f37ff929431de9c9adc3053bd3cee5630357" +checksum = "1c205cc82214f3594e2d50686730314f817c67ffa80fe800cf0db78c3c2b9d9e" dependencies = [ "bitflags", + "getopts", "memchr", "unicase", ] [[package]] name = "pulldown-cmark" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c205cc82214f3594e2d50686730314f817c67ffa80fe800cf0db78c3c2b9d9e" +checksum = "2c2d7fd131800e0d63df52aff46201acaab70b431a4a1ec6f0343fe8e64f35a4" dependencies = [ "bitflags", - "getopts", "memchr", "unicase", ] @@ -4158,7 +4158,7 @@ version = "0.0.0" dependencies = [ "itertools 0.8.0", "minifier", - "pulldown-cmark 0.5.3", + "pulldown-cmark 0.7.0", "rustc-rayon", "serde", "serde_json", @@ -5158,11 +5158,11 @@ checksum = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86" [[package]] name = "unicase" -version = "2.5.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e2e6bd1e59e56598518beb94fd6db628ded570326f0a98c679a304bd9f00150" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" dependencies = [ - "version_check 0.1.5", + "version_check", ] [[package]] @@ -5332,12 +5332,6 @@ dependencies = [ "failure", ] -[[package]] -name = "version_check" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" - [[package]] name = "version_check" version = "0.9.1" diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index 74e46b1eae362..4af13e4cd587a 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -9,7 +9,7 @@ name = "rustdoc" path = "lib.rs" [dependencies] -pulldown-cmark = { version = "0.5.3", default-features = false } +pulldown-cmark = { version = "0.7", default-features = false } minifier = "0.0.33" rayon = { version = "0.3.0", package = "rustc-rayon" } serde = { version = "1.0", features = ["derive"] } diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index c87964af0200c..07c092a511c04 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -33,7 +33,7 @@ use crate::html::highlight; use crate::html::toc::TocBuilder; use crate::test; -use pulldown_cmark::{html, CowStr, Event, Options, Parser, Tag}; +use pulldown_cmark::{html, CodeBlockKind, CowStr, Event, Options, Parser, Tag}; #[cfg(test)] mod tests; @@ -189,10 +189,15 @@ impl<'a, I: Iterator>> Iterator for CodeBlocks<'_, 'a, I> { let compile_fail; let ignore; let edition; - if let Some(Event::Start(Tag::CodeBlock(lang))) = event { - let parse_result = LangString::parse(&lang, self.check_error_codes, false); + if let Some(Event::Start(Tag::CodeBlock(kind))) = event { + let parse_result = match kind { + CodeBlockKind::Fenced(ref lang) => { + LangString::parse(&lang, self.check_error_codes, false) + } + CodeBlockKind::Indented => LangString::all_false(), + }; if !parse_result.rust { - return Some(Event::Start(Tag::CodeBlock(lang))); + return Some(Event::Start(Tag::CodeBlock(kind))); } compile_fail = parse_result.compile_fail; ignore = parse_result.ignore; @@ -370,11 +375,11 @@ impl<'a, 'b, 'ids, I: Iterator>> Iterator for HeadingLinks<'a, } let event = self.inner.next(); - if let Some(Event::Start(Tag::Header(level))) = event { + if let Some(Event::Start(Tag::Heading(level))) = event { let mut id = String::new(); for event in &mut self.inner { match &event { - Event::End(Tag::Header(..)) => break, + Event::End(Tag::Heading(..)) => break, Event::Text(text) | Event::Code(text) => { id.extend(text.chars().filter_map(slugify)); } @@ -391,10 +396,10 @@ impl<'a, 'b, 'ids, I: Iterator>> Iterator for HeadingLinks<'a, let mut html_header = String::new(); html::push_html(&mut html_header, self.buf.iter().cloned()); let sec = builder.push(level as u32, html_header, id.clone()); - self.buf.push_front(Event::InlineHtml(format!("{} ", sec).into())); + self.buf.push_front(Event::Html(format!("{} ", sec).into())); } - self.buf.push_back(Event::InlineHtml(format!("", level).into())); + self.buf.push_back(Event::Html(format!("", level).into())); let start_tags = format!( "\ @@ -402,7 +407,7 @@ impl<'a, 'b, 'ids, I: Iterator>> Iterator for HeadingLinks<'a, id = id, level = level ); - return Some(Event::InlineHtml(start_tags.into())); + return Some(Event::Html(start_tags.into())); } event } @@ -556,40 +561,44 @@ pub fn find_testable_code( error_codes: ErrorCodes, enable_per_target_ignores: bool, ) { - let mut parser = Parser::new(doc); + let mut parser = Parser::new(doc).into_offset_iter(); let mut prev_offset = 0; let mut nb_lines = 0; let mut register_header = None; - while let Some(event) = parser.next() { + while let Some((event, offset)) = parser.next() { match event { - Event::Start(Tag::CodeBlock(s)) => { - let offset = parser.get_offset(); - - let block_info = if s.is_empty() { - LangString::all_false() - } else { - LangString::parse(&*s, error_codes, enable_per_target_ignores) + Event::Start(Tag::CodeBlock(kind)) => { + let block_info = match kind { + CodeBlockKind::Fenced(ref lang) => { + if lang.is_empty() { + LangString::all_false() + } else { + LangString::parse(lang, error_codes, enable_per_target_ignores) + } + } + CodeBlockKind::Indented => LangString::all_false(), }; if !block_info.rust { continue; } + let mut test_s = String::new(); - while let Some(Event::Text(s)) = parser.next() { + while let Some((Event::Text(s), _)) = parser.next() { test_s.push_str(&s); } - let text = test_s .lines() .map(|l| map_line(l).for_code()) .collect::>>() .join("\n"); - nb_lines += doc[prev_offset..offset].lines().count(); - let line = tests.get_line() + nb_lines; + + nb_lines += doc[prev_offset..offset.start].lines().count(); + let line = tests.get_line() + nb_lines + 1; tests.add_test(text, block_info, line); - prev_offset = offset; + prev_offset = offset.start; } - Event::Start(Tag::Header(level)) => { + Event::Start(Tag::Heading(level)) => { register_header = Some(level as u32); } Event::Text(ref s) if register_header.is_some() => { @@ -783,7 +792,7 @@ impl MarkdownHtml<'_> { // Treat inline HTML as plain text. let p = p.map(|event| match event { - Event::Html(text) | Event::InlineHtml(text) => Event::Text(text), + Event::Html(text) => Event::Text(text), _ => event, }); @@ -842,10 +851,10 @@ pub fn plain_summary_line(md: &str) -> String { let next_event = next_event.unwrap(); let (ret, is_in) = match next_event { Event::Start(Tag::Paragraph) => (None, 1), - Event::Start(Tag::Header(_)) => (None, 1), + Event::Start(Tag::Heading(_)) => (None, 1), Event::Code(code) => (Some(format!("`{}`", code)), 0), Event::Text(ref s) if self.is_in > 0 => (Some(s.as_ref().to_owned()), 0), - Event::End(Tag::Paragraph) | Event::End(Tag::Header(_)) => (None, -1), + Event::End(Tag::Paragraph) | Event::End(Tag::Heading(_)) => (None, -1), _ => (None, 0), }; if is_in > 0 || (is_in < 0 && self.is_in > 0) { @@ -940,68 +949,79 @@ crate fn rust_code_blocks(md: &str) -> Vec { return code_blocks; } - let mut p = Parser::new_ext(md, opts()); - - let mut code_block_start = 0; - let mut code_start = 0; - let mut is_fenced = false; - let mut previous_offset = 0; - let mut in_rust_code_block = false; - while let Some(event) = p.next() { - let offset = p.get_offset(); + let mut p = Parser::new_ext(md, opts()).into_offset_iter(); + while let Some((event, offset)) = p.next() { match event { Event::Start(Tag::CodeBlock(syntax)) => { - let lang_string = if syntax.is_empty() { - LangString::all_false() - } else { - LangString::parse(&*syntax, ErrorCodes::Yes, false) - }; - - if lang_string.rust { - in_rust_code_block = true; - - code_start = offset; - code_block_start = match md[previous_offset..offset].find("```") { - Some(fence_idx) => { - is_fenced = true; - previous_offset + fence_idx + let (syntax, code_start, code_end, range, is_fenced) = match syntax { + CodeBlockKind::Fenced(syntax) => { + let syntax = syntax.as_ref(); + let lang_string = if syntax.is_empty() { + LangString::all_false() + } else { + LangString::parse(&*syntax, ErrorCodes::Yes, false) + }; + if !lang_string.rust { + continue; } - None => { - is_fenced = false; - offset + let syntax = if syntax.is_empty() { None } else { Some(syntax.to_owned()) }; + let (code_start, mut code_end) = match p.next() { + Some((Event::Text(_), offset)) => (offset.start, offset.end), + Some((_, sub_offset)) => { + let code = Range { start: sub_offset.start, end: sub_offset.start }; + code_blocks.push(RustCodeBlock { + is_fenced: true, + range: offset, + code, + syntax, + }); + continue; + } + None => { + let code = Range { start: offset.end, end: offset.end }; + code_blocks.push(RustCodeBlock { + is_fenced: true, + range: offset, + code, + syntax, + }); + continue; + } + }; + while let Some((Event::Text(_), offset)) = p.next() { + code_end = offset.end; } - }; - } - } - Event::End(Tag::CodeBlock(syntax)) if in_rust_code_block => { - in_rust_code_block = false; - - let code_block_end = if is_fenced { - let fence_str = &md[previous_offset..offset].chars().rev().collect::(); - fence_str - .find("```") - .map(|fence_idx| offset - fence_idx) - .unwrap_or_else(|| offset) - } else if md.as_bytes().get(offset).map(|b| *b == b'\n').unwrap_or_default() { - offset - 1 - } else { - offset + (syntax, code_start, code_end, offset, true) + } + CodeBlockKind::Indented => { + // The ending of the offset goes too far sometime so we reduce it by one in + // these cases. + if offset.end > offset.start + && md.get(offset.end..=offset.end) == Some(&"\n") + { + ( + None, + offset.start, + offset.end, + Range { start: offset.start, end: offset.end - 1 }, + false, + ) + } else { + (None, offset.start, offset.end, offset, false) + } + } }; - let code_end = if is_fenced { previous_offset } else { code_block_end }; - code_blocks.push(RustCodeBlock { is_fenced, - range: Range { start: code_block_start, end: code_block_end }, + range, code: Range { start: code_start, end: code_end }, - syntax: if !syntax.is_empty() { Some(syntax.into_string()) } else { None }, + syntax, }); } _ => (), } - - previous_offset = offset; } code_blocks