From bb398ca59447d8be1d9a9386de0f8ba2ecc3d9c1 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 29 Apr 2022 15:23:25 +1000 Subject: [PATCH 1/2] Remove hacks in `make_token_stream`. `make_tokenstream` has three commented hacks, and a comment at the top referring to #67062. These hacks have no observable effect, at least as judged by running the test suite. The hacks were added in #82608, with an explanation [here](https://github.com/rust-lang/rust/pull/82608#issuecomment-812877329). It appears that one of the following is true: (a) they never did anything useful, (b) they do something useful but we have no test coverage for them, or (c) something has changed in the meantime that means they are no longer necessary. This commit removes the hacks and the comments, in the hope that (b) is not true. --- .../rustc_parse/src/parser/attr_wrapper.rs | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index a12621564ab70..66db5bf9d7cf2 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -388,12 +388,6 @@ impl<'a> Parser<'a> { /// Converts a flattened iterator of tokens (including open and close delimiter tokens) /// into a `TokenStream`, creating a `TokenTree::Delimited` for each matching pair /// of open and close delims. -// FIXME(#67062): Currently, we don't parse `Invisible`-delimited groups correctly, -// which can cause us to end up with mismatched `Invisible` delimiters in our -// captured tokens. This function contains several hacks to work around this - -// essentially, we throw away mismatched `Invisible` delimiters when we encounter them. -// Once we properly parse `Invisible` delimiters, they can be captured just like any -// other tokens, and these hacks can be removed. fn make_token_stream( mut iter: impl Iterator, break_last_token: bool, @@ -412,35 +406,10 @@ fn make_token_stream( stack.push(FrameData { open_delim_sp: Some((delim, span)), inner: vec![] }); } FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => { - // HACK: If we encounter a mismatched `Invisible` delimiter at the top - // level, just ignore it. - if matches!(delim, Delimiter::Invisible) - && (stack.len() == 1 - || !matches!( - stack.last_mut().unwrap().open_delim_sp.unwrap().0, - Delimiter::Invisible - )) - { - token_and_spacing = iter.next(); - continue; - } let frame_data = stack .pop() .unwrap_or_else(|| panic!("Token stack was empty for token: {:?}", token)); - // HACK: If our current frame has a mismatched opening `Invisible` delimiter, - // merge our current frame with the one above it. That is, transform - // `[ { < first second } third ]` into `[ { first second } third ]` - if !matches!(delim, Delimiter::Invisible) - && matches!(frame_data.open_delim_sp.unwrap().0, Delimiter::Invisible) - { - stack.last_mut().unwrap().inner.extend(frame_data.inner); - // Process our closing delimiter again, this time at the previous - // frame in the stack - token_and_spacing = Some((token, spacing)); - continue; - } - let (open_delim, open_sp) = frame_data.open_delim_sp.unwrap(); assert_eq!( open_delim, delim, @@ -472,13 +441,6 @@ fn make_token_stream( } token_and_spacing = iter.next(); } - // HACK: If we don't have a closing `Invisible` delimiter for our last - // frame, merge the frame with the top-level frame. That is, - // turn `< first second` into `first second` - if stack.len() == 2 && stack[1].open_delim_sp.unwrap().0 == Delimiter::Invisible { - let temp_buf = stack.pop().unwrap(); - stack.last_mut().unwrap().inner.extend(temp_buf.inner); - } let mut final_buf = stack.pop().expect("Missing final buf!"); if break_last_token { let (last_token, spacing) = final_buf.inner.pop().unwrap(); From 3cd8e9866d55ddd962e6bda92018cbf4bc9ee08f Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 11 May 2022 10:14:49 +1000 Subject: [PATCH 2/2] Remove some unnecessary invisible delimiter checks. These seem to have no useful effect... they don't seem useful from a code inspection point of view, and they affect anything in the test suite. --- compiler/rustc_parse/src/parser/diagnostics.rs | 3 +-- src/tools/rustfmt/src/parse/macros/mod.rs | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index beffbdc5de410..369650edd57d0 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -2109,8 +2109,7 @@ impl<'a> Parser<'a> { brace_depth -= 1; continue; } - } else if self.token == token::Eof || self.eat(&token::CloseDelim(Delimiter::Invisible)) - { + } else if self.token == token::Eof { return; } else { self.bump(); diff --git a/src/tools/rustfmt/src/parse/macros/mod.rs b/src/tools/rustfmt/src/parse/macros/mod.rs index d4dbf21f8cab7..67f3985926e2f 100644 --- a/src/tools/rustfmt/src/parse/macros/mod.rs +++ b/src/tools/rustfmt/src/parse/macros/mod.rs @@ -79,9 +79,7 @@ fn check_keyword<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option { for &keyword in RUST_KW.iter() { if parser.token.is_keyword(keyword) && parser.look_ahead(1, |t| { - t.kind == TokenKind::Eof - || t.kind == TokenKind::Comma - || t.kind == TokenKind::CloseDelim(Delimiter::Invisible) + t.kind == TokenKind::Eof || t.kind == TokenKind::Comma }) { parser.bump();