Skip to content

Commit 3e1a63d

Browse files
authored
Rollup merge of #145568 - fee1-dead-contrib:push-uvsonuzxmkus, r=fmease
ignore frontmatters in `TokenStream::new` Fixes #145520 for now, we'd likely want to figure the stripping part later, so I noted it down on the list on the tracking issue. cc `@fmease`
2 parents 07518a7 + c335d57 commit 3e1a63d

File tree

4 files changed

+18
-8
lines changed

4 files changed

+18
-8
lines changed

compiler/rustc_parse/src/lexer/mod.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,15 @@ pub(crate) fn lex_token_trees<'psess, 'src>(
4949
mut src: &'src str,
5050
mut start_pos: BytePos,
5151
override_span: Option<Span>,
52+
frontmatter_allowed: FrontmatterAllowed,
5253
) -> Result<TokenStream, Vec<Diag<'psess>>> {
5354
// Skip `#!`, if present.
5455
if let Some(shebang_len) = rustc_lexer::strip_shebang(src) {
5556
src = &src[shebang_len..];
5657
start_pos = start_pos + BytePos::from_usize(shebang_len);
5758
}
5859

59-
let cursor = Cursor::new(src, FrontmatterAllowed::Yes);
60+
let cursor = Cursor::new(src, frontmatter_allowed);
6061
let mut lexer = Lexer {
6162
psess,
6263
start_pos,

compiler/rustc_parse/src/lib.rs

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ use rustc_ast::tokenstream::TokenStream;
2020
use rustc_ast::{AttrItem, Attribute, MetaItemInner, token};
2121
use rustc_ast_pretty::pprust;
2222
use rustc_errors::{Diag, EmissionGuarantee, FatalError, PResult, pluralize};
23+
use rustc_lexer::FrontmatterAllowed;
2324
use rustc_session::parse::ParseSess;
2425
use rustc_span::source_map::SourceMap;
2526
use rustc_span::{FileName, SourceFile, Span};
@@ -146,7 +147,7 @@ fn new_parser_from_source_file(
146147
source_file: Arc<SourceFile>,
147148
) -> Result<Parser<'_>, Vec<Diag<'_>>> {
148149
let end_pos = source_file.end_position();
149-
let stream = source_file_to_stream(psess, source_file, None)?;
150+
let stream = source_file_to_stream(psess, source_file, None, FrontmatterAllowed::Yes)?;
150151
let mut parser = Parser::new(psess, stream, None);
151152
if parser.token == token::Eof {
152153
parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt(), None);
@@ -161,7 +162,9 @@ pub fn source_str_to_stream(
161162
override_span: Option<Span>,
162163
) -> Result<TokenStream, Vec<Diag<'_>>> {
163164
let source_file = psess.source_map().new_source_file(name, source);
164-
source_file_to_stream(psess, source_file, override_span)
165+
// used mainly for `proc_macro` and the likes, not for our parsing purposes, so don't parse
166+
// frontmatters as frontmatters.
167+
source_file_to_stream(psess, source_file, override_span, FrontmatterAllowed::No)
165168
}
166169

167170
/// Given a source file, produces a sequence of token trees. Returns any buffered errors from
@@ -170,6 +173,7 @@ fn source_file_to_stream<'psess>(
170173
psess: &'psess ParseSess,
171174
source_file: Arc<SourceFile>,
172175
override_span: Option<Span>,
176+
frontmatter_allowed: FrontmatterAllowed,
173177
) -> Result<TokenStream, Vec<Diag<'psess>>> {
174178
let src = source_file.src.as_ref().unwrap_or_else(|| {
175179
psess.dcx().bug(format!(
@@ -178,7 +182,13 @@ fn source_file_to_stream<'psess>(
178182
));
179183
});
180184

181-
lexer::lex_token_trees(psess, src.as_str(), source_file.start_pos, override_span)
185+
lexer::lex_token_trees(
186+
psess,
187+
src.as_str(),
188+
source_file.start_pos,
189+
override_span,
190+
frontmatter_allowed,
191+
)
182192
}
183193

184194
/// Runs the given subparser `f` on the tokens of the given `attr`'s item.

tests/ui/frontmatter/auxiliary/makro.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@ use proc_macro::TokenStream;
33

44
#[proc_macro]
55
pub fn check(_: TokenStream) -> TokenStream {
6-
assert!("---\n---".parse::<TokenStream>().unwrap().is_empty());
6+
assert_eq!(6, "---\n---".parse::<TokenStream>().unwrap().into_iter().count());
77
Default::default()
88
}

tests/ui/frontmatter/proc-macro-observer.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,10 @@
22
//@ proc-macro: makro.rs
33
//@ edition: 2021
44

5-
#![feature(frontmatter)]
6-
75
makro::check!();
86

9-
// checks that a proc-macro cannot observe frontmatter tokens.
7+
// checks that a proc-macro doesn't know or parse frontmatters at all and instead treats
8+
// it as normal Rust code.
109
// see auxiliary/makro.rs for how it is tested.
1110

1211
fn main() {}

0 commit comments

Comments
 (0)