Skip to content

Commit f216be4

Browse files
borsbors
committed
Auto merge of #17153 - Veykril:doc-comment-desugaring, r=Veykril
fix: Fix doc comment desugaring for proc-macros Fixes #16259
2 parents e6f3399 + 36c1c77 commit f216be4

File tree

14 files changed

+228
-52
lines changed

14 files changed

+228
-52
lines changed

‎crates/cfg/src/tests.rs

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,32 @@
11
use arbitrary::{Arbitrary, Unstructured};
22
use expect_test::{expect, Expect};
3-
use mbe::{syntax_node_to_token_tree, DummyTestSpanMap, DUMMY};
3+
use mbe::{syntax_node_to_token_tree, DocCommentDesugarMode, DummyTestSpanMap, DUMMY};
44
use syntax::{ast, AstNode, Edition};
55

66
use crate::{CfgAtom, CfgExpr, CfgOptions, DnfExpr};
77

88
fn assert_parse_result(input: &str, expected: CfgExpr) {
99
let source_file = ast::SourceFile::parse(input, Edition::CURRENT).ok().unwrap();
1010
let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
11-
let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap, DUMMY);
11+
let tt = syntax_node_to_token_tree(
12+
tt.syntax(),
13+
DummyTestSpanMap,
14+
DUMMY,
15+
DocCommentDesugarMode::ProcMacro,
16+
);
1217
let cfg = CfgExpr::parse(&tt);
1318
assert_eq!(cfg, expected);
1419
}
1520

1621
fn check_dnf(input: &str, expect: Expect) {
1722
let source_file = ast::SourceFile::parse(input, Edition::CURRENT).ok().unwrap();
1823
let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
19-
let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap, DUMMY);
24+
let tt = syntax_node_to_token_tree(
25+
tt.syntax(),
26+
DummyTestSpanMap,
27+
DUMMY,
28+
DocCommentDesugarMode::ProcMacro,
29+
);
2030
let cfg = CfgExpr::parse(&tt);
2131
let actual = format!("#![cfg({})]", DnfExpr::new(cfg));
2232
expect.assert_eq(&actual);
@@ -25,7 +35,12 @@ fn check_dnf(input: &str, expect: Expect) {
2535
fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) {
2636
let source_file = ast::SourceFile::parse(input, Edition::CURRENT).ok().unwrap();
2737
let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
28-
let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap, DUMMY);
38+
let tt = syntax_node_to_token_tree(
39+
tt.syntax(),
40+
DummyTestSpanMap,
41+
DUMMY,
42+
DocCommentDesugarMode::ProcMacro,
43+
);
2944
let cfg = CfgExpr::parse(&tt);
3045
let dnf = DnfExpr::new(cfg);
3146
let why_inactive = dnf.why_inactive(opts).unwrap().to_string();
@@ -36,7 +51,12 @@ fn check_why_inactive(input: &str, opts: &CfgOptions, expect: Expect) {
3651
fn check_enable_hints(input: &str, opts: &CfgOptions, expected_hints: &[&str]) {
3752
let source_file = ast::SourceFile::parse(input, Edition::CURRENT).ok().unwrap();
3853
let tt = source_file.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
39-
let tt = syntax_node_to_token_tree(tt.syntax(), DummyTestSpanMap, DUMMY);
54+
let tt = syntax_node_to_token_tree(
55+
tt.syntax(),
56+
DummyTestSpanMap,
57+
DUMMY,
58+
DocCommentDesugarMode::ProcMacro,
59+
);
4060
let cfg = CfgExpr::parse(&tt);
4161
let dnf = DnfExpr::new(cfg);
4262
let hints = dnf.compute_enable_hints(opts).map(|diff| diff.to_string()).collect::<Vec<_>>();

‎crates/hir-def/src/attr/tests.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use triomphe::Arc;
55

66
use base_db::FileId;
77
use hir_expand::span_map::{RealSpanMap, SpanMap};
8-
use mbe::syntax_node_to_token_tree;
8+
use mbe::{syntax_node_to_token_tree, DocCommentDesugarMode};
99
use syntax::{ast, AstNode, TextRange};
1010

1111
use crate::attr::{DocAtom, DocExpr};
@@ -18,6 +18,7 @@ fn assert_parse_result(input: &str, expected: DocExpr) {
1818
tt.syntax(),
1919
map.as_ref(),
2020
map.span_for_range(TextRange::empty(0.into())),
21+
DocCommentDesugarMode::ProcMacro,
2122
);
2223
let cfg = DocExpr::parse(&tt);
2324
assert_eq!(cfg, expected);

‎crates/hir-def/src/macro_expansion_tests/proc_macros.rs

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,3 +186,33 @@ fn#0:[email protected]#0# foo#0:[email protected]#0#(#0:[email protected]#0#&#0:[email protected]#0#self#0:[email protected]#
186186
}#0:[email protected]#0#"#]],
187187
);
188188
}
189+
190+
#[test]
191+
fn attribute_macro_doc_desugaring() {
192+
check(
193+
r#"
194+
//- proc_macros: identity
195+
#[proc_macros::identity]
196+
/// doc string \n with newline
197+
/**
198+
MultiLines Doc
199+
MultiLines Doc
200+
*/
201+
#[doc = "doc attr"]
202+
struct S;
203+
"#,
204+
expect![[r##"
205+
#[proc_macros::identity]
206+
/// doc string \n with newline
207+
/**
208+
MultiLines Doc
209+
MultiLines Doc
210+
*/
211+
#[doc = "doc attr"]
212+
struct S;
213+
214+
#[doc = " doc string \\n with newline"]
215+
#[doc = "\n MultiLines Doc\n MultiLines Doc\n"]
216+
#[doc = "doc attr"] struct S;"##]],
217+
);
218+
}

‎crates/hir-expand/src/attrs.rs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use base_db::CrateId;
55
use cfg::CfgExpr;
66
use either::Either;
77
use intern::Interned;
8-
use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct};
8+
use mbe::{syntax_node_to_token_tree, DelimiterKind, DocCommentDesugarMode, Punct};
99
use smallvec::{smallvec, SmallVec};
1010
use span::{Span, SyntaxContextId};
1111
use syntax::unescape;
@@ -239,7 +239,12 @@ impl Attr {
239239
span,
240240
})))
241241
} else if let Some(tt) = ast.token_tree() {
242-
let tree = syntax_node_to_token_tree(tt.syntax(), span_map, span);
242+
let tree = syntax_node_to_token_tree(
243+
tt.syntax(),
244+
span_map,
245+
span,
246+
DocCommentDesugarMode::ProcMacro,
247+
);
243248
Some(Interned::new(AttrInput::TokenTree(Box::new(tree))))
244249
} else {
245250
None

‎crates/hir-expand/src/builtin_derive_macro.rs

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
//! Builtin derives.
22
33
use itertools::izip;
4+
use mbe::DocCommentDesugarMode;
45
use rustc_hash::FxHashSet;
56
use span::{MacroCallId, Span};
67
use stdx::never;
@@ -262,23 +263,40 @@ fn parse_adt(tt: &tt::Subtree, call_site: Span) -> Result<BasicAdtInfo, ExpandEr
262263
match this {
263264
Some(it) => {
264265
param_type_set.insert(it.as_name());
265-
mbe::syntax_node_to_token_tree(it.syntax(), tm, call_site)
266+
mbe::syntax_node_to_token_tree(
267+
it.syntax(),
268+
tm,
269+
call_site,
270+
DocCommentDesugarMode::ProcMacro,
271+
)
266272
}
267273
None => {
268274
tt::Subtree::empty(::tt::DelimSpan { open: call_site, close: call_site })
269275
}
270276
}
271277
};
272278
let bounds = match &param {
273-
ast::TypeOrConstParam::Type(it) => it
274-
.type_bound_list()
275-
.map(|it| mbe::syntax_node_to_token_tree(it.syntax(), tm, call_site)),
279+
ast::TypeOrConstParam::Type(it) => it.type_bound_list().map(|it| {
280+
mbe::syntax_node_to_token_tree(
281+
it.syntax(),
282+
tm,
283+
call_site,
284+
DocCommentDesugarMode::ProcMacro,
285+
)
286+
}),
276287
ast::TypeOrConstParam::Const(_) => None,
277288
};
278289
let ty = if let ast::TypeOrConstParam::Const(param) = param {
279290
let ty = param
280291
.ty()
281-
.map(|ty| mbe::syntax_node_to_token_tree(ty.syntax(), tm, call_site))
292+
.map(|ty| {
293+
mbe::syntax_node_to_token_tree(
294+
ty.syntax(),
295+
tm,
296+
call_site,
297+
DocCommentDesugarMode::ProcMacro,
298+
)
299+
})
282300
.unwrap_or_else(|| {
283301
tt::Subtree::empty(::tt::DelimSpan { open: call_site, close: call_site })
284302
});
@@ -292,7 +310,14 @@ fn parse_adt(tt: &tt::Subtree, call_site: Span) -> Result<BasicAdtInfo, ExpandEr
292310

293311
let where_clause = if let Some(w) = where_clause {
294312
w.predicates()
295-
.map(|it| mbe::syntax_node_to_token_tree(it.syntax(), tm, call_site))
313+
.map(|it| {
314+
mbe::syntax_node_to_token_tree(
315+
it.syntax(),
316+
tm,
317+
call_site,
318+
DocCommentDesugarMode::ProcMacro,
319+
)
320+
})
296321
.collect()
297322
} else {
298323
vec![]
@@ -322,7 +347,14 @@ fn parse_adt(tt: &tt::Subtree, call_site: Span) -> Result<BasicAdtInfo, ExpandEr
322347
let name = p.path()?.qualifier()?.as_single_name_ref()?.as_name();
323348
param_type_set.contains(&name).then_some(p)
324349
})
325-
.map(|it| mbe::syntax_node_to_token_tree(it.syntax(), tm, call_site))
350+
.map(|it| {
351+
mbe::syntax_node_to_token_tree(
352+
it.syntax(),
353+
tm,
354+
call_site,
355+
DocCommentDesugarMode::ProcMacro,
356+
)
357+
})
326358
.collect();
327359
let name_token = name_to_token(tm, name)?;
328360
Ok(BasicAdtInfo { name: name_token, shape, param_types, where_clause, associated_types })

‎crates/hir-expand/src/db.rs

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
use base_db::{salsa, CrateId, FileId, SourceDatabase};
44
use either::Either;
55
use limit::Limit;
6-
use mbe::{syntax_node_to_token_tree, MatchedArmIndex};
6+
use mbe::{syntax_node_to_token_tree, DocCommentDesugarMode, MatchedArmIndex};
77
use rustc_hash::FxHashSet;
88
use span::{AstIdMap, Span, SyntaxContextData, SyntaxContextId};
99
use syntax::{ast, AstNode, Parse, SyntaxElement, SyntaxError, SyntaxNode, SyntaxToken, T};
@@ -156,11 +156,25 @@ pub fn expand_speculative(
156156
// Build the subtree and token mapping for the speculative args
157157
let (mut tt, undo_info) = match loc.kind {
158158
MacroCallKind::FnLike { .. } => (
159-
mbe::syntax_node_to_token_tree(speculative_args, span_map, span),
159+
mbe::syntax_node_to_token_tree(
160+
speculative_args,
161+
span_map,
162+
span,
163+
if loc.def.is_proc_macro() {
164+
DocCommentDesugarMode::ProcMacro
165+
} else {
166+
DocCommentDesugarMode::Mbe
167+
},
168+
),
160169
SyntaxFixupUndoInfo::NONE,
161170
),
162171
MacroCallKind::Attr { .. } if loc.def.is_attribute_derive() => (
163-
mbe::syntax_node_to_token_tree(speculative_args, span_map, span),
172+
mbe::syntax_node_to_token_tree(
173+
speculative_args,
174+
span_map,
175+
span,
176+
DocCommentDesugarMode::ProcMacro,
177+
),
164178
SyntaxFixupUndoInfo::NONE,
165179
),
166180
MacroCallKind::Derive { derive_attr_index: index, .. }
@@ -176,7 +190,12 @@ pub fn expand_speculative(
176190

177191
let censor_cfg =
178192
cfg_process::process_cfg_attrs(db, speculative_args, &loc).unwrap_or_default();
179-
let mut fixups = fixup::fixup_syntax(span_map, speculative_args, span);
193+
let mut fixups = fixup::fixup_syntax(
194+
span_map,
195+
speculative_args,
196+
span,
197+
DocCommentDesugarMode::ProcMacro,
198+
);
180199
fixups.append.retain(|it, _| match it {
181200
syntax::NodeOrToken::Token(_) => true,
182201
it => !censor.contains(it) && !censor_cfg.contains(it),
@@ -191,6 +210,7 @@ pub fn expand_speculative(
191210
fixups.append,
192211
fixups.remove,
193212
span,
213+
DocCommentDesugarMode::ProcMacro,
194214
),
195215
fixups.undo_info,
196216
)
@@ -212,7 +232,12 @@ pub fn expand_speculative(
212232
}?;
213233
match attr.token_tree() {
214234
Some(token_tree) => {
215-
let mut tree = syntax_node_to_token_tree(token_tree.syntax(), span_map, span);
235+
let mut tree = syntax_node_to_token_tree(
236+
token_tree.syntax(),
237+
span_map,
238+
span,
239+
DocCommentDesugarMode::ProcMacro,
240+
);
216241
tree.delimiter = tt::Delimiter::invisible_spanned(span);
217242

218243
Some(tree)
@@ -432,7 +457,16 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult {
432457
return dummy_tt(kind);
433458
}
434459

435-
let mut tt = mbe::syntax_node_to_token_tree(tt.syntax(), map.as_ref(), span);
460+
let mut tt = mbe::syntax_node_to_token_tree(
461+
tt.syntax(),
462+
map.as_ref(),
463+
span,
464+
if loc.def.is_proc_macro() {
465+
DocCommentDesugarMode::ProcMacro
466+
} else {
467+
DocCommentDesugarMode::Mbe
468+
},
469+
);
436470
if loc.def.is_proc_macro() {
437471
// proc macros expect their inputs without parentheses, MBEs expect it with them included
438472
tt.delimiter.kind = tt::DelimiterKind::Invisible;
@@ -469,7 +503,8 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult {
469503
let (mut tt, undo_info) = {
470504
let syntax = item_node.syntax();
471505
let censor_cfg = cfg_process::process_cfg_attrs(db, syntax, &loc).unwrap_or_default();
472-
let mut fixups = fixup::fixup_syntax(map.as_ref(), syntax, span);
506+
let mut fixups =
507+
fixup::fixup_syntax(map.as_ref(), syntax, span, DocCommentDesugarMode::ProcMacro);
473508
fixups.append.retain(|it, _| match it {
474509
syntax::NodeOrToken::Token(_) => true,
475510
it => !censor.contains(it) && !censor_cfg.contains(it),
@@ -484,6 +519,7 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult {
484519
fixups.append,
485520
fixups.remove,
486521
span,
522+
DocCommentDesugarMode::ProcMacro,
487523
),
488524
fixups.undo_info,
489525
)

‎crates/hir-expand/src/declarative.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
use std::sync::OnceLock;
33

44
use base_db::{CrateId, VersionReq};
5+
use mbe::DocCommentDesugarMode;
56
use span::{Edition, MacroCallId, Span, SyntaxContextId};
67
use stdx::TupleExt;
78
use syntax::{ast, AstNode};
@@ -158,6 +159,7 @@ impl DeclarativeMacroExpander {
158159
map.span_for_range(
159160
macro_rules.macro_rules_token().unwrap().text_range(),
160161
),
162+
DocCommentDesugarMode::Mbe,
161163
);
162164

163165
mbe::DeclarativeMacro::parse_macro_rules(&tt, edition, new_meta_vars)
@@ -175,6 +177,7 @@ impl DeclarativeMacroExpander {
175177
arg.syntax(),
176178
map.as_ref(),
177179
map.span_for_range(macro_def.macro_token().unwrap().text_range()),
180+
DocCommentDesugarMode::Mbe,
178181
);
179182

180183
mbe::DeclarativeMacro::parse_macro2(&tt, edition, new_meta_vars)

‎crates/hir-expand/src/eager.rs

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
//!
2020
//! See the full discussion : <https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/Eager.20expansion.20of.20built-in.20macros>
2121
use base_db::CrateId;
22+
use mbe::DocCommentDesugarMode;
2223
use span::SyntaxContextId;
2324
use syntax::{ted, Parse, SyntaxElement, SyntaxNode, TextSize, WalkEvent};
2425
use triomphe::Arc;
@@ -80,7 +81,12 @@ pub fn expand_eager_macro_input(
8081
return ExpandResult { value: None, err };
8182
};
8283

83-
let mut subtree = mbe::syntax_node_to_token_tree(&expanded_eager_input, arg_map, span);
84+
let mut subtree = mbe::syntax_node_to_token_tree(
85+
&expanded_eager_input,
86+
arg_map,
87+
span,
88+
DocCommentDesugarMode::Mbe,
89+
);
8490

8591
subtree.delimiter.kind = crate::tt::DelimiterKind::Invisible;
8692

0 commit comments

Comments
 (0)