Remove token::Lit from ast::MetaItemLit.

compiler/rustc_ast/src/ast.rs

@@ -1733,8 +1733,10 @@ pub enum StrStyle {
 /// A literal in a meta item.
 #[derive(Clone, Encodable, Decodable, Debug, HashStable_Generic)]
 pub struct MetaItemLit {
-    /// The original literal token as written in source code.
-    pub token_lit: token::Lit,
+    /// The original literal as written in the source code.
+    pub symbol: Symbol,
+    /// The original suffix as written in the source code.
+    pub suffix: Option<Symbol>,
     /// The "semantic" representation of the literal lowered from the original tokens.
     /// Strings are unescaped, hexadecimal forms are eliminated, etc.
     pub kind: LitKind,
@@ -1744,13 +1746,14 @@ pub struct MetaItemLit {
 /// Similar to `MetaItemLit`, but restricted to string literals.
 #[derive(Clone, Copy, Encodable, Decodable, Debug)]
 pub struct StrLit {
-    /// The original literal token as written in source code.
-    pub style: StrStyle,
+    /// The original literal as written in source code.
     pub symbol: Symbol,
+    /// The original suffix as written in source code.
     pub suffix: Option<Symbol>,
-    pub span: Span,
-    /// The unescaped "semantic" representation of the literal lowered from the original token.
+    /// The semantic (unescaped) representation of the literal.
     pub symbol_unescaped: Symbol,
+    pub style: StrStyle,
+    pub span: Span,
 }
 
 impl StrLit {
@@ -1796,8 +1799,9 @@ pub enum LitKind {
     /// A string literal (`"foo"`). The symbol is unescaped, and so may differ
     /// from the original token's symbol.
     Str(Symbol, StrStyle),
-    /// A byte string (`b"foo"`).
-    ByteStr(Lrc<[u8]>),
+    /// A byte string (`b"foo"`). Not stored as a symbol because it might be
+    /// non-utf8, and symbols only allow utf8 strings.
+    ByteStr(Lrc<[u8]>, StrStyle),
     /// A byte char (`b'f'`).
     Byte(u8),
     /// A character literal (`'a'`).
@@ -1822,7 +1826,7 @@ impl LitKind {
 
     /// Returns `true` if this literal is byte literal string.
     pub fn is_bytestr(&self) -> bool {
-        matches!(self, LitKind::ByteStr(_))
+        matches!(self, LitKind::ByteStr(..))
     }
 
     /// Returns `true` if this is a numeric literal.
@@ -3102,7 +3106,7 @@ mod size_asserts {
     static_assert_size!(ItemKind, 112);
     static_assert_size!(LitKind, 24);
     static_assert_size!(Local, 72);
-    static_assert_size!(MetaItemLit, 48);
+    static_assert_size!(MetaItemLit, 40);
     static_assert_size!(Param, 40);
     static_assert_size!(Pat, 88);
     static_assert_size!(Path, 24);

compiler/rustc_ast/src/attr/mod.rs

@@ -328,7 +328,9 @@ pub fn mk_name_value_item_str(ident: Ident, str: Symbol, str_span: Span) -> Meta
 }
 
 pub fn mk_name_value_item(ident: Ident, kind: LitKind, lit_span: Span) -> MetaItem {
-    let lit = MetaItemLit { token_lit: kind.to_token_lit(), kind, span: lit_span };
+    let token_lit = kind.synthesize_token_lit();
+    let lit =
+        MetaItemLit { symbol: token_lit.symbol, suffix: token_lit.suffix, kind, span: lit_span };
     let span = ident.span.to(lit_span);
     MetaItem { path: Path::from_ident(ident), kind: MetaItemKind::NameValue(lit), span }
 }
@@ -408,7 +410,7 @@ pub fn mk_attr_name_value_str(
     val: Symbol,
     span: Span,
 ) -> Attribute {
-    let lit = LitKind::Str(val, StrStyle::Cooked).to_token_lit();
+    let lit = LitKind::Str(val, StrStyle::Cooked).synthesize_token_lit();
     let expr = P(Expr {
         id: DUMMY_NODE_ID,
         kind: ExprKind::Lit(lit),

compiler/rustc_ast/src/util/literal.rs

@@ -1,11 +1,12 @@
 //! Code related to parsing literals.
 
-use crate::ast::{self, LitKind, MetaItemLit};
+use crate::ast::{self, LitKind, MetaItemLit, StrStyle};
 use crate::token::{self, Token};
 use rustc_lexer::unescape::{byte_from_char, unescape_byte, unescape_char, unescape_literal, Mode};
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::Span;
 use std::ascii;
+use std::str;
 
 #[derive(Debug)]
 pub enum LitError {
@@ -115,9 +116,9 @@ impl LitKind {
                     }
                 });
                 error?;
-                LitKind::ByteStr(buf.into())
+                LitKind::ByteStr(buf.into(), StrStyle::Cooked)
             }
-            token::ByteStrRaw(_) => {
+            token::ByteStrRaw(n) => {
                 let s = symbol.as_str();
                 let bytes = if s.contains('\r') {
                     let mut buf = Vec::with_capacity(s.len());
@@ -136,16 +137,16 @@ impl LitKind {
                     symbol.to_string().into_bytes()
                 };
 
-                LitKind::ByteStr(bytes.into())
+                LitKind::ByteStr(bytes.into(), StrStyle::Raw(n))
             }
             token::Err => LitKind::Err,
         })
     }
 
-    /// Attempts to recover a token from semantic literal.
+    /// Synthesizes a token from a semantic literal.
     /// This function is used when the original token doesn't exist (e.g. the literal is created
     /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
-    pub fn to_token_lit(&self) -> token::Lit {
+    pub fn synthesize_token_lit(&self) -> token::Lit {
         let (kind, symbol, suffix) = match *self {
             LitKind::Str(symbol, ast::StrStyle::Cooked) => {
                 // Don't re-intern unless the escaped string is different.
@@ -155,10 +156,15 @@ impl LitKind {
                 (token::Str, symbol, None)
             }
             LitKind::Str(symbol, ast::StrStyle::Raw(n)) => (token::StrRaw(n), symbol, None),
-            LitKind::ByteStr(ref bytes) => {
+            LitKind::ByteStr(ref bytes, ast::StrStyle::Cooked) => {
                 let string = bytes.escape_ascii().to_string();
                 (token::ByteStr, Symbol::intern(&string), None)
             }
+            LitKind::ByteStr(ref bytes, ast::StrStyle::Raw(n)) => {
+                // Unwrap because raw byte string literals can only contain ASCII.
+                let string = str::from_utf8(bytes).unwrap();
+                (token::ByteStrRaw(n), Symbol::intern(&string), None)
+            }
             LitKind::Byte(byte) => {
                 let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
                 (token::Byte, Symbol::intern(&string), None)
@@ -196,9 +202,32 @@ impl LitKind {
 }
 
 impl MetaItemLit {
-    /// Converts token literal into a meta item literal.
+    /// Converts a token literal into a meta item literal.
     pub fn from_token_lit(token_lit: token::Lit, span: Span) -> Result<MetaItemLit, LitError> {
-        Ok(MetaItemLit { token_lit, kind: LitKind::from_token_lit(token_lit)?, span })
+        Ok(MetaItemLit {
+            symbol: token_lit.symbol,
+            suffix: token_lit.suffix,
+            kind: LitKind::from_token_lit(token_lit)?,
+            span,
+        })
+    }
+
+    /// Cheaply converts a meta item literal into a token literal.
+    pub fn as_token_lit(&self) -> token::Lit {
+        let kind = match self.kind {
+            LitKind::Bool(_) => token::Bool,
+            LitKind::Str(_, ast::StrStyle::Cooked) => token::Str,
+            LitKind::Str(_, ast::StrStyle::Raw(n)) => token::StrRaw(n),
+            LitKind::ByteStr(_, ast::StrStyle::Cooked) => token::ByteStr,
+            LitKind::ByteStr(_, ast::StrStyle::Raw(n)) => token::ByteStrRaw(n),
+            LitKind::Byte(_) => token::Byte,
+            LitKind::Char(_) => token::Char,
+            LitKind::Int(..) => token::Integer,
+            LitKind::Float(..) => token::Float,
+            LitKind::Err => token::Err,
+        };
+
+        token::Lit::new(kind, self.symbol, self.suffix)
     }
 
     /// Converts an arbitrary token into meta item literal.

compiler/rustc_ast_lowering/src/expr.rs

@@ -97,7 +97,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 }
                 ExprKind::IncludedBytes(bytes) => hir::ExprKind::Lit(respan(
                     self.lower_span(e.span),
-                    LitKind::ByteStr(bytes.clone()),
+                    LitKind::ByteStr(bytes.clone(), StrStyle::Cooked),
                 )),
                 ExprKind::Cast(expr, ty) => {
                     let expr = self.lower_expr(expr);

compiler/rustc_ast_lowering/src/lib.rs

@@ -954,7 +954,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     lit
                 } else {
                     MetaItemLit {
-                        token_lit: token::Lit::new(token::LitKind::Err, kw::Empty, None),
+                        symbol: kw::Empty,
+                        suffix: None,
                         kind: LitKind::Err,
                         span: DUMMY_SP,
                     }

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -376,7 +376,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
     }
 
     fn print_meta_item_lit(&mut self, lit: &ast::MetaItemLit) {
-        self.print_token_literal(lit.token_lit, lit.span)
+        self.print_token_literal(lit.as_token_lit(), lit.span)
     }
 
     fn print_token_literal(&mut self, token_lit: token::Lit, span: Span) {

compiler/rustc_ast_pretty/src/pprust/state/expr.rs

@@ -323,7 +323,8 @@ impl<'a> State<'a> {
                 self.print_token_literal(*token_lit, expr.span);
             }
             ast::ExprKind::IncludedBytes(bytes) => {
-                let lit = ast::LitKind::ByteStr(bytes.clone()).to_token_lit();
+                let lit = ast::LitKind::ByteStr(bytes.clone(), ast::StrStyle::Cooked)
+                    .synthesize_token_lit();
                 self.print_token_literal(lit, expr.span)
             }
             ast::ExprKind::Cast(expr, ty) => {

compiler/rustc_builtin_macros/src/concat_bytes.rs

@@ -69,7 +69,7 @@ fn invalid_type_err(
         Ok(ast::LitKind::Int(_, _)) => {
             cx.span_err(span, "numeric literal is not a `u8`");
         }
-        Ok(ast::LitKind::ByteStr(_) | ast::LitKind::Byte(_)) => unreachable!(),
+        Ok(ast::LitKind::ByteStr(..) | ast::LitKind::Byte(_)) => unreachable!(),
         Err(err) => {
             report_lit_error(&cx.sess.parse_sess, err, token_lit, span);
         }
@@ -97,7 +97,7 @@ fn handle_array_element(
             )) if val <= u8::MAX.into() => Some(val as u8),
 
             Ok(ast::LitKind::Byte(val)) => Some(val),
-            Ok(ast::LitKind::ByteStr(_)) => {
+            Ok(ast::LitKind::ByteStr(..)) => {
                 if !*has_errors {
                     cx.struct_span_err(expr.span, "cannot concatenate doubly nested array")
                         .note("byte strings are treated as arrays of bytes")
@@ -174,7 +174,7 @@ pub fn expand_concat_bytes(
                 Ok(ast::LitKind::Byte(val)) => {
                     accumulator.push(val);
                 }
-                Ok(ast::LitKind::ByteStr(ref bytes)) => {
+                Ok(ast::LitKind::ByteStr(ref bytes, _)) => {
                     accumulator.extend_from_slice(&bytes);
                 }
                 _ => {

compiler/rustc_builtin_macros/src/derive.rs

@@ -1,7 +1,7 @@
 use crate::cfg_eval::cfg_eval;
 
 use rustc_ast as ast;
-use rustc_ast::{token, GenericParamKind, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
+use rustc_ast::{GenericParamKind, ItemKind, MetaItemKind, NestedMetaItem, StmtKind};
 use rustc_errors::{struct_span_err, Applicability};
 use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier};
 use rustc_feature::AttributeTemplate;
@@ -130,9 +130,11 @@ fn report_bad_target(sess: &Session, item: &Annotatable, span: Span) -> bool {
 }
 
 fn report_unexpected_meta_item_lit(sess: &Session, lit: &ast::MetaItemLit) {
-    let help_msg = match lit.token_lit.kind {
-        token::Str if rustc_lexer::is_ident(lit.token_lit.symbol.as_str()) => {
-            format!("try using `#[derive({})]`", lit.token_lit.symbol)
+    let help_msg = match lit.kind {
+        ast::LitKind::Str(_, ast::StrStyle::Cooked)
+            if rustc_lexer::is_ident(lit.symbol.as_str()) =>
+        {
+            format!("try using `#[derive({})]`", lit.symbol)
         }
         _ => "for example, write `#[derive(Debug)]` for `Debug`".to_string(),
     };

compiler/rustc_expand/src/base.rs

@@ -1234,7 +1234,7 @@ pub fn expr_to_spanned_string<'a>(
     Err(match expr.kind {
         ast::ExprKind::Lit(token_lit) => match ast::LitKind::from_token_lit(token_lit) {
             Ok(ast::LitKind::Str(s, style)) => return Ok((s, style, expr.span)),
-            Ok(ast::LitKind::ByteStr(_)) => {
+            Ok(ast::LitKind::ByteStr(..)) => {
                 let mut err = cx.struct_span_err(expr.span, err_msg);
                 let span = expr.span.shrink_to_lo();
                 err.span_suggestion(

compiler/rustc_expand/src/build.rs

@@ -333,7 +333,7 @@ impl<'a> ExtCtxt<'a> {
     }
 
     fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P<ast::Expr> {
-        let token_lit = lit_kind.to_token_lit();
+        let token_lit = lit_kind.synthesize_token_lit();
         self.expr(span, ast::ExprKind::Lit(token_lit))
     }
 
@@ -361,7 +361,7 @@ impl<'a> ExtCtxt<'a> {
     }
 
     pub fn expr_byte_str(&self, sp: Span, bytes: Vec<u8>) -> P<ast::Expr> {
-        self.expr_lit(sp, ast::LitKind::ByteStr(Lrc::from(bytes)))
+        self.expr_lit(sp, ast::LitKind::ByteStr(Lrc::from(bytes), ast::StrStyle::Cooked))
     }
 
     /// `[expr1, expr2, ...]`

compiler/rustc_expand/src/proc_macro_server.rs

@@ -526,7 +526,8 @@ impl server::TokenStream for Rustc<'_, '_> {
                 Ok(tokenstream::TokenStream::token_alone(token::Literal(*token_lit), expr.span))
             }
             ast::ExprKind::IncludedBytes(bytes) => {
-                let lit = ast::LitKind::ByteStr(bytes.clone()).to_token_lit();
+                let lit = ast::LitKind::ByteStr(bytes.clone(), ast::StrStyle::Cooked)
+                    .synthesize_token_lit();
                 Ok(tokenstream::TokenStream::token_alone(token::TokenKind::Literal(lit), expr.span))
             }
             ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind {

compiler/rustc_hir_pretty/src/lib.rs

@@ -1256,7 +1256,7 @@ impl<'a> State<'a> {
 
     fn print_literal(&mut self, lit: &hir::Lit) {
         self.maybe_print_comment(lit.span.lo());
-        self.word(lit.node.to_token_lit().to_string())
+        self.word(lit.node.synthesize_token_lit().to_string())
     }
 
     fn print_inline_asm(&mut self, asm: &hir::InlineAsm<'_>) {

compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs

@@ -1169,7 +1169,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         match lit.node {
             ast::LitKind::Str(..) => tcx.mk_static_str(),
-            ast::LitKind::ByteStr(ref v) => {
+            ast::LitKind::ByteStr(ref v, _) => {
                 tcx.mk_imm_ref(tcx.lifetimes.re_static, tcx.mk_array(tcx.types.u8, v.len() as u64))
             }
             ast::LitKind::Byte(_) => tcx.types.u8,

compiler/rustc_hir_typeck/src/pat.rs

@@ -386,7 +386,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // Byte string patterns behave the same way as array patterns
         // They can denote both statically and dynamically-sized byte arrays.
         let mut pat_ty = ty;
-        if let hir::ExprKind::Lit(Spanned { node: ast::LitKind::ByteStr(_), .. }) = lt.kind {
+        if let hir::ExprKind::Lit(Spanned { node: ast::LitKind::ByteStr(..), .. }) = lt.kind {
             let expected = self.structurally_resolved_type(span, expected);
             if let ty::Ref(_, inner_ty, _) = expected.kind()
                 && matches!(inner_ty.kind(), ty::Slice(_))

compiler/rustc_mir_build/src/build/expr/as_constant.rs

@@ -135,14 +135,14 @@ pub(crate) fn lit_to_mir_constant<'tcx>(
             let allocation = tcx.intern_const_alloc(allocation);
             ConstValue::Slice { data: allocation, start: 0, end: s.len() }
         }
-        (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _))
+        (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _))
             if matches!(inner_ty.kind(), ty::Slice(_)) =>
         {
             let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
             let allocation = tcx.intern_const_alloc(allocation);
             ConstValue::Slice { data: allocation, start: 0, end: data.len() }
         }
-        (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
+        (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
             let id = tcx.allocate_bytes(data);
             ConstValue::Scalar(Scalar::from_pointer(id.into(), &tcx))
         }

compiler/rustc_mir_build/src/thir/constant.rs

@@ -33,13 +33,13 @@ pub(crate) fn lit_to_const<'tcx>(
             let str_bytes = s.as_str().as_bytes();
             ty::ValTree::from_raw_bytes(tcx, str_bytes)
         }
-        (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _))
+        (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _))
             if matches!(inner_ty.kind(), ty::Slice(_)) =>
         {
             let bytes = data as &[u8];
             ty::ValTree::from_raw_bytes(tcx, bytes)
         }
-        (ast::LitKind::ByteStr(data), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
+        (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
             let bytes = data as &[u8];
             ty::ValTree::from_raw_bytes(tcx, bytes)
         }