squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

Rollup merge of #95301 - nnethercote:rm-NtTT, r=petrochenkov

Browse source 
Remove `Nonterminal::NtTT`.

It's only needed for macro expansion, not as a general element in the
AST. This commit removes it, adds `NtOrTt` for the parser and macro
expansion cases, and renames the variants in `NamedMatch` to better
match the new type.

r? `@petrochenkov`
This commit is contained in:
Dylan DPC 2022-03-28 16:08:07 +02:00 committed by GitHub
parent 0e4524e5b4 364b908d57
commit ae037a86f9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 52 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
compiler/rustc_ast/src/ast_like.rs
View file

@ -51,7 +51,6 @@ impl AstLike for crate::token::Nonterminal {
| Nonterminal::NtMeta(_)
| Nonterminal::NtPath(_)
| Nonterminal::NtVis(_)
| Nonterminal::NtTT(_)
| Nonterminal::NtBlock(_)
| Nonterminal::NtIdent(..)
| Nonterminal::NtLifetime(_) => &[],
@ -67,7 +66,6 @@ impl AstLike for crate::token::Nonterminal {
| Nonterminal::NtMeta(_)
| Nonterminal::NtPath(_)
| Nonterminal::NtVis(_)
| Nonterminal::NtTT(_)
| Nonterminal::NtBlock(_)
| Nonterminal::NtIdent(..)
| Nonterminal::NtLifetime(_) => {}
@ -84,7 +82,7 @@ impl AstLike for crate::token::Nonterminal {
Nonterminal::NtPath(path) => path.tokens_mut(),
Nonterminal::NtVis(vis) => vis.tokens_mut(),
Nonterminal::NtBlock(block) => block.tokens_mut(),
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) | Nonterminal::NtTT(..) => None,
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
}
}
}

1
compiler/rustc_ast/src/mut_visit.rs
View file

@ -787,7 +787,6 @@ pub fn visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut
visit_lazy_tts(tokens, vis);
}
token::NtPath(path) => vis.visit_path(path),
token::NtTT(tt) => visit_tt(tt, vis),
token::NtVis(visib) => vis.visit_vis(visib),
}
}

5
compiler/rustc_ast/src/token.rs
View file

@ -6,7 +6,6 @@ pub use TokenKind::*;
use crate::ast;
use crate::ptr::P;
use crate::tokenstream::TokenTree;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::sync::Lrc;
@ -680,7 +679,6 @@ pub enum Nonterminal {
NtMeta(P<ast::AttrItem>),
NtPath(ast::Path),
NtVis(ast::Visibility),
NtTT(TokenTree),
}
// `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
@ -778,7 +776,6 @@ impl Nonterminal {
NtMeta(attr_item) => attr_item.span(),
NtPath(path) => path.span,
NtVis(vis) => vis.span,
NtTT(tt) => tt.span(),
}
}
}
@ -790,7 +787,6 @@ impl PartialEq for Nonterminal {
ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
}
(NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
(NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs,
// FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
// correctly based on data from AST. This will prevent them from matching each other
// in macros. The comparison will become possible only when each nonterminal has an
@ -813,7 +809,6 @@ impl fmt::Debug for Nonterminal {
NtLiteral(..) => f.pad("NtLiteral(..)"),
NtMeta(..) => f.pad("NtMeta(..)"),
NtPath(..) => f.pad("NtPath(..)"),
NtTT(..) => f.pad("NtTT(..)"),
NtVis(..) => f.pad("NtVis(..)"),
NtLifetime(..) => f.pad("NtLifetime(..)"),
}

1
compiler/rustc_ast_pretty/src/pprust/state.rs
View file

@ -714,7 +714,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw).to_string(),
token::NtLifetime(e) => e.to_string(),
token::NtLiteral(ref e) => self.expr_to_string(e),
token::NtTT(ref tree) => self.tt_to_string(tree),
token::NtVis(ref e) => self.vis_to_string(e),
}
}

37
compiler/rustc_expand/src/mbe/macro_parser.rs
View file

@ -76,7 +76,7 @@ crate use ParseResult::*;
use crate::mbe::{self, SequenceRepetition, TokenTree};
use rustc_ast::token::{self, DocComment, Nonterminal, Token};
use rustc_parse::parser::Parser;
use rustc_parse::parser::{NtOrTt, Parser};
use rustc_session::parse::ParseSess;
use rustc_span::symbol::MacroRulesNormalizedIdent;
@ -275,7 +275,7 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize {
}
/// `NamedMatch` is a pattern-match result for a single metavar. All
/// `MatchedNtNonTt`s in the `NamedMatch` have the same non-terminal type
/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type
/// (expr, item, etc).
///
/// The in-memory structure of a particular `NamedMatch` represents the match
@ -306,17 +306,17 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize {
/// ```rust
/// MatchedSeq([
/// MatchedSeq([
/// MatchedNtNonTt(a),
/// MatchedNtNonTt(b),
/// MatchedNtNonTt(c),
/// MatchedNtNonTt(d),
/// MatchedNonterminal(a),
/// MatchedNonterminal(b),
/// MatchedNonterminal(c),
/// MatchedNonterminal(d),
/// ]),
/// MatchedSeq([
/// MatchedNtNonTt(a),
/// MatchedNtNonTt(b),
/// MatchedNtNonTt(c),
/// MatchedNtNonTt(d),
/// MatchedNtNonTt(e),
/// MatchedNonterminal(a),
/// MatchedNonterminal(b),
/// MatchedNonterminal(c),
/// MatchedNonterminal(d),
/// MatchedNonterminal(e),
/// ])
/// ])
/// ```
@ -324,14 +324,11 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize {
crate enum NamedMatch {
MatchedSeq(Lrc<NamedMatchVec>),
// This variant should never hold an `NtTT`. `MatchedNtTt` should be used
// for that case.
MatchedNtNonTt(Lrc<Nonterminal>),
// A metavar match of type `tt`.
MatchedTokenTree(rustc_ast::tokenstream::TokenTree),
// `NtTT` is handled without any cloning when transcribing, unlike other
// nonterminals. Therefore, an `Lrc` isn't helpful and causes unnecessary
// allocations. Hence this separate variant.
MatchedNtTt(rustc_ast::tokenstream::TokenTree),
// A metavar match of any type other than `tt`.
MatchedNonterminal(Lrc<Nonterminal>),
}
/// Takes a slice of token trees `ms` representing a matcher which successfully matched input
@ -677,8 +674,8 @@ impl<'tt> TtParser<'tt> {
Ok(nt) => nt,
};
let m = match nt {
Nonterminal::NtTT(tt) => MatchedNtTt(tt),
_ => MatchedNtNonTt(Lrc::new(nt)),
NtOrTt::Nt(nt) => MatchedNonterminal(Lrc::new(nt)),
NtOrTt::Tt(tt) => MatchedTokenTree(tt),
};
item.push_match(match_cur, m);
item.idx += 1;

6
compiler/rustc_expand/src/mbe/macro_rules.rs
View file

@ -4,7 +4,7 @@ use crate::expand::{ensure_complete_parse, parse_ast_fragment, AstFragment, AstF
use crate::mbe;
use crate::mbe::macro_check;
use crate::mbe::macro_parser::{Error, ErrorReported, Failure, Success, TtParser};
use crate::mbe::macro_parser::{MatchedNtTt, MatchedSeq};
use crate::mbe::macro_parser::{MatchedSeq, MatchedTokenTree};
use crate::mbe::transcribe::transcribe;
use rustc_ast as ast;
@ -470,7 +470,7 @@ pub fn compile_declarative_macro(
MatchedSeq(ref s) => s
.iter()
.map(|m| {
if let MatchedNtTt(ref tt) = *m {
if let MatchedTokenTree(ref tt) = *m {
let mut tts = vec![];
mbe::quoted::parse(
tt.clone().into(),
@ -495,7 +495,7 @@ pub fn compile_declarative_macro(
MatchedSeq(ref s) => s
.iter()
.map(|m| {
if let MatchedNtTt(ref tt) = *m {
if let MatchedTokenTree(ref tt) = *m {
let mut tts = vec![];
mbe::quoted::parse(
tt.clone().into(),

15
compiler/rustc_expand/src/mbe/transcribe.rs
View file

@ -1,8 +1,8 @@
use crate::base::ExtCtxt;
use crate::mbe::macro_parser::{MatchedNtNonTt, MatchedNtTt, MatchedSeq, NamedMatch};
use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree, NamedMatch};
use crate::mbe::{self, MetaVarExpr};
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::token::{self, Nonterminal, Token, TokenKind};
use rustc_ast::token::{self, Token, TokenKind};
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::Lrc;
@ -234,17 +234,16 @@ pub(super) fn transcribe<'a>(
let ident = MacroRulesNormalizedIdent::new(orignal_ident);
if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
match cur_matched {
MatchedNtTt(ref tt) => {
MatchedTokenTree(ref tt) => {
// `tt`s are emitted into the output stream directly as "raw tokens",
// without wrapping them into groups.
let token = tt.clone();
result.push(token.into());
}
MatchedNtNonTt(ref nt) => {
MatchedNonterminal(ref nt) => {
// Other variables are emitted into the output stream as groups with
// `Delimiter::None` to maintain parsing priorities.
// `Interpolated` is currently used for such groups in rustc parser.
debug_assert!(!matches!(**nt, Nonterminal::NtTT(_)));
marker.visit_span(&mut sp);
let token = TokenTree::token(token::Interpolated(nt.clone()), sp);
result.push(token.into());
@ -312,7 +311,7 @@ fn lookup_cur_matched<'a>(
let mut matched = matched;
for &(idx, _) in repeats {
match matched {
MatchedNtTt(_) | MatchedNtNonTt(_) => break,
MatchedTokenTree(_) | MatchedNonterminal(_) => break,
MatchedSeq(ref ads) => matched = ads.get(idx).unwrap(),
}
}
@ -402,7 +401,7 @@ fn lockstep_iter_size(
let name = MacroRulesNormalizedIdent::new(name);
match lookup_cur_matched(name, interpolations, repeats) {
Some(matched) => match matched {
MatchedNtTt(_) | MatchedNtNonTt(_) => LockstepIterSize::Unconstrained,
MatchedTokenTree(_) | MatchedNonterminal(_) => LockstepIterSize::Unconstrained,
MatchedSeq(ref ads) => LockstepIterSize::Constraint(ads.len(), name),
},
_ => LockstepIterSize::Unconstrained,
@ -449,7 +448,7 @@ fn count_repetitions<'a>(
sp: &DelimSpan,
) -> PResult<'a, usize> {
match matched {
MatchedNtTt(_) | MatchedNtNonTt(_) => {
MatchedTokenTree(_) | MatchedNonterminal(_) => {
if declared_lhs_depth == 0 {
return Err(cx.struct_span_err(
sp.entire(),

1
compiler/rustc_parse/src/lib.rs
View file

@ -290,7 +290,6 @@ pub fn nt_to_tokenstream(
Nonterminal::NtMeta(ref attr) => convert_tokens(attr.tokens.as_ref()),
Nonterminal::NtPath(ref path) => convert_tokens(path.tokens.as_ref()),
Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()),
Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),
Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => {
prepend_attrs(&expr.attrs, expr.tokens.as_ref())
}

8
compiler/rustc_parse/src/parser/mod.rs
View file

@ -19,7 +19,7 @@ pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
pub use path::PathStyle;
use rustc_ast::ptr::P;
use rustc_ast::token::{self, DelimToken, Token, TokenKind};
use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind};
use rustc_ast::tokenstream::AttributesData;
use rustc_ast::tokenstream::{self, DelimSpan, Spacing};
use rustc_ast::tokenstream::{TokenStream, TokenTree};
@ -1507,3 +1507,9 @@ pub enum FlatToken {
/// handling of replace ranges.
Empty,
}
#[derive(Debug)]
pub enum NtOrTt {
Nt(Nonterminal),
Tt(TokenTree),
}

29
compiler/rustc_parse/src/parser/nonterminal.rs
View file

@ -1,12 +1,12 @@
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Nonterminal, NonterminalKind, Token};
use rustc_ast::token::{self, NonterminalKind, Token};
use rustc_ast::AstLike;
use rustc_ast_pretty::pprust;
use rustc_errors::PResult;
use rustc_span::symbol::{kw, Ident};
use crate::parser::pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
use crate::parser::{FollowedByType, ForceCollect, Parser, PathStyle};
use crate::parser::{FollowedByType, ForceCollect, NtOrTt, Parser, PathStyle};
impl<'a> Parser<'a> {
/// Checks whether a non-terminal may begin with a particular token.
@ -85,7 +85,7 @@ impl<'a> Parser<'a> {
NonterminalKind::Lifetime => match token.kind {
token::Lifetime(_) => true,
token::Interpolated(ref nt) => {
matches!(**nt, token::NtLifetime(_) | token::NtTT(_))
matches!(**nt, token::NtLifetime(_))
}
_ => false,
},
@ -96,7 +96,7 @@ impl<'a> Parser<'a> {
}
/// Parse a non-terminal (e.g. MBE `:pat` or `:ident`).
pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonterminal> {
pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, NtOrTt> {
// Any `Nonterminal` which stores its tokens (currently `NtItem` and `NtExpr`)
// needs to have them force-captured here.
// A `macro_rules!` invocation may pass a captured item/expr to a proc-macro,
@ -104,6 +104,8 @@ impl<'a> Parser<'a> {
// in advance whether or not a proc-macro will be (transitively) invoked,
// we always capture tokens for any `Nonterminal` which needs them.
let mut nt = match kind {
// Note that TT is treated differently to all the others.
NonterminalKind::TT => return Ok(NtOrTt::Tt(self.parse_token_tree())),
NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? {
Some(item) => token::NtItem(item),
None => {
@ -124,9 +126,12 @@ impl<'a> Parser<'a> {
NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr { .. } => {
token::NtPat(self.collect_tokens_no_attrs(|this| match kind {
NonterminalKind::PatParam { .. } => this.parse_pat_no_top_alt(None),
NonterminalKind::PatWithOr { .. } => {
this.parse_pat_allow_top_alt(None, RecoverComma::No, RecoverColon::No, CommaRecoveryMode::EitherTupleOrPipe)
}
NonterminalKind::PatWithOr { .. } => this.parse_pat_allow_top_alt(
None,
RecoverComma::No,
RecoverColon::No,
CommaRecoveryMode::EitherTupleOrPipe,
),
_ => unreachable!(),
})?)
}
@ -139,9 +144,10 @@ impl<'a> Parser<'a> {
)
}
NonterminalKind::Ty => {
token::NtTy(self.collect_tokens_no_attrs(|this| this.parse_no_question_mark_recover())?)
}
NonterminalKind::Ty => token::NtTy(
self.collect_tokens_no_attrs(|this| this.parse_no_question_mark_recover())?,
),
// this could be handled like a token, since it is one
NonterminalKind::Ident
if let Some((ident, is_raw)) = get_macro_ident(&self.token) =>
@ -158,7 +164,6 @@ impl<'a> Parser<'a> {
self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?,
),
NonterminalKind::Meta => token::NtMeta(P(self.parse_attr_item(true)?)),
NonterminalKind::TT => token::NtTT(self.parse_token_tree()),
NonterminalKind::Vis => token::NtVis(
self.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?,
),
@ -183,7 +188,7 @@ impl<'a> Parser<'a> {
);
}
Ok(nt)
Ok(NtOrTt::Nt(nt))
}
}