From 645cdca9ba2fd3e47dedeecbb580d490fa9ef85b Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 22 Sep 2019 17:42:17 +0300
Subject: [PATCH 1/8] reduce visibility of a bunch of stuff in ext::tt

---
 src/librustc_resolve/macros.rs       |  4 +-
 src/libsyntax/ext/tt/macro_check.rs  |  2 +-
 src/libsyntax/ext/tt/macro_parser.rs | 16 ++++----
 src/libsyntax/ext/tt/macro_rules.rs  | 14 ++++---
 src/libsyntax/ext/tt/quoted.rs       | 57 +++++++++++-----------------
 src/libsyntax/lib.rs                 | 13 ++++---
 src/libsyntax/tokenstream.rs         |  2 +-
 7 files changed, 50 insertions(+), 58 deletions(-)

diff --git a/src/librustc_resolve/macros.rs b/src/librustc_resolve/macros.rs
index 3900a3dbb38..73ad0670659 100644
--- a/src/librustc_resolve/macros.rs
+++ b/src/librustc_resolve/macros.rs
@@ -18,7 +18,7 @@ use syntax::ext::base::{self, InvocationRes, Indeterminate, SpecialDerives};
 use syntax::ext::base::{MacroKind, SyntaxExtension};
 use syntax::ext::expand::{AstFragment, AstFragmentKind, Invocation, InvocationKind};
 use syntax::ext::hygiene::{self, ExpnId, ExpnData, ExpnKind};
-use syntax::ext::tt::macro_rules;
+use syntax::ext::compile_declarative_macro;
 use syntax::feature_gate::{emit_feature_err, is_builtin_attr_name};
 use syntax::feature_gate::GateIssue;
 use syntax::symbol::{Symbol, kw, sym};
@@ -843,7 +843,7 @@ impl<'a> Resolver<'a> {
     /// Compile the macro into a `SyntaxExtension` and possibly replace it with a pre-defined
     /// extension partially or entirely for built-in macros and legacy plugin macros.
     crate fn compile_macro(&mut self, item: &ast::Item, edition: Edition) -> SyntaxExtension {
-        let mut result = macro_rules::compile(
+        let mut result = compile_declarative_macro(
             &self.session.parse_sess, self.session.features_untracked(), item, edition
         );
 
diff --git a/src/libsyntax/ext/tt/macro_check.rs b/src/libsyntax/ext/tt/macro_check.rs
index 5af97199902..a1734689595 100644
--- a/src/libsyntax/ext/tt/macro_check.rs
+++ b/src/libsyntax/ext/tt/macro_check.rs
@@ -196,7 +196,7 @@ struct MacroState<'a> {
 /// - `node_id` is used to emit lints
 /// - `span` is used when no spans are available
 /// - `lhses` and `rhses` should have the same length and represent the macro definition
-pub fn check_meta_variables(
+crate fn check_meta_variables(
     sess: &ParseSess,
     node_id: NodeId,
     span: Span,
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index dbf14daa30e..a34a0344f27 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -70,8 +70,8 @@
 //! eof: [a $( a )* a b ·]
 //! ```
 
-pub use NamedMatch::*;
-pub use ParseResult::*;
+crate use NamedMatch::*;
+crate use ParseResult::*;
 use TokenTreeOrTokenTreeSlice::*;
 
 use crate::ast::{Ident, Name};
@@ -267,7 +267,7 @@ impl<'root, 'tt> DerefMut for MatcherPosHandle<'root, 'tt> {
 }
 
 /// Represents the possible results of an attempted parse.
-pub enum ParseResult<T> {
+crate enum ParseResult<T> {
     /// Parsed successfully.
     Success(T),
     /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected
@@ -279,10 +279,10 @@ pub enum ParseResult<T> {
 
 /// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es.
 /// This represents the mapping of metavars to the token trees they bind to.
-pub type NamedParseResult = ParseResult<FxHashMap<Ident, NamedMatch>>;
+crate type NamedParseResult = ParseResult<FxHashMap<Ident, NamedMatch>>;
 
 /// Count how many metavars are named in the given matcher `ms`.
-pub fn count_names(ms: &[TokenTree]) -> usize {
+crate fn count_names(ms: &[TokenTree]) -> usize {
     ms.iter().fold(0, |count, elt| {
         count + match *elt {
             TokenTree::Sequence(_, ref seq) => seq.num_captures,
@@ -352,7 +352,7 @@ fn initial_matcher_pos<'root, 'tt>(ms: &'tt [TokenTree], open: Span) -> MatcherP
 /// only on the nesting depth of `ast::TTSeq`s in the originating
 /// token tree it was derived from.
 #[derive(Debug, Clone)]
-pub enum NamedMatch {
+crate enum NamedMatch {
     MatchedSeq(Lrc<NamedMatchVec>, DelimSpan),
     MatchedNonterminal(Lrc<Nonterminal>),
 }
@@ -415,7 +415,7 @@ fn nameize<I: Iterator<Item = NamedMatch>>(
 
 /// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For
 /// other tokens, this is "unexpected token...".
-pub fn parse_failure_msg(tok: &Token) -> String {
+crate fn parse_failure_msg(tok: &Token) -> String {
     match tok.kind {
         token::Eof => "unexpected end of macro invocation".to_string(),
         _ => format!(
@@ -648,7 +648,7 @@ fn inner_parse_loop<'root, 'tt>(
 /// - `directory`: Information about the file locations (needed for the black-box parser)
 /// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box
 ///   parser)
-pub fn parse(
+crate fn parse(
     sess: &ParseSess,
     tts: TokenStream,
     ms: &[TokenTree],
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index b27e9c54337..90dfa6e7ac8 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -35,7 +35,7 @@ const VALID_FRAGMENT_NAMES_MSG: &str = "valid fragment specifiers are \
                                         `ident`, `block`, `stmt`, `expr`, `pat`, `ty`, `lifetime`, \
                                         `literal`, `path`, `meta`, `tt`, `item` and `vis`";
 
-pub struct ParserAnyMacro<'a> {
+crate struct ParserAnyMacro<'a> {
     parser: Parser<'a>,
 
     /// Span of the expansion site of the macro this parser is for
@@ -45,7 +45,11 @@ pub struct ParserAnyMacro<'a> {
     arm_span: Span,
 }
 
-pub fn annotate_err_with_kind(err: &mut DiagnosticBuilder<'_>, kind: AstFragmentKind, span: Span) {
+crate fn annotate_err_with_kind(
+    err: &mut DiagnosticBuilder<'_>,
+    kind: AstFragmentKind,
+    span: Span,
+) {
     match kind {
         AstFragmentKind::Ty => {
             err.span_label(span, "this macro call doesn't expand to a type");
@@ -58,7 +62,7 @@ pub fn annotate_err_with_kind(err: &mut DiagnosticBuilder<'_>, kind: AstFragment
 }
 
 impl<'a> ParserAnyMacro<'a> {
-    pub fn make(mut self: Box<ParserAnyMacro<'a>>, kind: AstFragmentKind) -> AstFragment {
+    crate fn make(mut self: Box<ParserAnyMacro<'a>>, kind: AstFragmentKind) -> AstFragment {
         let ParserAnyMacro { site_span, macro_ident, ref mut parser, arm_span } = *self;
         let fragment = panictry!(parser.parse_ast_fragment(kind, true).map_err(|mut e| {
             if parser.token == token::Eof && e.message().ends_with(", found `<eof>`") {
@@ -284,8 +288,8 @@ fn generic_extension<'cx>(
 //
 // Holy self-referential!
 
-/// Converts a `macro_rules!` invocation into a syntax extension.
-pub fn compile(
+/// Converts a macro item into a syntax extension.
+pub fn compile_declarative_macro(
     sess: &ParseSess,
     features: &Features,
     def: &ast::Item,
diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index cad94a0e4c1..d161e6638bf 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -16,14 +16,14 @@ use std::iter::Peekable;
 /// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note
 /// that the delimiter itself might be `NoDelim`.
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)]
-pub struct Delimited {
-    pub delim: token::DelimToken,
-    pub tts: Vec<TokenTree>,
+crate struct Delimited {
+    crate delim: token::DelimToken,
+    crate tts: Vec<TokenTree>,
 }
 
 impl Delimited {
     /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
-    pub fn open_tt(&self, span: Span) -> TokenTree {
+    crate fn open_tt(&self, span: Span) -> TokenTree {
         let open_span = if span.is_dummy() {
             span
         } else {
@@ -33,7 +33,7 @@ impl Delimited {
     }
 
     /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
-    pub fn close_tt(&self, span: Span) -> TokenTree {
+    crate fn close_tt(&self, span: Span) -> TokenTree {
         let close_span = if span.is_dummy() {
             span
         } else {
@@ -44,25 +44,25 @@ impl Delimited {
 }
 
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)]
-pub struct SequenceRepetition {
+crate struct SequenceRepetition {
     /// The sequence of token trees
-    pub tts: Vec<TokenTree>,
+    crate tts: Vec<TokenTree>,
     /// The optional separator
-    pub separator: Option<Token>,
+    crate separator: Option<Token>,
     /// Whether the sequence can be repeated zero (*), or one or more times (+)
-    pub kleene: KleeneToken,
+    crate kleene: KleeneToken,
     /// The number of `Match`s that appear in the sequence (and subsequences)
-    pub num_captures: usize,
+    crate num_captures: usize,
 }
 
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)]
-pub struct KleeneToken {
-    pub span: Span,
-    pub op: KleeneOp,
+crate struct KleeneToken {
+    crate span: Span,
+    crate op: KleeneOp,
 }
 
 impl KleeneToken {
-    pub fn new(op: KleeneOp, span: Span) -> KleeneToken {
+    crate fn new(op: KleeneOp, span: Span) -> KleeneToken {
         KleeneToken { span, op }
     }
 }
@@ -70,7 +70,7 @@ impl KleeneToken {
 /// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star)
 /// for token sequences.
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
-pub enum KleeneOp {
+crate enum KleeneOp {
     /// Kleene star (`*`) for zero or more repetitions
     ZeroOrMore,
     /// Kleene plus (`+`) for one or more repetitions
@@ -82,7 +82,7 @@ pub enum KleeneOp {
 /// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
 /// are "first-class" token trees. Useful for parsing macros.
 #[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)]
-pub enum TokenTree {
+crate enum TokenTree {
     Token(Token),
     Delimited(DelimSpan, Lrc<Delimited>),
     /// A kleene-style repetition sequence
@@ -99,7 +99,7 @@ pub enum TokenTree {
 
 impl TokenTree {
     /// Return the number of tokens in the tree.
-    pub fn len(&self) -> usize {
+    crate fn len(&self) -> usize {
         match *self {
             TokenTree::Delimited(_, ref delimed) => match delimed.delim {
                 token::NoDelim => delimed.tts.len(),
@@ -110,21 +110,8 @@ impl TokenTree {
         }
     }
 
-    /// Returns `true` if the given token tree contains no other tokens. This is vacuously true for
-    /// single tokens or metavar/decls, but may be false for delimited trees or sequences.
-    pub fn is_empty(&self) -> bool {
-        match *self {
-            TokenTree::Delimited(_, ref delimed) => match delimed.delim {
-                token::NoDelim => delimed.tts.is_empty(),
-                _ => false,
-            },
-            TokenTree::Sequence(_, ref seq) => seq.tts.is_empty(),
-            _ => true,
-        }
-    }
-
     /// Returns `true` if the given token tree is delimited.
-    pub fn is_delimited(&self) -> bool {
+    crate fn is_delimited(&self) -> bool {
         match *self {
             TokenTree::Delimited(..) => true,
             _ => false,
@@ -132,7 +119,7 @@ impl TokenTree {
     }
 
     /// Returns `true` if the given token tree is a token of the given kind.
-    pub fn is_token(&self, expected_kind: &TokenKind) -> bool {
+    crate fn is_token(&self, expected_kind: &TokenKind) -> bool {
         match self {
             TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind,
             _ => false,
@@ -140,7 +127,7 @@ impl TokenTree {
     }
 
     /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences.
-    pub fn get_tt(&self, index: usize) -> TokenTree {
+    crate fn get_tt(&self, index: usize) -> TokenTree {
         match (self, index) {
             (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => {
                 delimed.tts[index].clone()
@@ -160,7 +147,7 @@ impl TokenTree {
     }
 
     /// Retrieves the `TokenTree`'s span.
-    pub fn span(&self) -> Span {
+    crate fn span(&self) -> Span {
         match *self {
             TokenTree::Token(Token { span, .. })
             | TokenTree::MetaVar(span, _)
@@ -195,7 +182,7 @@ impl TokenTree {
 /// # Returns
 ///
 /// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`.
-pub fn parse(
+crate fn parse(
     input: tokenstream::TokenStream,
     expect_matchers: bool,
     sess: &ParseSess,
diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs
index aaf6f3e537e..c06f4302102 100644
--- a/src/libsyntax/lib.rs
+++ b/src/libsyntax/lib.rs
@@ -162,18 +162,19 @@ pub mod ext {
     mod proc_macro_server;
 
     pub use syntax_pos::hygiene;
+    pub use tt::macro_rules::compile_declarative_macro;
     pub mod allocator;
     pub mod base;
     pub mod build;
     pub mod expand;
     pub mod proc_macro;
 
-    pub mod tt {
-        pub mod transcribe;
-        pub mod macro_check;
-        pub mod macro_parser;
-        pub mod macro_rules;
-        pub mod quoted;
+    crate mod tt {
+        crate mod transcribe;
+        crate mod macro_check;
+        crate mod macro_parser;
+        crate mod macro_rules;
+        crate mod quoted;
     }
 }
 
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index d702038f54e..15fc4b02041 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -64,7 +64,7 @@ where
 
 impl TokenTree {
     /// Use this token tree as a matcher to parse given tts.
-    pub fn parse(cx: &base::ExtCtxt<'_>, mtch: &[quoted::TokenTree], tts: TokenStream)
+    crate fn parse(cx: &base::ExtCtxt<'_>, mtch: &[quoted::TokenTree], tts: TokenStream)
                  -> macro_parser::NamedParseResult {
         // `None` is because we're not interpolating
         let directory = Directory {

From 827a5b2ea8dc66bfdf817f52011f470f00bc6fee Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 22 Sep 2019 18:19:51 +0300
Subject: [PATCH 2/8] rename libsyntax::ext::tt to mbe

mbe stands for macro-by-example
---
 src/libsyntax/ext/{tt => mbe}/macro_check.rs  | 0
 src/libsyntax/ext/{tt => mbe}/macro_parser.rs | 0
 src/libsyntax/ext/{tt => mbe}/macro_rules.rs  | 0
 src/libsyntax/ext/{tt => mbe}/quoted.rs       | 0
 src/libsyntax/ext/{tt => mbe}/transcribe.rs   | 0
 5 files changed, 0 insertions(+), 0 deletions(-)
 rename src/libsyntax/ext/{tt => mbe}/macro_check.rs (100%)
 rename src/libsyntax/ext/{tt => mbe}/macro_parser.rs (100%)
 rename src/libsyntax/ext/{tt => mbe}/macro_rules.rs (100%)
 rename src/libsyntax/ext/{tt => mbe}/quoted.rs (100%)
 rename src/libsyntax/ext/{tt => mbe}/transcribe.rs (100%)

diff --git a/src/libsyntax/ext/tt/macro_check.rs b/src/libsyntax/ext/mbe/macro_check.rs
similarity index 100%
rename from src/libsyntax/ext/tt/macro_check.rs
rename to src/libsyntax/ext/mbe/macro_check.rs
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/mbe/macro_parser.rs
similarity index 100%
rename from src/libsyntax/ext/tt/macro_parser.rs
rename to src/libsyntax/ext/mbe/macro_parser.rs
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/mbe/macro_rules.rs
similarity index 100%
rename from src/libsyntax/ext/tt/macro_rules.rs
rename to src/libsyntax/ext/mbe/macro_rules.rs
diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/mbe/quoted.rs
similarity index 100%
rename from src/libsyntax/ext/tt/quoted.rs
rename to src/libsyntax/ext/mbe/quoted.rs
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/mbe/transcribe.rs
similarity index 100%
rename from src/libsyntax/ext/tt/transcribe.rs
rename to src/libsyntax/ext/mbe/transcribe.rs

From 49f849cdb7910162f15ac6ac26a98ba22009dc9a Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 22 Sep 2019 18:25:29 +0300
Subject: [PATCH 3/8] rename tt -> mbe, part 2

---
 src/libsyntax/ext/expand.rs           |  6 +++---
 src/libsyntax/ext/mbe/macro_check.rs  |  2 +-
 src/libsyntax/ext/mbe/macro_parser.rs |  2 +-
 src/libsyntax/ext/mbe/macro_rules.rs  | 12 ++++++------
 src/libsyntax/ext/mbe/quoted.rs       |  2 +-
 src/libsyntax/ext/mbe/transcribe.rs   |  4 ++--
 src/libsyntax/lib.rs                  |  4 ++--
 src/libsyntax/tokenstream.rs          |  2 +-
 8 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index b80c530731d..2cd802f22d3 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -6,7 +6,7 @@ use crate::config::StripUnconfigured;
 use crate::ext::base::*;
 use crate::ext::proc_macro::{collect_derives, MarkAttrs};
 use crate::ext::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind};
-use crate::ext::tt::macro_rules::annotate_err_with_kind;
+use crate::ext::mbe::macro_rules::annotate_err_with_kind;
 use crate::ext::placeholders::{placeholder, PlaceholderExpander};
 use crate::feature_gate::{self, Features, GateIssue, is_builtin_attr, emit_feature_err};
 use crate::mut_visit::*;
@@ -115,8 +115,8 @@ macro_rules! ast_fragments {
             }
         }
 
-        impl<'a> MacResult for crate::ext::tt::macro_rules::ParserAnyMacro<'a> {
-            $(fn $make_ast(self: Box<crate::ext::tt::macro_rules::ParserAnyMacro<'a>>)
+        impl<'a> MacResult for crate::ext::mbe::macro_rules::ParserAnyMacro<'a> {
+            $(fn $make_ast(self: Box<crate::ext::mbe::macro_rules::ParserAnyMacro<'a>>)
                            -> Option<$AstTy> {
                 Some(self.make(AstFragmentKind::$Kind).$make_ast())
             })*
diff --git a/src/libsyntax/ext/mbe/macro_check.rs b/src/libsyntax/ext/mbe/macro_check.rs
index a1734689595..aa88d71666e 100644
--- a/src/libsyntax/ext/mbe/macro_check.rs
+++ b/src/libsyntax/ext/mbe/macro_check.rs
@@ -106,7 +106,7 @@
 //! bound.
 use crate::ast::NodeId;
 use crate::early_buffered_lints::BufferedEarlyLintId;
-use crate::ext::tt::quoted::{KleeneToken, TokenTree};
+use crate::ext::mbe::quoted::{KleeneToken, TokenTree};
 use crate::parse::token::TokenKind;
 use crate::parse::token::{DelimToken, Token};
 use crate::parse::ParseSess;
diff --git a/src/libsyntax/ext/mbe/macro_parser.rs b/src/libsyntax/ext/mbe/macro_parser.rs
index a34a0344f27..88145cd121f 100644
--- a/src/libsyntax/ext/mbe/macro_parser.rs
+++ b/src/libsyntax/ext/mbe/macro_parser.rs
@@ -75,7 +75,7 @@ crate use ParseResult::*;
 use TokenTreeOrTokenTreeSlice::*;
 
 use crate::ast::{Ident, Name};
-use crate::ext::tt::quoted::{self, TokenTree};
+use crate::ext::mbe::quoted::{self, TokenTree};
 use crate::parse::{Directory, ParseSess};
 use crate::parse::parser::{Parser, PathStyle};
 use crate::parse::token::{self, DocComment, Nonterminal, Token};
diff --git a/src/libsyntax/ext/mbe/macro_rules.rs b/src/libsyntax/ext/mbe/macro_rules.rs
index 90dfa6e7ac8..939f5b8f51d 100644
--- a/src/libsyntax/ext/mbe/macro_rules.rs
+++ b/src/libsyntax/ext/mbe/macro_rules.rs
@@ -4,12 +4,12 @@ use crate::edition::Edition;
 use crate::ext::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander};
 use crate::ext::base::{SyntaxExtension, SyntaxExtensionKind};
 use crate::ext::expand::{AstFragment, AstFragmentKind};
-use crate::ext::tt::macro_check;
-use crate::ext::tt::macro_parser::{parse, parse_failure_msg};
-use crate::ext::tt::macro_parser::{Error, Failure, Success};
-use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq};
-use crate::ext::tt::quoted;
-use crate::ext::tt::transcribe::transcribe;
+use crate::ext::mbe::macro_check;
+use crate::ext::mbe::macro_parser::{parse, parse_failure_msg};
+use crate::ext::mbe::macro_parser::{Error, Failure, Success};
+use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq};
+use crate::ext::mbe::quoted;
+use crate::ext::mbe::transcribe::transcribe;
 use crate::feature_gate::Features;
 use crate::parse::parser::Parser;
 use crate::parse::token::TokenKind::*;
diff --git a/src/libsyntax/ext/mbe/quoted.rs b/src/libsyntax/ext/mbe/quoted.rs
index d161e6638bf..8ef28345231 100644
--- a/src/libsyntax/ext/mbe/quoted.rs
+++ b/src/libsyntax/ext/mbe/quoted.rs
@@ -1,6 +1,6 @@
 use crate::ast;
 use crate::ast::NodeId;
-use crate::ext::tt::macro_parser;
+use crate::ext::mbe::macro_parser;
 use crate::feature_gate::Features;
 use crate::parse::token::{self, Token, TokenKind};
 use crate::parse::ParseSess;
diff --git a/src/libsyntax/ext/mbe/transcribe.rs b/src/libsyntax/ext/mbe/transcribe.rs
index f9c07e3a2e4..94ac1f60c64 100644
--- a/src/libsyntax/ext/mbe/transcribe.rs
+++ b/src/libsyntax/ext/mbe/transcribe.rs
@@ -1,7 +1,7 @@
 use crate::ast::{Ident, Mac};
 use crate::ext::base::ExtCtxt;
-use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
-use crate::ext::tt::quoted;
+use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
+use crate::ext::mbe::quoted;
 use crate::mut_visit::{self, MutVisitor};
 use crate::parse::token::{self, NtTT, Token};
 use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs
index c06f4302102..045a3693188 100644
--- a/src/libsyntax/lib.rs
+++ b/src/libsyntax/lib.rs
@@ -162,14 +162,14 @@ pub mod ext {
     mod proc_macro_server;
 
     pub use syntax_pos::hygiene;
-    pub use tt::macro_rules::compile_declarative_macro;
+    pub use mbe::macro_rules::compile_declarative_macro;
     pub mod allocator;
     pub mod base;
     pub mod build;
     pub mod expand;
     pub mod proc_macro;
 
-    crate mod tt {
+    crate mod mbe {
         crate mod transcribe;
         crate mod macro_check;
         crate mod macro_parser;
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index 15fc4b02041..d46e38f53d5 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -14,7 +14,7 @@
 //! ownership of the original.
 
 use crate::ext::base;
-use crate::ext::tt::{macro_parser, quoted};
+use crate::ext::mbe::{macro_parser, quoted};
 use crate::parse::Directory;
 use crate::parse::token::{self, DelimToken, Token, TokenKind};
 use crate::print::pprust;

From e30c5166db72f9f1ceccc355df25b49df826df02 Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 22 Sep 2019 18:28:32 +0300
Subject: [PATCH 4/8] move mbe module to a separate file

---
 src/libsyntax/ext/mbe.rs | 5 +++++
 src/libsyntax/lib.rs     | 8 +-------
 2 files changed, 6 insertions(+), 7 deletions(-)
 create mode 100644 src/libsyntax/ext/mbe.rs

diff --git a/src/libsyntax/ext/mbe.rs b/src/libsyntax/ext/mbe.rs
new file mode 100644
index 00000000000..b77b59fea70
--- /dev/null
+++ b/src/libsyntax/ext/mbe.rs
@@ -0,0 +1,5 @@
+crate mod transcribe;
+crate mod macro_check;
+crate mod macro_parser;
+crate mod macro_rules;
+crate mod quoted;
diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs
index 045a3693188..402d563010f 100644
--- a/src/libsyntax/lib.rs
+++ b/src/libsyntax/lib.rs
@@ -169,13 +169,7 @@ pub mod ext {
     pub mod expand;
     pub mod proc_macro;
 
-    crate mod mbe {
-        crate mod transcribe;
-        crate mod macro_check;
-        crate mod macro_parser;
-        crate mod macro_rules;
-        crate mod quoted;
-    }
+    crate mod mbe;
 }
 
 pub mod early_buffered_lints;

From 636b3543c27e1eba301605afd2143fc96fd14431 Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 22 Sep 2019 20:36:35 +0300
Subject: [PATCH 5/8] docstring for mbe module

---
 src/libsyntax/ext/mbe.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/libsyntax/ext/mbe.rs b/src/libsyntax/ext/mbe.rs
index b77b59fea70..78d96f471fa 100644
--- a/src/libsyntax/ext/mbe.rs
+++ b/src/libsyntax/ext/mbe.rs
@@ -1,3 +1,8 @@
+//! This module implements declarative macros: old `macro_rules` and the newer
+//! `macro`. Declarative macros are also known as "macro by example", and that's
+//! why we call this module `mbe`. For external documentation, prefer the
+//! official terminology: "declarative macros".
+
 crate mod transcribe;
 crate mod macro_check;
 crate mod macro_parser;

From 9fd75f52877a3e23a7b9a6a9675f37a42340a428 Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 22 Sep 2019 19:17:30 +0300
Subject: [PATCH 6/8] pull mbe token tree definition up

---
 src/libsyntax/ext/mbe.rs              | 156 ++++++++++++++++++++++++++
 src/libsyntax/ext/mbe/macro_check.rs  |   2 +-
 src/libsyntax/ext/mbe/macro_parser.rs |  10 +-
 src/libsyntax/ext/mbe/macro_rules.rs  | 116 +++++++++----------
 src/libsyntax/ext/mbe/quoted.rs       | 155 +------------------------
 src/libsyntax/ext/mbe/transcribe.rs   |  34 +++---
 src/libsyntax/tokenstream.rs          |   4 +-
 7 files changed, 243 insertions(+), 234 deletions(-)

diff --git a/src/libsyntax/ext/mbe.rs b/src/libsyntax/ext/mbe.rs
index 78d96f471fa..c18b8918ff1 100644
--- a/src/libsyntax/ext/mbe.rs
+++ b/src/libsyntax/ext/mbe.rs
@@ -8,3 +8,159 @@ crate mod macro_check;
 crate mod macro_parser;
 crate mod macro_rules;
 crate mod quoted;
+
+use crate::ast;
+use crate::parse::token::{self, Token, TokenKind};
+use crate::tokenstream::{DelimSpan};
+
+use syntax_pos::{BytePos, Span};
+
+use rustc_data_structures::sync::Lrc;
+
+/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note
+/// that the delimiter itself might be `NoDelim`.
+#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)]
+crate struct Delimited {
+    crate delim: token::DelimToken,
+    crate tts: Vec<TokenTree>,
+}
+
+impl Delimited {
+    /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
+    crate fn open_tt(&self, span: Span) -> TokenTree {
+        let open_span = if span.is_dummy() {
+            span
+        } else {
+            span.with_hi(span.lo() + BytePos(self.delim.len() as u32))
+        };
+        TokenTree::token(token::OpenDelim(self.delim), open_span)
+    }
+
+    /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
+    crate fn close_tt(&self, span: Span) -> TokenTree {
+        let close_span = if span.is_dummy() {
+            span
+        } else {
+            span.with_lo(span.hi() - BytePos(self.delim.len() as u32))
+        };
+        TokenTree::token(token::CloseDelim(self.delim), close_span)
+    }
+}
+
+#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)]
+crate struct SequenceRepetition {
+    /// The sequence of token trees
+    crate tts: Vec<TokenTree>,
+    /// The optional separator
+    crate separator: Option<Token>,
+    /// Whether the sequence can be repeated zero (*), or one or more times (+)
+    crate kleene: KleeneToken,
+    /// The number of `Match`s that appear in the sequence (and subsequences)
+    crate num_captures: usize,
+}
+
+#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)]
+crate struct KleeneToken {
+    crate span: Span,
+    crate op: KleeneOp,
+}
+
+impl KleeneToken {
+    crate fn new(op: KleeneOp, span: Span) -> KleeneToken {
+        KleeneToken { span, op }
+    }
+}
+
+/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star)
+/// for token sequences.
+#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
+crate enum KleeneOp {
+    /// Kleene star (`*`) for zero or more repetitions
+    ZeroOrMore,
+    /// Kleene plus (`+`) for one or more repetitions
+    OneOrMore,
+    /// Kleene optional (`?`) for zero or one reptitions
+    ZeroOrOne,
+}
+
+/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
+/// are "first-class" token trees. Useful for parsing macros.
+#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)]
+crate enum TokenTree {
+    Token(Token),
+    Delimited(DelimSpan, Lrc<Delimited>),
+    /// A kleene-style repetition sequence
+    Sequence(DelimSpan, Lrc<SequenceRepetition>),
+    /// e.g., `$var`
+    MetaVar(Span, ast::Ident),
+    /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros.
+    MetaVarDecl(
+        Span,
+        ast::Ident, /* name to bind */
+        ast::Ident, /* kind of nonterminal */
+    ),
+}
+
+impl TokenTree {
+    /// Return the number of tokens in the tree.
+    crate fn len(&self) -> usize {
+        match *self {
+            TokenTree::Delimited(_, ref delimed) => match delimed.delim {
+                token::NoDelim => delimed.tts.len(),
+                _ => delimed.tts.len() + 2,
+            },
+            TokenTree::Sequence(_, ref seq) => seq.tts.len(),
+            _ => 0,
+        }
+    }
+
+    /// Returns `true` if the given token tree is delimited.
+    crate fn is_delimited(&self) -> bool {
+        match *self {
+            TokenTree::Delimited(..) => true,
+            _ => false,
+        }
+    }
+
+    /// Returns `true` if the given token tree is a token of the given kind.
+    crate fn is_token(&self, expected_kind: &TokenKind) -> bool {
+        match self {
+            TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind,
+            _ => false,
+        }
+    }
+
+    /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences.
+    crate fn get_tt(&self, index: usize) -> TokenTree {
+        match (self, index) {
+            (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => {
+                delimed.tts[index].clone()
+            }
+            (&TokenTree::Delimited(span, ref delimed), _) => {
+                if index == 0 {
+                    return delimed.open_tt(span.open);
+                }
+                if index == delimed.tts.len() + 1 {
+                    return delimed.close_tt(span.close);
+                }
+                delimed.tts[index - 1].clone()
+            }
+            (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(),
+            _ => panic!("Cannot expand a token tree"),
+        }
+    }
+
+    /// Retrieves the `TokenTree`'s span.
+    crate fn span(&self) -> Span {
+        match *self {
+            TokenTree::Token(Token { span, .. })
+            | TokenTree::MetaVar(span, _)
+            | TokenTree::MetaVarDecl(span, _, _) => span,
+            TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(),
+        }
+    }
+
+    crate fn token(kind: TokenKind, span: Span) -> TokenTree {
+        TokenTree::Token(Token::new(kind, span))
+    }
+}
diff --git a/src/libsyntax/ext/mbe/macro_check.rs b/src/libsyntax/ext/mbe/macro_check.rs
index aa88d71666e..a3750e5c469 100644
--- a/src/libsyntax/ext/mbe/macro_check.rs
+++ b/src/libsyntax/ext/mbe/macro_check.rs
@@ -106,7 +106,7 @@
 //! bound.
 use crate::ast::NodeId;
 use crate::early_buffered_lints::BufferedEarlyLintId;
-use crate::ext::mbe::quoted::{KleeneToken, TokenTree};
+use crate::ext::mbe::{KleeneToken, TokenTree};
 use crate::parse::token::TokenKind;
 use crate::parse::token::{DelimToken, Token};
 use crate::parse::ParseSess;
diff --git a/src/libsyntax/ext/mbe/macro_parser.rs b/src/libsyntax/ext/mbe/macro_parser.rs
index 88145cd121f..35fc6cf21b8 100644
--- a/src/libsyntax/ext/mbe/macro_parser.rs
+++ b/src/libsyntax/ext/mbe/macro_parser.rs
@@ -75,7 +75,7 @@ crate use ParseResult::*;
 use TokenTreeOrTokenTreeSlice::*;
 
 use crate::ast::{Ident, Name};
-use crate::ext::mbe::quoted::{self, TokenTree};
+use crate::ext::mbe::{self, TokenTree};
 use crate::parse::{Directory, ParseSess};
 use crate::parse::parser::{Parser, PathStyle};
 use crate::parse::token::{self, DocComment, Nonterminal, Token};
@@ -195,7 +195,7 @@ struct MatcherPos<'root, 'tt> {
     // `None`.
 
     /// The KleeneOp of this sequence if we are in a repetition.
-    seq_op: Option<quoted::KleeneOp>,
+    seq_op: Option<mbe::KleeneOp>,
 
     /// The separator if we are in a repetition.
     sep: Option<Token>,
@@ -532,7 +532,7 @@ fn inner_parse_loop<'root, 'tt>(
                 }
                 // We don't need a separator. Move the "dot" back to the beginning of the matcher
                 // and try to match again UNLESS we are only allowed to have _one_ repetition.
-                else if item.seq_op != Some(quoted::KleeneOp::ZeroOrOne) {
+                else if item.seq_op != Some(mbe::KleeneOp::ZeroOrOne) {
                     item.match_cur = item.match_lo;
                     item.idx = 0;
                     cur_items.push(item);
@@ -555,8 +555,8 @@ fn inner_parse_loop<'root, 'tt>(
                     // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will
                     // result in a "no rules expected token" error by virtue of this matcher not
                     // working.
-                    if seq.kleene.op == quoted::KleeneOp::ZeroOrMore
-                        || seq.kleene.op == quoted::KleeneOp::ZeroOrOne
+                    if seq.kleene.op == mbe::KleeneOp::ZeroOrMore
+                        || seq.kleene.op == mbe::KleeneOp::ZeroOrOne
                     {
                         let mut new_item = item.clone();
                         new_item.match_cur += seq.num_captures;
diff --git a/src/libsyntax/ext/mbe/macro_rules.rs b/src/libsyntax/ext/mbe/macro_rules.rs
index 939f5b8f51d..2ec171a7fb5 100644
--- a/src/libsyntax/ext/mbe/macro_rules.rs
+++ b/src/libsyntax/ext/mbe/macro_rules.rs
@@ -4,11 +4,11 @@ use crate::edition::Edition;
 use crate::ext::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander};
 use crate::ext::base::{SyntaxExtension, SyntaxExtensionKind};
 use crate::ext::expand::{AstFragment, AstFragmentKind};
+use crate::ext::mbe;
 use crate::ext::mbe::macro_check;
 use crate::ext::mbe::macro_parser::{parse, parse_failure_msg};
 use crate::ext::mbe::macro_parser::{Error, Failure, Success};
 use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq};
-use crate::ext::mbe::quoted;
 use crate::ext::mbe::transcribe::transcribe;
 use crate::feature_gate::Features;
 use crate::parse::parser::Parser;
@@ -135,8 +135,8 @@ struct MacroRulesMacroExpander {
     name: ast::Ident,
     span: Span,
     transparency: Transparency,
-    lhses: Vec<quoted::TokenTree>,
-    rhses: Vec<quoted::TokenTree>,
+    lhses: Vec<mbe::TokenTree>,
+    rhses: Vec<mbe::TokenTree>,
     valid: bool,
 }
 
@@ -169,8 +169,8 @@ fn generic_extension<'cx>(
     name: ast::Ident,
     transparency: Transparency,
     arg: TokenStream,
-    lhses: &[quoted::TokenTree],
-    rhses: &[quoted::TokenTree],
+    lhses: &[mbe::TokenTree],
+    rhses: &[mbe::TokenTree],
 ) -> Box<dyn MacResult + 'cx> {
     if cx.trace_macros() {
         trace_macros_note(cx, sp, format!("expanding `{}! {{ {} }}`", name, arg));
@@ -182,7 +182,7 @@ fn generic_extension<'cx>(
     for (i, lhs) in lhses.iter().enumerate() {
         // try each arm's matchers
         let lhs_tt = match *lhs {
-            quoted::TokenTree::Delimited(_, ref delim) => &delim.tts[..],
+            mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..],
             _ => cx.span_bug(sp, "malformed macro lhs"),
         };
 
@@ -190,7 +190,7 @@ fn generic_extension<'cx>(
             Success(named_matches) => {
                 let rhs = match rhses[i] {
                     // ignore delimiters
-                    quoted::TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(),
+                    mbe::TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(),
                     _ => cx.span_bug(sp, "malformed macro rhs"),
                 };
                 let arm_span = rhses[i].span();
@@ -258,7 +258,7 @@ fn generic_extension<'cx>(
         for lhs in lhses {
             // try each arm's matchers
             let lhs_tt = match *lhs {
-                quoted::TokenTree::Delimited(_, ref delim) => &delim.tts[..],
+                mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..],
                 _ => continue,
             };
             match TokenTree::parse(cx, lhs_tt, arg.clone()) {
@@ -312,32 +312,32 @@ pub fn compile_declarative_macro(
     // ...quasiquoting this would be nice.
     // These spans won't matter, anyways
     let argument_gram = vec![
-        quoted::TokenTree::Sequence(
+        mbe::TokenTree::Sequence(
             DelimSpan::dummy(),
-            Lrc::new(quoted::SequenceRepetition {
+            Lrc::new(mbe::SequenceRepetition {
                 tts: vec![
-                    quoted::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec),
-                    quoted::TokenTree::token(token::FatArrow, def.span),
-                    quoted::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec),
+                    mbe::TokenTree::MetaVarDecl(def.span, lhs_nm, tt_spec),
+                    mbe::TokenTree::token(token::FatArrow, def.span),
+                    mbe::TokenTree::MetaVarDecl(def.span, rhs_nm, tt_spec),
                 ],
                 separator: Some(Token::new(
                     if body.legacy { token::Semi } else { token::Comma },
                     def.span,
                 )),
-                kleene: quoted::KleeneToken::new(quoted::KleeneOp::OneOrMore, def.span),
+                kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, def.span),
                 num_captures: 2,
             }),
         ),
         // to phase into semicolon-termination instead of semicolon-separation
-        quoted::TokenTree::Sequence(
+        mbe::TokenTree::Sequence(
             DelimSpan::dummy(),
-            Lrc::new(quoted::SequenceRepetition {
-                tts: vec![quoted::TokenTree::token(
+            Lrc::new(mbe::SequenceRepetition {
+                tts: vec![mbe::TokenTree::token(
                     if body.legacy { token::Semi } else { token::Comma },
                     def.span,
                 )],
                 separator: None,
-                kleene: quoted::KleeneToken::new(quoted::KleeneOp::ZeroOrMore, def.span),
+                kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, def.span),
                 num_captures: 0,
             }),
         ),
@@ -367,7 +367,7 @@ pub fn compile_declarative_macro(
             .map(|m| {
                 if let MatchedNonterminal(ref nt) = *m {
                     if let NtTT(ref tt) = **nt {
-                        let tt = quoted::parse(
+                        let tt = mbe::quoted::parse(
                             tt.clone().into(),
                             true,
                             sess,
@@ -384,7 +384,7 @@ pub fn compile_declarative_macro(
                 }
                 sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs")
             })
-            .collect::<Vec<quoted::TokenTree>>(),
+            .collect::<Vec<mbe::TokenTree>>(),
         _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs"),
     };
 
@@ -394,7 +394,7 @@ pub fn compile_declarative_macro(
             .map(|m| {
                 if let MatchedNonterminal(ref nt) = *m {
                     if let NtTT(ref tt) = **nt {
-                        return quoted::parse(
+                        return mbe::quoted::parse(
                             tt.clone().into(),
                             false,
                             sess,
@@ -409,7 +409,7 @@ pub fn compile_declarative_macro(
                 }
                 sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs")
             })
-            .collect::<Vec<quoted::TokenTree>>(),
+            .collect::<Vec<mbe::TokenTree>>(),
         _ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs"),
     };
 
@@ -454,11 +454,11 @@ fn check_lhs_nt_follows(
     sess: &ParseSess,
     features: &Features,
     attrs: &[ast::Attribute],
-    lhs: &quoted::TokenTree,
+    lhs: &mbe::TokenTree,
 ) -> bool {
     // lhs is going to be like TokenTree::Delimited(...), where the
     // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens.
-    if let quoted::TokenTree::Delimited(_, ref tts) = *lhs {
+    if let mbe::TokenTree::Delimited(_, ref tts) = *lhs {
         check_matcher(sess, features, attrs, &tts.tts)
     } else {
         let msg = "invalid macro matcher; matchers must be contained in balanced delimiters";
@@ -471,8 +471,8 @@ fn check_lhs_nt_follows(
 
 /// Checks that the lhs contains no repetition which could match an empty token
 /// tree, because then the matcher would hang indefinitely.
-fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[quoted::TokenTree]) -> bool {
-    use quoted::TokenTree;
+fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool {
+    use mbe::TokenTree;
     for tt in tts {
         match *tt {
             TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (),
@@ -486,8 +486,8 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[quoted::TokenTree]) -> bool {
                     && seq.tts.iter().all(|seq_tt| match *seq_tt {
                         TokenTree::MetaVarDecl(_, _, id) => id.name == sym::vis,
                         TokenTree::Sequence(_, ref sub_seq) => {
-                            sub_seq.kleene.op == quoted::KleeneOp::ZeroOrMore
-                                || sub_seq.kleene.op == quoted::KleeneOp::ZeroOrOne
+                            sub_seq.kleene.op == mbe::KleeneOp::ZeroOrMore
+                                || sub_seq.kleene.op == mbe::KleeneOp::ZeroOrOne
                         }
                         _ => false,
                     })
@@ -506,9 +506,9 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[quoted::TokenTree]) -> bool {
     true
 }
 
-fn check_rhs(sess: &ParseSess, rhs: &quoted::TokenTree) -> bool {
+fn check_rhs(sess: &ParseSess, rhs: &mbe::TokenTree) -> bool {
     match *rhs {
-        quoted::TokenTree::Delimited(..) => return true,
+        mbe::TokenTree::Delimited(..) => return true,
         _ => sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited"),
     }
     false
@@ -518,7 +518,7 @@ fn check_matcher(
     sess: &ParseSess,
     features: &Features,
     attrs: &[ast::Attribute],
-    matcher: &[quoted::TokenTree],
+    matcher: &[mbe::TokenTree],
 ) -> bool {
     let first_sets = FirstSets::new(matcher);
     let empty_suffix = TokenSet::empty();
@@ -550,8 +550,8 @@ struct FirstSets {
 }
 
 impl FirstSets {
-    fn new(tts: &[quoted::TokenTree]) -> FirstSets {
-        use quoted::TokenTree;
+    fn new(tts: &[mbe::TokenTree]) -> FirstSets {
+        use mbe::TokenTree;
 
         let mut sets = FirstSets { first: FxHashMap::default() };
         build_recur(&mut sets, tts);
@@ -598,8 +598,8 @@ impl FirstSets {
 
                         // Reverse scan: Sequence comes before `first`.
                         if subfirst.maybe_empty
-                            || seq_rep.kleene.op == quoted::KleeneOp::ZeroOrMore
-                            || seq_rep.kleene.op == quoted::KleeneOp::ZeroOrOne
+                            || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore
+                            || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne
                         {
                             // If sequence is potentially empty, then
                             // union them (preserving first emptiness).
@@ -619,8 +619,8 @@ impl FirstSets {
 
     // walks forward over `tts` until all potential FIRST tokens are
     // identified.
-    fn first(&self, tts: &[quoted::TokenTree]) -> TokenSet {
-        use quoted::TokenTree;
+    fn first(&self, tts: &[mbe::TokenTree]) -> TokenSet {
+        use mbe::TokenTree;
 
         let mut first = TokenSet::empty();
         for tt in tts.iter() {
@@ -656,8 +656,8 @@ impl FirstSets {
                     assert!(first.maybe_empty);
                     first.add_all(subfirst);
                     if subfirst.maybe_empty
-                        || seq_rep.kleene.op == quoted::KleeneOp::ZeroOrMore
-                        || seq_rep.kleene.op == quoted::KleeneOp::ZeroOrOne
+                        || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrMore
+                        || seq_rep.kleene.op == mbe::KleeneOp::ZeroOrOne
                     {
                         // Continue scanning for more first
                         // tokens, but also make sure we
@@ -678,7 +678,7 @@ impl FirstSets {
     }
 }
 
-// A set of `quoted::TokenTree`s, which may include `TokenTree::Match`s
+// A set of `mbe::TokenTree`s, which may include `TokenTree::Match`s
 // (for macro-by-example syntactic variables). It also carries the
 // `maybe_empty` flag; that is true if and only if the matcher can
 // match an empty token sequence.
@@ -690,7 +690,7 @@ impl FirstSets {
 // (Notably, we must allow for *-op to occur zero times.)
 #[derive(Clone, Debug)]
 struct TokenSet {
-    tokens: Vec<quoted::TokenTree>,
+    tokens: Vec<mbe::TokenTree>,
     maybe_empty: bool,
 }
 
@@ -702,13 +702,13 @@ impl TokenSet {
 
     // Returns the set `{ tok }` for the single-token (and thus
     // non-empty) sequence [tok].
-    fn singleton(tok: quoted::TokenTree) -> Self {
+    fn singleton(tok: mbe::TokenTree) -> Self {
         TokenSet { tokens: vec![tok], maybe_empty: false }
     }
 
     // Changes self to be the set `{ tok }`.
     // Since `tok` is always present, marks self as non-empty.
-    fn replace_with(&mut self, tok: quoted::TokenTree) {
+    fn replace_with(&mut self, tok: mbe::TokenTree) {
         self.tokens.clear();
         self.tokens.push(tok);
         self.maybe_empty = false;
@@ -723,7 +723,7 @@ impl TokenSet {
     }
 
     // Adds `tok` to the set for `self`, marking sequence as non-empy.
-    fn add_one(&mut self, tok: quoted::TokenTree) {
+    fn add_one(&mut self, tok: mbe::TokenTree) {
         if !self.tokens.contains(&tok) {
             self.tokens.push(tok);
         }
@@ -731,7 +731,7 @@ impl TokenSet {
     }
 
     // Adds `tok` to the set for `self`. (Leaves `maybe_empty` flag alone.)
-    fn add_one_maybe(&mut self, tok: quoted::TokenTree) {
+    fn add_one_maybe(&mut self, tok: mbe::TokenTree) {
         if !self.tokens.contains(&tok) {
             self.tokens.push(tok);
         }
@@ -772,10 +772,10 @@ fn check_matcher_core(
     features: &Features,
     attrs: &[ast::Attribute],
     first_sets: &FirstSets,
-    matcher: &[quoted::TokenTree],
+    matcher: &[mbe::TokenTree],
     follow: &TokenSet,
 ) -> TokenSet {
-    use quoted::TokenTree;
+    use mbe::TokenTree;
 
     let mut last = TokenSet::empty();
 
@@ -950,8 +950,8 @@ fn check_matcher_core(
     last
 }
 
-fn token_can_be_followed_by_any(tok: &quoted::TokenTree) -> bool {
-    if let quoted::TokenTree::MetaVarDecl(_, _, frag_spec) = *tok {
+fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool {
+    if let mbe::TokenTree::MetaVarDecl(_, _, frag_spec) = *tok {
         frag_can_be_followed_by_any(frag_spec.name)
     } else {
         // (Non NT's can always be followed by anthing in matchers.)
@@ -997,8 +997,8 @@ enum IsInFollow {
 /// break macros that were relying on that binary operator as a
 /// separator.
 // when changing this do not forget to update doc/book/macros.md!
-fn is_in_follow(tok: &quoted::TokenTree, frag: Symbol) -> IsInFollow {
-    use quoted::TokenTree;
+fn is_in_follow(tok: &mbe::TokenTree, frag: Symbol) -> IsInFollow {
+    use mbe::TokenTree;
 
     if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok {
         // closing a token tree can never be matched by any fragment;
@@ -1116,10 +1116,10 @@ fn has_legal_fragment_specifier(
     sess: &ParseSess,
     features: &Features,
     attrs: &[ast::Attribute],
-    tok: &quoted::TokenTree,
+    tok: &mbe::TokenTree,
 ) -> Result<(), String> {
     debug!("has_legal_fragment_specifier({:?})", tok);
-    if let quoted::TokenTree::MetaVarDecl(_, _, ref frag_spec) = *tok {
+    if let mbe::TokenTree::MetaVarDecl(_, _, ref frag_spec) = *tok {
         let frag_span = tok.span();
         if !is_legal_fragment_specifier(sess, features, attrs, frag_spec.name, frag_span) {
             return Err(frag_spec.to_string());
@@ -1160,13 +1160,13 @@ fn is_legal_fragment_specifier(
     }
 }
 
-fn quoted_tt_to_string(tt: &quoted::TokenTree) -> String {
+fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
     match *tt {
-        quoted::TokenTree::Token(ref token) => crate::print::pprust::token_to_string(&token),
-        quoted::TokenTree::MetaVar(_, name) => format!("${}", name),
-        quoted::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind),
+        mbe::TokenTree::Token(ref token) => crate::print::pprust::token_to_string(&token),
+        mbe::TokenTree::MetaVar(_, name) => format!("${}", name),
+        mbe::TokenTree::MetaVarDecl(_, name, kind) => format!("${}:{}", name, kind),
         _ => panic!(
-            "unexpected quoted::TokenTree::{{Sequence or Delimited}} \
+            "unexpected mbe::TokenTree::{{Sequence or Delimited}} \
              in follow set checker"
         ),
     }
diff --git a/src/libsyntax/ext/mbe/quoted.rs b/src/libsyntax/ext/mbe/quoted.rs
index 8ef28345231..82289d7449d 100644
--- a/src/libsyntax/ext/mbe/quoted.rs
+++ b/src/libsyntax/ext/mbe/quoted.rs
@@ -1,166 +1,19 @@
 use crate::ast;
 use crate::ast::NodeId;
 use crate::ext::mbe::macro_parser;
+use crate::ext::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited};
 use crate::feature_gate::Features;
-use crate::parse::token::{self, Token, TokenKind};
+use crate::parse::token::{self, Token};
 use crate::parse::ParseSess;
 use crate::print::pprust;
 use crate::symbol::kw;
-use crate::tokenstream::{self, DelimSpan};
+use crate::tokenstream;
 
-use syntax_pos::{edition::Edition, BytePos, Span};
+use syntax_pos::{edition::Edition, Span};
 
 use rustc_data_structures::sync::Lrc;
 use std::iter::Peekable;
 
-/// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note
-/// that the delimiter itself might be `NoDelim`.
-#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)]
-crate struct Delimited {
-    crate delim: token::DelimToken,
-    crate tts: Vec<TokenTree>,
-}
-
-impl Delimited {
-    /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
-    crate fn open_tt(&self, span: Span) -> TokenTree {
-        let open_span = if span.is_dummy() {
-            span
-        } else {
-            span.with_hi(span.lo() + BytePos(self.delim.len() as u32))
-        };
-        TokenTree::token(token::OpenDelim(self.delim), open_span)
-    }
-
-    /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
-    crate fn close_tt(&self, span: Span) -> TokenTree {
-        let close_span = if span.is_dummy() {
-            span
-        } else {
-            span.with_lo(span.hi() - BytePos(self.delim.len() as u32))
-        };
-        TokenTree::token(token::CloseDelim(self.delim), close_span)
-    }
-}
-
-#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)]
-crate struct SequenceRepetition {
-    /// The sequence of token trees
-    crate tts: Vec<TokenTree>,
-    /// The optional separator
-    crate separator: Option<Token>,
-    /// Whether the sequence can be repeated zero (*), or one or more times (+)
-    crate kleene: KleeneToken,
-    /// The number of `Match`s that appear in the sequence (and subsequences)
-    crate num_captures: usize,
-}
-
-#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)]
-crate struct KleeneToken {
-    crate span: Span,
-    crate op: KleeneOp,
-}
-
-impl KleeneToken {
-    crate fn new(op: KleeneOp, span: Span) -> KleeneToken {
-        KleeneToken { span, op }
-    }
-}
-
-/// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star)
-/// for token sequences.
-#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
-crate enum KleeneOp {
-    /// Kleene star (`*`) for zero or more repetitions
-    ZeroOrMore,
-    /// Kleene plus (`+`) for one or more repetitions
-    OneOrMore,
-    /// Kleene optional (`?`) for zero or one reptitions
-    ZeroOrOne,
-}
-
-/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
-/// are "first-class" token trees. Useful for parsing macros.
-#[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)]
-crate enum TokenTree {
-    Token(Token),
-    Delimited(DelimSpan, Lrc<Delimited>),
-    /// A kleene-style repetition sequence
-    Sequence(DelimSpan, Lrc<SequenceRepetition>),
-    /// e.g., `$var`
-    MetaVar(Span, ast::Ident),
-    /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros.
-    MetaVarDecl(
-        Span,
-        ast::Ident, /* name to bind */
-        ast::Ident, /* kind of nonterminal */
-    ),
-}
-
-impl TokenTree {
-    /// Return the number of tokens in the tree.
-    crate fn len(&self) -> usize {
-        match *self {
-            TokenTree::Delimited(_, ref delimed) => match delimed.delim {
-                token::NoDelim => delimed.tts.len(),
-                _ => delimed.tts.len() + 2,
-            },
-            TokenTree::Sequence(_, ref seq) => seq.tts.len(),
-            _ => 0,
-        }
-    }
-
-    /// Returns `true` if the given token tree is delimited.
-    crate fn is_delimited(&self) -> bool {
-        match *self {
-            TokenTree::Delimited(..) => true,
-            _ => false,
-        }
-    }
-
-    /// Returns `true` if the given token tree is a token of the given kind.
-    crate fn is_token(&self, expected_kind: &TokenKind) -> bool {
-        match self {
-            TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind,
-            _ => false,
-        }
-    }
-
-    /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences.
-    crate fn get_tt(&self, index: usize) -> TokenTree {
-        match (self, index) {
-            (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => {
-                delimed.tts[index].clone()
-            }
-            (&TokenTree::Delimited(span, ref delimed), _) => {
-                if index == 0 {
-                    return delimed.open_tt(span.open);
-                }
-                if index == delimed.tts.len() + 1 {
-                    return delimed.close_tt(span.close);
-                }
-                delimed.tts[index - 1].clone()
-            }
-            (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(),
-            _ => panic!("Cannot expand a token tree"),
-        }
-    }
-
-    /// Retrieves the `TokenTree`'s span.
-    crate fn span(&self) -> Span {
-        match *self {
-            TokenTree::Token(Token { span, .. })
-            | TokenTree::MetaVar(span, _)
-            | TokenTree::MetaVarDecl(span, _, _) => span,
-            TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(),
-        }
-    }
-
-    crate fn token(kind: TokenKind, span: Span) -> TokenTree {
-        TokenTree::Token(Token::new(kind, span))
-    }
-}
-
 /// Takes a `tokenstream::TokenStream` and returns a `Vec<self::TokenTree>`. Specifically, this
 /// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a
 /// collection of `TokenTree` for use in parsing a macro.
diff --git a/src/libsyntax/ext/mbe/transcribe.rs b/src/libsyntax/ext/mbe/transcribe.rs
index 94ac1f60c64..ba818ebd35c 100644
--- a/src/libsyntax/ext/mbe/transcribe.rs
+++ b/src/libsyntax/ext/mbe/transcribe.rs
@@ -1,7 +1,7 @@
 use crate::ast::{Ident, Mac};
 use crate::ext::base::ExtCtxt;
+use crate::ext::mbe;
 use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
-use crate::ext::mbe::quoted;
 use crate::mut_visit::{self, MutVisitor};
 use crate::parse::token::{self, NtTT, Token};
 use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
@@ -38,22 +38,22 @@ impl Marker {
 
 /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
 enum Frame {
-    Delimited { forest: Lrc<quoted::Delimited>, idx: usize, span: DelimSpan },
-    Sequence { forest: Lrc<quoted::SequenceRepetition>, idx: usize, sep: Option<Token> },
+    Delimited { forest: Lrc<mbe::Delimited>, idx: usize, span: DelimSpan },
+    Sequence { forest: Lrc<mbe::SequenceRepetition>, idx: usize, sep: Option<Token> },
 }
 
 impl Frame {
     /// Construct a new frame around the delimited set of tokens.
-    fn new(tts: Vec<quoted::TokenTree>) -> Frame {
-        let forest = Lrc::new(quoted::Delimited { delim: token::NoDelim, tts });
+    fn new(tts: Vec<mbe::TokenTree>) -> Frame {
+        let forest = Lrc::new(mbe::Delimited { delim: token::NoDelim, tts });
         Frame::Delimited { forest, idx: 0, span: DelimSpan::dummy() }
     }
 }
 
 impl Iterator for Frame {
-    type Item = quoted::TokenTree;
+    type Item = mbe::TokenTree;
 
-    fn next(&mut self) -> Option<quoted::TokenTree> {
+    fn next(&mut self) -> Option<mbe::TokenTree> {
         match *self {
             Frame::Delimited { ref forest, ref mut idx, .. } => {
                 *idx += 1;
@@ -90,7 +90,7 @@ impl Iterator for Frame {
 pub(super) fn transcribe(
     cx: &ExtCtxt<'_>,
     interp: &FxHashMap<Ident, NamedMatch>,
-    src: Vec<quoted::TokenTree>,
+    src: Vec<mbe::TokenTree>,
     transparency: Transparency,
 ) -> TokenStream {
     // Nothing for us to transcribe...
@@ -178,7 +178,7 @@ pub(super) fn transcribe(
             // We are descending into a sequence. We first make sure that the matchers in the RHS
             // and the matches in `interp` have the same shape. Otherwise, either the caller or the
             // macro writer has made a mistake.
-            seq @ quoted::TokenTree::Sequence(..) => {
+            seq @ mbe::TokenTree::Sequence(..) => {
                 match lockstep_iter_size(&seq, interp, &repeats) {
                     LockstepIterSize::Unconstrained => {
                         cx.span_fatal(
@@ -199,7 +199,7 @@ pub(super) fn transcribe(
                     LockstepIterSize::Constraint(len, _) => {
                         // We do this to avoid an extra clone above. We know that this is a
                         // sequence already.
-                        let (sp, seq) = if let quoted::TokenTree::Sequence(sp, seq) = seq {
+                        let (sp, seq) = if let mbe::TokenTree::Sequence(sp, seq) = seq {
                             (sp, seq)
                         } else {
                             unreachable!()
@@ -207,7 +207,7 @@ pub(super) fn transcribe(
 
                         // Is the repetition empty?
                         if len == 0 {
-                            if seq.kleene.op == quoted::KleeneOp::OneOrMore {
+                            if seq.kleene.op == mbe::KleeneOp::OneOrMore {
                                 // FIXME: this really ought to be caught at macro definition
                                 // time... It happens when the Kleene operator in the matcher and
                                 // the body for the same meta-variable do not match.
@@ -232,7 +232,7 @@ pub(super) fn transcribe(
             }
 
             // Replace the meta-var with the matched token tree from the invocation.
-            quoted::TokenTree::MetaVar(mut sp, mut ident) => {
+            mbe::TokenTree::MetaVar(mut sp, mut ident) => {
                 // Find the matched nonterminal from the macro invocation, and use it to replace
                 // the meta-var.
                 if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
@@ -269,7 +269,7 @@ pub(super) fn transcribe(
             // We will produce all of the results of the inside of the `Delimited` and then we will
             // jump back out of the Delimited, pop the result_stack and add the new results back to
             // the previous results (from outside the Delimited).
-            quoted::TokenTree::Delimited(mut span, delimited) => {
+            mbe::TokenTree::Delimited(mut span, delimited) => {
                 marker.visit_delim_span(&mut span);
                 stack.push(Frame::Delimited { forest: delimited, idx: 0, span });
                 result_stack.push(mem::take(&mut result));
@@ -277,14 +277,14 @@ pub(super) fn transcribe(
 
             // Nothing much to do here. Just push the token to the result, being careful to
             // preserve syntax context.
-            quoted::TokenTree::Token(token) => {
+            mbe::TokenTree::Token(token) => {
                 let mut tt = TokenTree::Token(token);
                 marker.visit_tt(&mut tt);
                 result.push(tt.into());
             }
 
             // There should be no meta-var declarations in the invocation of a macro.
-            quoted::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"),
+            mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"),
         }
     }
 }
@@ -368,11 +368,11 @@ impl LockstepIterSize {
 /// `lookup_cur_matched` will return `None`, which is why this still works even in the presnece of
 /// multiple nested matcher sequences.
 fn lockstep_iter_size(
-    tree: &quoted::TokenTree,
+    tree: &mbe::TokenTree,
     interpolations: &FxHashMap<Ident, NamedMatch>,
     repeats: &[(usize, usize)],
 ) -> LockstepIterSize {
-    use quoted::TokenTree;
+    use mbe::TokenTree;
     match *tree {
         TokenTree::Delimited(_, ref delimed) => {
             delimed.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| {
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index d46e38f53d5..fd87f8a1f67 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -14,7 +14,7 @@
 //! ownership of the original.
 
 use crate::ext::base;
-use crate::ext::mbe::{macro_parser, quoted};
+use crate::ext::mbe::{self, macro_parser};
 use crate::parse::Directory;
 use crate::parse::token::{self, DelimToken, Token, TokenKind};
 use crate::print::pprust;
@@ -64,7 +64,7 @@ where
 
 impl TokenTree {
     /// Use this token tree as a matcher to parse given tts.
-    crate fn parse(cx: &base::ExtCtxt<'_>, mtch: &[quoted::TokenTree], tts: TokenStream)
+    crate fn parse(cx: &base::ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream)
                  -> macro_parser::NamedParseResult {
         // `None` is because we're not interpolating
         let directory = Directory {

From 983569732d24df434c644dd1764e9c5e4ecfd081 Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 22 Sep 2019 19:41:04 +0300
Subject: [PATCH 7/8] push TokenTree::parse down

---
 src/libsyntax/ext/mbe/macro_rules.rs | 15 ++++++++++++++-
 src/libsyntax/tokenstream.rs         | 15 ---------------
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/src/libsyntax/ext/mbe/macro_rules.rs b/src/libsyntax/ext/mbe/macro_rules.rs
index 2ec171a7fb5..816baadb12f 100644
--- a/src/libsyntax/ext/mbe/macro_rules.rs
+++ b/src/libsyntax/ext/mbe/macro_rules.rs
@@ -8,7 +8,7 @@ use crate::ext::mbe;
 use crate::ext::mbe::macro_check;
 use crate::ext::mbe::macro_parser::{parse, parse_failure_msg};
 use crate::ext::mbe::macro_parser::{Error, Failure, Success};
-use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq};
+use crate::ext::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult};
 use crate::ext::mbe::transcribe::transcribe;
 use crate::feature_gate::Features;
 use crate::parse::parser::Parser;
@@ -1171,3 +1171,16 @@ fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
         ),
     }
 }
+
+impl TokenTree {
+    /// Use this token tree as a matcher to parse given tts.
+    fn parse(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream)
+             -> NamedParseResult {
+        // `None` is because we're not interpolating
+        let directory = Directory {
+            path: Cow::from(cx.current_expansion.module.directory.as_path()),
+            ownership: cx.current_expansion.directory_ownership,
+        };
+        parse(cx.parse_sess(), tts, mtch, Some(directory), true)
+    }
+}
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index fd87f8a1f67..26cae2a8e7c 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -13,9 +13,6 @@
 //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
 //! ownership of the original.
 
-use crate::ext::base;
-use crate::ext::mbe::{self, macro_parser};
-use crate::parse::Directory;
 use crate::parse::token::{self, DelimToken, Token, TokenKind};
 use crate::print::pprust;
 
@@ -26,7 +23,6 @@ use rustc_data_structures::sync::Lrc;
 use rustc_serialize::{Decoder, Decodable, Encoder, Encodable};
 use smallvec::{SmallVec, smallvec};
 
-use std::borrow::Cow;
 use std::{fmt, iter, mem};
 
 #[cfg(test)]
@@ -63,17 +59,6 @@ where
 {}
 
 impl TokenTree {
-    /// Use this token tree as a matcher to parse given tts.
-    crate fn parse(cx: &base::ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream)
-                 -> macro_parser::NamedParseResult {
-        // `None` is because we're not interpolating
-        let directory = Directory {
-            path: Cow::from(cx.current_expansion.module.directory.as_path()),
-            ownership: cx.current_expansion.directory_ownership,
-        };
-        macro_parser::parse(cx.parse_sess(), tts, mtch, Some(directory), true)
-    }
-
     /// Checks if this TokenTree is equal to the other, regardless of span information.
     pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
         match (self, other) {

From 81fe85710d7f749c87494c4b968861adc67a9c4a Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 22 Sep 2019 19:42:52 +0300
Subject: [PATCH 8/8] make mbe::TokenTree private to module

---
 src/libsyntax/ext/mbe.rs              | 44 +++++++++++++--------------
 src/libsyntax/ext/mbe/macro_check.rs  |  2 +-
 src/libsyntax/ext/mbe/macro_parser.rs |  4 +--
 src/libsyntax/ext/mbe/quoted.rs       |  2 +-
 4 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/src/libsyntax/ext/mbe.rs b/src/libsyntax/ext/mbe.rs
index c18b8918ff1..a87da791c9b 100644
--- a/src/libsyntax/ext/mbe.rs
+++ b/src/libsyntax/ext/mbe.rs
@@ -20,14 +20,14 @@ use rustc_data_structures::sync::Lrc;
 /// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note
 /// that the delimiter itself might be `NoDelim`.
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)]
-crate struct Delimited {
-    crate delim: token::DelimToken,
-    crate tts: Vec<TokenTree>,
+struct Delimited {
+    delim: token::DelimToken,
+    tts: Vec<TokenTree>,
 }
 
 impl Delimited {
     /// Returns a `self::TokenTree` with a `Span` corresponding to the opening delimiter.
-    crate fn open_tt(&self, span: Span) -> TokenTree {
+    fn open_tt(&self, span: Span) -> TokenTree {
         let open_span = if span.is_dummy() {
             span
         } else {
@@ -37,7 +37,7 @@ impl Delimited {
     }
 
     /// Returns a `self::TokenTree` with a `Span` corresponding to the closing delimiter.
-    crate fn close_tt(&self, span: Span) -> TokenTree {
+    fn close_tt(&self, span: Span) -> TokenTree {
         let close_span = if span.is_dummy() {
             span
         } else {
@@ -48,25 +48,25 @@ impl Delimited {
 }
 
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)]
-crate struct SequenceRepetition {
+struct SequenceRepetition {
     /// The sequence of token trees
-    crate tts: Vec<TokenTree>,
+    tts: Vec<TokenTree>,
     /// The optional separator
-    crate separator: Option<Token>,
+    separator: Option<Token>,
     /// Whether the sequence can be repeated zero (*), or one or more times (+)
-    crate kleene: KleeneToken,
+    kleene: KleeneToken,
     /// The number of `Match`s that appear in the sequence (and subsequences)
-    crate num_captures: usize,
+    num_captures: usize,
 }
 
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, Copy)]
-crate struct KleeneToken {
-    crate span: Span,
-    crate op: KleeneOp,
+struct KleeneToken {
+    span: Span,
+    op: KleeneOp,
 }
 
 impl KleeneToken {
-    crate fn new(op: KleeneOp, span: Span) -> KleeneToken {
+    fn new(op: KleeneOp, span: Span) -> KleeneToken {
         KleeneToken { span, op }
     }
 }
@@ -74,7 +74,7 @@ impl KleeneToken {
 /// A Kleene-style [repetition operator](http://en.wikipedia.org/wiki/Kleene_star)
 /// for token sequences.
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
-crate enum KleeneOp {
+enum KleeneOp {
     /// Kleene star (`*`) for zero or more repetitions
     ZeroOrMore,
     /// Kleene plus (`+`) for one or more repetitions
@@ -86,7 +86,7 @@ crate enum KleeneOp {
 /// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
 /// are "first-class" token trees. Useful for parsing macros.
 #[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)]
-crate enum TokenTree {
+enum TokenTree {
     Token(Token),
     Delimited(DelimSpan, Lrc<Delimited>),
     /// A kleene-style repetition sequence
@@ -103,7 +103,7 @@ crate enum TokenTree {
 
 impl TokenTree {
     /// Return the number of tokens in the tree.
-    crate fn len(&self) -> usize {
+    fn len(&self) -> usize {
         match *self {
             TokenTree::Delimited(_, ref delimed) => match delimed.delim {
                 token::NoDelim => delimed.tts.len(),
@@ -115,7 +115,7 @@ impl TokenTree {
     }
 
     /// Returns `true` if the given token tree is delimited.
-    crate fn is_delimited(&self) -> bool {
+    fn is_delimited(&self) -> bool {
         match *self {
             TokenTree::Delimited(..) => true,
             _ => false,
@@ -123,7 +123,7 @@ impl TokenTree {
     }
 
     /// Returns `true` if the given token tree is a token of the given kind.
-    crate fn is_token(&self, expected_kind: &TokenKind) -> bool {
+    fn is_token(&self, expected_kind: &TokenKind) -> bool {
         match self {
             TokenTree::Token(Token { kind: actual_kind, .. }) => actual_kind == expected_kind,
             _ => false,
@@ -131,7 +131,7 @@ impl TokenTree {
     }
 
     /// Gets the `index`-th sub-token-tree. This only makes sense for delimited trees and sequences.
-    crate fn get_tt(&self, index: usize) -> TokenTree {
+    fn get_tt(&self, index: usize) -> TokenTree {
         match (self, index) {
             (&TokenTree::Delimited(_, ref delimed), _) if delimed.delim == token::NoDelim => {
                 delimed.tts[index].clone()
@@ -151,7 +151,7 @@ impl TokenTree {
     }
 
     /// Retrieves the `TokenTree`'s span.
-    crate fn span(&self) -> Span {
+    fn span(&self) -> Span {
         match *self {
             TokenTree::Token(Token { span, .. })
             | TokenTree::MetaVar(span, _)
@@ -160,7 +160,7 @@ impl TokenTree {
         }
     }
 
-    crate fn token(kind: TokenKind, span: Span) -> TokenTree {
+    fn token(kind: TokenKind, span: Span) -> TokenTree {
         TokenTree::Token(Token::new(kind, span))
     }
 }
diff --git a/src/libsyntax/ext/mbe/macro_check.rs b/src/libsyntax/ext/mbe/macro_check.rs
index a3750e5c469..97074f5cbe4 100644
--- a/src/libsyntax/ext/mbe/macro_check.rs
+++ b/src/libsyntax/ext/mbe/macro_check.rs
@@ -196,7 +196,7 @@ struct MacroState<'a> {
 /// - `node_id` is used to emit lints
 /// - `span` is used when no spans are available
 /// - `lhses` and `rhses` should have the same length and represent the macro definition
-crate fn check_meta_variables(
+pub(super) fn check_meta_variables(
     sess: &ParseSess,
     node_id: NodeId,
     span: Span,
diff --git a/src/libsyntax/ext/mbe/macro_parser.rs b/src/libsyntax/ext/mbe/macro_parser.rs
index 35fc6cf21b8..b51384d3b15 100644
--- a/src/libsyntax/ext/mbe/macro_parser.rs
+++ b/src/libsyntax/ext/mbe/macro_parser.rs
@@ -282,7 +282,7 @@ crate enum ParseResult<T> {
 crate type NamedParseResult = ParseResult<FxHashMap<Ident, NamedMatch>>;
 
 /// Count how many metavars are named in the given matcher `ms`.
-crate fn count_names(ms: &[TokenTree]) -> usize {
+pub(super) fn count_names(ms: &[TokenTree]) -> usize {
     ms.iter().fold(0, |count, elt| {
         count + match *elt {
             TokenTree::Sequence(_, ref seq) => seq.num_captures,
@@ -648,7 +648,7 @@ fn inner_parse_loop<'root, 'tt>(
 /// - `directory`: Information about the file locations (needed for the black-box parser)
 /// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box
 ///   parser)
-crate fn parse(
+pub(super) fn parse(
     sess: &ParseSess,
     tts: TokenStream,
     ms: &[TokenTree],
diff --git a/src/libsyntax/ext/mbe/quoted.rs b/src/libsyntax/ext/mbe/quoted.rs
index 82289d7449d..3952e29a5f0 100644
--- a/src/libsyntax/ext/mbe/quoted.rs
+++ b/src/libsyntax/ext/mbe/quoted.rs
@@ -35,7 +35,7 @@ use std::iter::Peekable;
 /// # Returns
 ///
 /// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`.
-crate fn parse(
+pub(super) fn parse(
     input: tokenstream::TokenStream,
     expect_matchers: bool,
     sess: &ParseSess,