Use a proc macro to declare preallocated symbols

Cargo.lock

@@ -3387,6 +3387,7 @@ dependencies = [
  "arena 0.0.0",
  "cfg-if 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc_data_structures 0.0.0",
+ "rustc_macros 0.1.0",
  "scoped-tls 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serialize 0.0.0",
  "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",

src/librustc_macros/src/lib.rs

@@ -9,10 +9,16 @@ use proc_macro::TokenStream;
 
 mod hash_stable;
 mod query;
+mod symbols;
 
 #[proc_macro]
 pub fn rustc_queries(input: TokenStream) -> TokenStream {
     query::rustc_queries(input)
 }
 
+#[proc_macro]
+pub fn symbols(input: TokenStream) -> TokenStream {
+    symbols::symbols(input)
+}
+
 decl_derive!([HashStable, attributes(stable_hasher)] => hash_stable::hash_stable_derive);

src/librustc_macros/src/symbols.rs

@@ -0,0 +1,173 @@
+use proc_macro::TokenStream;
+use syn::{
+    Token, Ident, LitStr,
+    braced, parse_macro_input,
+};
+use syn::parse::{Result, Parse, ParseStream};
+use syn;
+use std::collections::HashSet;
+use quote::quote;
+
+#[allow(non_camel_case_types)]
+mod kw {
+    syn::custom_keyword!(Keywords);
+    syn::custom_keyword!(Other);
+}
+
+struct Keyword {
+    name: Ident,
+    value: LitStr,
+}
+
+impl Parse for Keyword {
+    fn parse(input: ParseStream<'_>) -> Result<Self> {
+        let name = input.parse()?;
+        input.parse::<Token![,]>()?;
+        let value = input.parse()?;
+        input.parse::<Token![,]>()?;
+
+        Ok(Keyword {
+            name,
+            value,
+        })
+    }
+}
+
+struct Symbol(Ident);
+
+impl Parse for Symbol {
+    fn parse(input: ParseStream<'_>) -> Result<Self> {
+        let ident: Ident = input.parse()?;
+        input.parse::<Token![,]>()?;
+
+        Ok(Symbol(ident))
+    }
+}
+
+/// A type used to greedily parse another type until the input is empty.
+struct List<T>(Vec<T>);
+
+impl<T: Parse> Parse for List<T> {
+    fn parse(input: ParseStream<'_>) -> Result<Self> {
+        let mut list = Vec::new();
+        while !input.is_empty() {
+            list.push(input.parse()?);
+        }
+        Ok(List(list))
+    }
+}
+
+struct Input {
+    keywords: List<Keyword>,
+    symbols: List<Symbol>,
+}
+
+impl Parse for Input {
+    fn parse(input: ParseStream<'_>) -> Result<Self> {
+        input.parse::<kw::Keywords>()?;
+        let content;
+        braced!(content in input);
+        let keywords = content.parse()?;
+
+        input.parse::<kw::Other>()?;
+        let content;
+        braced!(content in input);
+        let symbols = content.parse()?;
+
+        Ok(Input {
+            keywords,
+            symbols,
+        })
+    }
+}
+
+pub fn symbols(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as Input);
+
+    let mut keyword_stream = quote! {};
+    let mut symbols_stream = quote! {};
+    let mut prefill_stream = quote! {};
+    let mut from_str_stream = quote! {};
+    let mut counter = 0u32;
+    let mut keys = HashSet::<String>::new();
+
+    let mut check_dup = |str: &str| {
+        if !keys.insert(str.to_string()) {
+            panic!("Symbol `{}` is duplicated", str);
+        }
+    };
+
+    for keyword in &input.keywords.0 {
+        let name = &keyword.name;
+        let value = &keyword.value;
+        check_dup(&value.value());
+        prefill_stream.extend(quote! {
+            #value,
+        });
+        keyword_stream.extend(quote! {
+            pub const #name: Keyword = Keyword {
+                ident: Ident::with_empty_ctxt(super::Symbol::new(#counter))
+            };
+        });
+        from_str_stream.extend(quote! {
+            #value => Ok(#name),
+        });
+        counter += 1;
+    }
+
+    for symbol in &input.symbols.0 {
+        let value = &symbol.0;
+        let value_str = value.to_string();
+        check_dup(&value_str);
+        prefill_stream.extend(quote! {
+            #value_str,
+        });
+        symbols_stream.extend(quote! {
+            pub const #value: Symbol = Symbol::new(#counter);
+        });
+        counter += 1;
+    }
+
+    TokenStream::from(quote! {
+        #[allow(non_upper_case_globals)]
+        pub mod keywords {
+            use super::{Symbol, Ident};
+            #[derive(Clone, Copy, PartialEq, Eq)]
+            pub struct Keyword {
+                ident: Ident,
+            }
+            impl Keyword {
+                #[inline] pub fn ident(self) -> Ident { self.ident }
+                #[inline] pub fn name(self) -> Symbol { self.ident.name }
+            }
+
+            #keyword_stream
+
+            impl std::str::FromStr for Keyword {
+                type Err = ();
+
+                fn from_str(s: &str) -> Result<Self, ()> {
+                    match s {
+                        #from_str_stream
+                        _ => Err(()),
+                    }
+                }
+            }
+        }
+
+        #[allow(non_upper_case_globals)]
+        pub mod symbols {
+            use super::Symbol;
+
+            #symbols_stream
+        }
+
+        impl Interner {
+            pub fn fresh() -> Self {
+                Interner::prefill(&[
+                    #prefill_stream
+                ])
+            }
+        }
+    })
+}

src/libsyntax/ast.rs

@@ -68,6 +68,12 @@ pub struct Path {
     pub segments: Vec<PathSegment>,
 }
 
+impl PartialEq<Symbol> for Path {
+    fn eq(&self, symbol: &Symbol) -> bool {
+        self.segments.len() == 1 && self.segments[0].ident.name.interned() == *symbol
+    }
+}
+
 impl<'a> PartialEq<&'a str> for Path {
     fn eq(&self, string: &&'a str) -> bool {
         self.segments.len() == 1 && self.segments[0].ident.name == *string

src/libsyntax/attr/mod.rs

@@ -85,6 +85,11 @@ impl NestedMetaItem {
         self.meta_item().map_or(false, |meta_item| meta_item.check_name(name))
     }
 
+    /// Returns `true` if this list item is a MetaItem with a name of `name`.
+    pub fn check_name_symbol(&self, name: Symbol) -> bool {
+        self.meta_item().map_or(false, |meta_item| meta_item.check_name_symbol(name))
+    }
+
     /// For a single-segment meta-item returns its name, otherwise returns `None`.
     pub fn ident(&self) -> Option<Ident> {
         self.meta_item().and_then(|meta_item| meta_item.ident())
@@ -159,6 +164,18 @@ impl Attribute {
         matches
     }
 
+    /// Returns `true` if the attribute's path matches the argument. If it matches, then the
+    /// attribute is marked as used.
+    ///
+    /// To check the attribute name without marking it used, use the `path` field directly.
+    pub fn check_name_symbol(&self, name: Symbol) -> bool {
+        let matches = self.path == name;
+        if matches {
+            mark_used(self);
+        }
+        matches
+    }
+
     /// For a single-segment attribute returns its name, otherwise returns `None`.
     pub fn ident(&self) -> Option<Ident> {
         if self.path.segments.len() == 1 {
@@ -248,6 +265,10 @@ impl MetaItem {
         self.path == name
     }
 
+    pub fn check_name_symbol(&self, name: Symbol) -> bool {
+        self.path == name
+    }
+
     pub fn is_value_str(&self) -> bool {
         self.value_str().is_some()
     }

src/libsyntax/feature_gate.rs

@@ -28,7 +28,7 @@ use crate::tokenstream::TokenTree;
 use errors::{DiagnosticBuilder, Handler};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_target::spec::abi::Abi;
-use syntax_pos::{Span, DUMMY_SP};
+use syntax_pos::{Span, DUMMY_SP, symbols};
 use log::debug;
 
 use std::env;
@@ -1366,7 +1366,7 @@ impl<'a> Context<'a> {
                     }
                 } else if n == "doc" {
                     if let Some(content) = attr.meta_item_list() {
-                        if content.iter().any(|c| c.check_name("include")) {
+                        if content.iter().any(|c| c.check_name_symbol(symbols::include)) {
                             gate_feature!(self, external_doc, attr.span,
                                 "#[doc(include = \"...\")] is experimental"
                             );
@@ -1667,25 +1667,25 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
         // check for gated attributes
         self.context.check_attribute(attr, false);
 
-        if attr.check_name("doc") {
+        if attr.check_name_symbol(symbols::doc) {
             if let Some(content) = attr.meta_item_list() {
-                if content.len() == 1 && content[0].check_name("cfg") {
+                if content.len() == 1 && content[0].check_name_symbol(symbols::cfg) {
                     gate_feature_post!(&self, doc_cfg, attr.span,
                         "#[doc(cfg(...))] is experimental"
                     );
-                } else if content.iter().any(|c| c.check_name("masked")) {
+                } else if content.iter().any(|c| c.check_name_symbol(symbols::masked)) {
                     gate_feature_post!(&self, doc_masked, attr.span,
                         "#[doc(masked)] is experimental"
                     );
-                } else if content.iter().any(|c| c.check_name("spotlight")) {
+                } else if content.iter().any(|c| c.check_name_symbol(symbols::spotlight)) {
                     gate_feature_post!(&self, doc_spotlight, attr.span,
                         "#[doc(spotlight)] is experimental"
                     );
-                } else if content.iter().any(|c| c.check_name("alias")) {
+                } else if content.iter().any(|c| c.check_name_symbol(symbols::alias)) {
                     gate_feature_post!(&self, doc_alias, attr.span,
                         "#[doc(alias = \"...\")] is experimental"
                     );
-                } else if content.iter().any(|c| c.check_name("keyword")) {
+                } else if content.iter().any(|c| c.check_name_symbol(symbols::keyword)) {
                     gate_feature_post!(&self, doc_keyword, attr.span,
                         "#[doc(keyword = \"...\")] is experimental"
                     );
@@ -1693,7 +1693,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             }
         }
 
-        match BUILTIN_ATTRIBUTES.iter().find(|(name, ..)| attr.path == name) {
+        match BUILTIN_ATTRIBUTES.iter().find(|(name, ..)| attr.path == *name) {
             Some(&(name, _, template, _)) => self.check_builtin_attribute(attr, name, template),
             None => if let Some(TokenTree::Token(_, token::Eq)) = attr.tokens.trees().next() {
                 // All key-value attributes are restricted to meta-item syntax.
@@ -1748,7 +1748,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             ast::ItemKind::Struct(..) => {
                 for attr in attr::filter_by_name(&i.attrs[..], "repr") {
                     for item in attr.meta_item_list().unwrap_or_else(Vec::new) {
-                        if item.check_name("simd") {
+                        if item.check_name_symbol(symbols::simd) {
                             gate_feature_post!(&self, repr_simd, attr.span,
                                                "SIMD types are experimental and possibly buggy");
                         }
@@ -1759,7 +1759,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             ast::ItemKind::Enum(..) => {
                 for attr in attr::filter_by_name(&i.attrs[..], "repr") {
                     for item in attr.meta_item_list().unwrap_or_else(Vec::new) {
-                        if item.check_name("align") {
+                        if item.check_name_symbol(symbols::align) {
                             gate_feature_post!(&self, repr_align_enum, attr.span,
                                                "`#[repr(align(x))]` on enums is experimental");
                         }
@@ -2083,7 +2083,7 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute],
     // Process the edition umbrella feature-gates first, to ensure
     // `edition_enabled_features` is completed before it's queried.
     for attr in krate_attrs {
-        if !attr.check_name("feature") {
+        if !attr.check_name_symbol(symbols::feature) {
             continue
         }
 
@@ -2128,7 +2128,7 @@ pub fn get_features(span_handler: &Handler, krate_attrs: &[ast::Attribute],
     }
 
     for attr in krate_attrs {
-        if !attr.check_name("feature") {
+        if !attr.check_name_symbol(symbols::feature) {
             continue
         }
 
@@ -2258,7 +2258,7 @@ fn maybe_stage_features(span_handler: &Handler, krate: &ast::Crate,
     };
     if !allow_features {
         for attr in &krate.attrs {
-            if attr.check_name("feature") {
+            if attr.check_name_symbol(symbols::feature) {
                 let release_channel = option_env!("CFG_RELEASE_CHANNEL").unwrap_or("(unknown)");
                 span_err!(span_handler, attr.span, E0554,
                           "#![feature] may not be used on the {} release channel",

src/libsyntax_pos/Cargo.toml

@@ -11,6 +11,7 @@ crate-type = ["dylib"]
 
 [dependencies]
 serialize = { path = "../libserialize" }
+rustc_macros = { path = "../librustc_macros" }
 rustc_data_structures = { path = "../librustc_data_structures" }
 arena = { path = "../libarena" }
 scoped-tls = "1.0"

src/libsyntax_pos/lib.rs

@@ -16,6 +16,7 @@
 #![feature(non_exhaustive)]
 #![feature(optin_builtin_traits)]
 #![feature(rustc_attrs)]
+#![feature(proc_macro_hygiene)]
 #![feature(specialization)]
 #![feature(step_trait)]
 
@@ -32,6 +33,7 @@ mod span_encoding;
 pub use span_encoding::{Span, DUMMY_SP};
 
 pub mod symbol;
+pub use symbol::symbols;
 
 mod analyze_source_file;
 

src/libsyntax_pos/symbol.rs

@@ -6,6 +6,7 @@ use arena::DroplessArena;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::indexed_vec::Idx;
 use rustc_data_structures::newtype_index;
+use rustc_macros::symbols;
 use serialize::{Decodable, Decoder, Encodable, Encoder};
 
 use std::fmt;
@@ -16,6 +17,93 @@ use std::hash::{Hash, Hasher};
 use crate::hygiene::SyntaxContext;
 use crate::{Span, DUMMY_SP, GLOBALS};
 
+symbols! {
+    // After modifying this list adjust `is_special`, `is_used_keyword`/`is_unused_keyword`,
+    // this should be rarely necessary though if the keywords are kept in alphabetic order.
+    Keywords {
+        // Special reserved identifiers used internally for elided lifetimes,
+        // unnamed method parameters, crate root module, error recovery etc.
+        Invalid,            "",
+        PathRoot,           "{{root}}",
+        DollarCrate,        "$crate",
+        Underscore,         "_",
+
+        // Keywords that are used in stable Rust.
+        As,                 "as",
+        Box,                "box",
+        Break,              "break",
+        Const,              "const",
+        Continue,           "continue",
+        Crate,              "crate",
+        Else,               "else",
+        Enum,               "enum",
+        Extern,             "extern",
+        False,              "false",
+        Fn,                 "fn",
+        For,                "for",
+        If,                 "if",
+        Impl,               "impl",
+        In,                 "in",
+        Let,                "let",
+        Loop,               "loop",
+        Match,              "match",
+        Mod,                "mod",
+        Move,               "move",
+        Mut,                "mut",
+        Pub,                "pub",
+        Ref,                "ref",
+        Return,             "return",
+        SelfLower,          "self",
+        SelfUpper,          "Self",
+        Static,             "static",
+        Struct,             "struct",
+        Super,              "super",
+        Trait,              "trait",
+        True,               "true",
+        Type,               "type",
+        Unsafe,             "unsafe",
+        Use,                "use",
+        Where,              "where",
+        While,              "while",
+
+        // Keywords that are used in unstable Rust or reserved for future use.
+        Abstract,           "abstract",
+        Become,             "become",
+        Do,                 "do",
+        Final,              "final",
+        Macro,              "macro",
+        Override,           "override",
+        Priv,               "priv",
+        Typeof,             "typeof",
+        Unsized,            "unsized",
+        Virtual,            "virtual",
+        Yield,              "yield",
+
+        // Edition-specific keywords that are used in stable Rust.
+        Dyn,                "dyn", // >= 2018 Edition only
+
+        // Edition-specific keywords that are used in unstable Rust or reserved for future use.
+        Async,              "async", // >= 2018 Edition only
+        Try,                "try", // >= 2018 Edition only
+
+        // Special lifetime names
+        UnderscoreLifetime, "'_",
+        StaticLifetime,     "'static",
+
+        // Weak keywords, have special meaning only in specific contexts.
+        Auto,               "auto",
+        Catch,              "catch",
+        Default,            "default",
+        Existential,        "existential",
+        Union,              "union",
+    }
+
+    // Other symbols that can be referred to with syntax_pos::symbols::*
+    Other {
+        doc, cfg, masked, spotlight, alias, keyword, feature, include, simd, align,
+    }
+}
+
 #[derive(Copy, Clone, Eq)]
 pub struct Ident {
     pub name: Symbol,
@@ -317,131 +405,6 @@ impl Interner {
     }
 }
 
-// In this macro, there is the requirement that the name (the number) must be monotonically
-// increasing by one in the special identifiers, starting at 0; the same holds for the keywords,
-// except starting from the next number instead of zero.
-macro_rules! declare_keywords {(
-    $( ($index: expr, $konst: ident, $string: expr) )*
-) => {
-    pub mod keywords {
-        use super::{Symbol, Ident};
-        #[derive(Clone, Copy, PartialEq, Eq)]
-        pub struct Keyword {
-            ident: Ident,
-        }
-        impl Keyword {
-            #[inline] pub fn ident(self) -> Ident { self.ident }
-            #[inline] pub fn name(self) -> Symbol { self.ident.name }
-        }
-        $(
-            #[allow(non_upper_case_globals)]
-            pub const $konst: Keyword = Keyword {
-                ident: Ident::with_empty_ctxt(super::Symbol::new($index))
-            };
-        )*
-
-        impl std::str::FromStr for Keyword {
-            type Err = ();
-
-            fn from_str(s: &str) -> Result<Self, ()> {
-                match s {
-                    $($string => Ok($konst),)*
-                    _ => Err(()),
-                }
-            }
-        }
-    }
-
-    impl Interner {
-        pub fn fresh() -> Self {
-            Interner::prefill(&[$($string,)*])
-        }
-    }
-}}
-
-// N.B., leaving holes in the ident table is bad! a different ident will get
-// interned with the id from the hole, but it will be between the min and max
-// of the reserved words, and thus tagged as "reserved".
-// After modifying this list adjust `is_special`, `is_used_keyword`/`is_unused_keyword`,
-// this should be rarely necessary though if the keywords are kept in alphabetic order.
-declare_keywords! {
-    // Special reserved identifiers used internally for elided lifetimes,
-    // unnamed method parameters, crate root module, error recovery etc.
-    (0,  Invalid,            "")
-    (1,  PathRoot,           "{{root}}")
-    (2,  DollarCrate,        "$crate")
-    (3,  Underscore,         "_")
-
-    // Keywords that are used in stable Rust.
-    (4,  As,                 "as")
-    (5,  Box,                "box")
-    (6,  Break,              "break")
-    (7,  Const,              "const")
-    (8,  Continue,           "continue")
-    (9,  Crate,              "crate")
-    (10, Else,               "else")
-    (11, Enum,               "enum")
-    (12, Extern,             "extern")
-    (13, False,              "false")
-    (14, Fn,                 "fn")
-    (15, For,                "for")
-    (16, If,                 "if")
-    (17, Impl,               "impl")
-    (18, In,                 "in")
-    (19, Let,                "let")
-    (20, Loop,               "loop")
-    (21, Match,              "match")
-    (22, Mod,                "mod")
-    (23, Move,               "move")
-    (24, Mut,                "mut")
-    (25, Pub,                "pub")
-    (26, Ref,                "ref")
-    (27, Return,             "return")
-    (28, SelfLower,          "self")
-    (29, SelfUpper,          "Self")
-    (30, Static,             "static")
-    (31, Struct,             "struct")
-    (32, Super,              "super")
-    (33, Trait,              "trait")
-    (34, True,               "true")
-    (35, Type,               "type")
-    (36, Unsafe,             "unsafe")
-    (37, Use,                "use")
-    (38, Where,              "where")
-    (39, While,              "while")
-
-    // Keywords that are used in unstable Rust or reserved for future use.
-    (40, Abstract,           "abstract")
-    (41, Become,             "become")
-    (42, Do,                 "do")
-    (43, Final,              "final")
-    (44, Macro,              "macro")
-    (45, Override,           "override")
-    (46, Priv,               "priv")
-    (47, Typeof,             "typeof")
-    (48, Unsized,            "unsized")
-    (49, Virtual,            "virtual")
-    (50, Yield,              "yield")
-
-    // Edition-specific keywords that are used in stable Rust.
-    (51, Dyn,                "dyn") // >= 2018 Edition only
-
-    // Edition-specific keywords that are used in unstable Rust or reserved for future use.
-    (52, Async,              "async") // >= 2018 Edition only
-    (53, Try,                "try") // >= 2018 Edition only
-
-    // Special lifetime names
-    (54, UnderscoreLifetime, "'_")
-    (55, StaticLifetime,     "'static")
-
-    // Weak keywords, have special meaning only in specific contexts.
-    (56, Auto,               "auto")
-    (57, Catch,              "catch")
-    (58, Default,            "default")
-    (59, Existential,        "existential")
-    (60, Union,              "union")
-}
-
 impl Symbol {
     fn is_used_keyword_2018(self) -> bool {
         self == keywords::Dyn.name()