squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

Merge #7884

Browse source 
7884: Simplify TokenStream FromStr r=edwin0cheng a=edwin0cheng

Make sure `FromStr` ignore all `TokenMap` in all cases.

bors r+

Co-authored-by: Edwin Cheng <edwin0cheng@gmail.com>
This commit is contained in:
bors[bot] 2021-03-05 20:00:04 +00:00 committed by GitHub
parent 7199d5b56d 83230b2704
commit 94012e24e0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 43 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
crates/proc_macro_srv/src/rustc_server.rs
View file

@ -184,6 +184,7 @@ pub mod token_stream {
let (subtree, _token_map) =
mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?;
let subtree = subtree_replace_token_ids_with_unspecified(subtree);
Ok(TokenStream { subtree })
}
}
@ -226,6 +227,44 @@ pub mod token_stream {
}
}
}
fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
tt::Subtree {
delimiter: subtree
.delimiter
.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
token_trees: subtree
.token_trees
.into_iter()
.map(|t| token_tree_replace_token_ids_with_unspecified(t))
.collect(),
}
}
fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree {
match tt {
tt::TokenTree::Leaf(leaf) => {
tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf))
}
tt::TokenTree::Subtree(subtree) => {
tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree))
}
}
}
fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
match leaf {
tt::Leaf::Literal(lit) => {
tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
}
tt::Leaf::Punct(punct) => {
tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
}
tt::Leaf::Ident(ident) => {
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
}
}
}
}
impl TokenStreamBuilder {
@ -277,42 +316,6 @@ impl server::FreeFunctions for Rustc {
}
}
fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
tt::Subtree {
delimiter: subtree.delimiter.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
token_trees: subtree
.token_trees
.into_iter()
.map(|t| token_tree_replace_token_ids_with_unspecified(t))
.collect(),
}
}
fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree {
match tt {
tt::TokenTree::Leaf(leaf) => {
tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf))
}
tt::TokenTree::Subtree(subtree) => {
tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree))
}
}
}
fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
match leaf {
tt::Leaf::Literal(lit) => {
tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
}
tt::Leaf::Punct(punct) => {
tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
}
tt::Leaf::Ident(ident) => {
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
}
}
}
impl server::TokenStream for Rustc {
fn new(&mut self) -> Self::TokenStream {
Self::TokenStream::new()
@ -322,8 +325,9 @@ impl server::TokenStream for Rustc {
stream.is_empty()
}
fn from_str(&mut self, src: &str) -> Self::TokenStream {
let (subtree, _) = mbe::parse_to_token_tree(src).expect("cannot parse string");
TokenStream::with_subtree(subtree_replace_token_ids_with_unspecified(subtree))
use std::str::FromStr;
Self::TokenStream::from_str(src).expect("cannot parse string")
}
fn to_string(&mut self, stream: &Self::TokenStream) -> String {
stream.to_string()

2
crates/proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt vendored
View file

@ -101,7 +101,7 @@ SUBTREE $
PUNCH : [alone] 4294967295
IDENT Serialize 4294967295
IDENT for 4294967295
IDENT Foo 1
IDENT Foo 4294967295
SUBTREE {} 4294967295
IDENT fn 4294967295
IDENT serialize 4294967295