squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

Derive Rustc{En,De}codable for TokenStream.

Browse source 
`TokenStream` used to be a complex type, but it is now just a newtype
around a `Lrc<Vec<TreeAndJoint>>`. Currently it uses custom encoding
that discards the `IsJoint` and custom decoding that adds `NonJoint`
back in for every token tree. This requires building intermediate
`Vec<TokenTree>`s.

This commit makes `TokenStream` derive `Rustc{En,De}codable`. This
simplifies the code, and avoids the creation of the intermediate
vectors, saving up to 3% on various benchmarks. It also changes the AST
JSON output in one test.
This commit is contained in:
Nicholas Nethercote 2019-10-16 10:33:16 +11:00
parent c23a7aa778
commit c290293cf2
2 changed files with 3 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/libsyntax/tokenstream.rs
View file

@ -19,7 +19,6 @@ use syntax_pos::{BytePos, Span, DUMMY_SP};
#[cfg(target_arch = "x86_64")]
use rustc_data_structures::static_assert_size;
use rustc_data_structures::sync::Lrc;
use rustc_serialize::{Decoder, Decodable, Encoder, Encodable};
use smallvec::{SmallVec, smallvec};
use std::{iter, mem};
@ -136,7 +135,7 @@ impl TokenTree {
/// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
/// instead of a representation of the abstract syntax tree.
/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for back-compat.
#[derive(Clone, Debug, Default)]
#[derive(Clone, Debug, Default, RustcEncodable, RustcDecodable)]
pub struct TokenStream(pub Lrc<Vec<TreeAndJoint>>);
pub type TreeAndJoint = (TokenTree, IsJoint);
@ -145,7 +144,7 @@ pub type TreeAndJoint = (TokenTree, IsJoint);
#[cfg(target_arch = "x86_64")]
static_assert_size!(TokenStream, 8);
#[derive(Clone, Copy, Debug, PartialEq)]
#[derive(Clone, Copy, Debug, PartialEq, RustcEncodable, RustcDecodable)]
pub enum IsJoint {
Joint,
NonJoint
@ -460,18 +459,6 @@ impl Cursor {
}
}
impl Encodable for TokenStream {
fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), E::Error> {
self.trees().collect::<Vec<_>>().encode(encoder)
}
}
impl Decodable for TokenStream {
fn decode<D: Decoder>(decoder: &mut D) -> Result<TokenStream, D::Error> {
Vec::<TokenTree>::decode(decoder).map(|vec| vec.into_iter().collect())
}
}
#[derive(Debug, Copy, Clone, PartialEq, RustcEncodable, RustcDecodable)]
pub struct DelimSpan {
pub open: Span,

2
src/test/ui/ast-json/ast-json-output.stdout
View file

@ -1 +1 @@
{"module":{"inner":{"lo":0,"hi":0},"items":[{"ident":{"name":"core","span":{"lo":0,"hi":0}},"attrs":[],"id":0,"kind":{"variant":"ExternCrate","fields":[null]},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"span":{"lo":0,"hi":0},"tokens":[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["extern",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["core",false]},"span":{"lo":0,"hi":0}}]},{"variant":"Token","fields":[{"kind":"Semi","span":{"lo":0,"hi":0}}]}]}],"inline":true},"attrs":[],"span":{"lo":0,"hi":0}}
{"module":{"inner":{"lo":0,"hi":0},"items":[{"ident":{"name":"core","span":{"lo":0,"hi":0}},"attrs":[],"id":0,"kind":{"variant":"ExternCrate","fields":[null]},"vis":{"node":"Inherited","span":{"lo":0,"hi":0}},"span":{"lo":0,"hi":0},"tokens":{"_field0":[[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["extern",false]},"span":{"lo":0,"hi":0}}]},"NonJoint"],[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["crate",false]},"span":{"lo":0,"hi":0}}]},"NonJoint"],[{"variant":"Token","fields":[{"kind":{"variant":"Ident","fields":["core",false]},"span":{"lo":0,"hi":0}}]},"NonJoint"],[{"variant":"Token","fields":[{"kind":"Semi","span":{"lo":0,"hi":0}}]},"NonJoint"]]}}],"inline":true},"attrs":[],"span":{"lo":0,"hi":0}}