squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

Commenting, test cases, cleanup

Browse source
This commit is contained in:
John Clements 2013-02-04 13:15:17 -08:00
parent 6d09fc2cd8
commit 25c4676dfa
8 changed files with 121 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/libcore/dvec.rs
View file

@ -229,7 +229,7 @@ impl<A> DVec<A> {
impl<A: Copy> DVec<A> {
/**
* Append all elements of a vector to the end of the list
* Append all elements of a vector to the end of the list.
*
* Equivalent to `append_iter()` but potentially more efficient.
*/

11
src/libsyntax/ast.rs
View file

@ -779,10 +779,19 @@ pub enum expr_ {
#[auto_decode]
#[doc="For macro invocations; parsing is delegated to the macro"]
pub enum token_tree {
// a single token
tt_tok(span, ::parse::token::Token),
// a delimited sequence (the delimiters appear as the first
// and last elements of the vector)
tt_delim(~[token_tree]),
// These only make sense for right-hand-sides of MBE macros
// These only make sense for right-hand-sides of MBE macros:
// a kleene-style repetition sequence with a span, a tt_forest,
// an optional separator (?), and a boolean where true indicates
// zero or more (*), and false indicates one or more (+).
tt_seq(span, ~[token_tree], Option<::parse::token::Token>, bool),
// a syntactic variable that will be filled in by macro expansion.
tt_nonterminal(span, ident)
}

12
src/libsyntax/ext/base.rs
View file

@ -44,8 +44,8 @@ pub struct SyntaxExpanderTT {
span: Option<span>
}
pub type SyntaxExpanderTTFun = fn@(ext_ctxt, span, ~[ast::token_tree])
-> MacResult;
pub type SyntaxExpanderTTFun
= fn@(ext_ctxt, span, ~[ast::token_tree]) -> MacResult;
pub struct SyntaxExpanderTTItem {
expander: SyntaxExpanderTTItemFun,
@ -59,7 +59,7 @@ pub enum MacResult {
MRExpr(@ast::expr),
MRItem(@ast::item),
MRAny(fn@()-> @ast::expr, fn@()-> Option<@ast::item>, fn@()->@ast::stmt),
MRDef(MacroDef)
MRDef(MacroDef),
}
pub enum SyntaxExtension {
@ -78,9 +78,11 @@ pub enum SyntaxExtension {
// A temporary hard-coded map of methods for expanding syntax extension
// AST nodes into full ASTs
pub fn syntax_expander_table() -> HashMap<~str, SyntaxExtension> {
// utility function to simplify creating NormalTT syntax extensions
fn builtin_normal_tt(f: SyntaxExpanderTTFun) -> SyntaxExtension {
NormalTT(SyntaxExpanderTT{expander: f, span: None})
}
// utility function to simplify creating ItemTT syntax extensions
fn builtin_item_tt(f: SyntaxExpanderTTItemFun) -> SyntaxExtension {
ItemTT(SyntaxExpanderTTItem{expander: f, span: None})
}
@ -112,8 +114,8 @@ pub fn syntax_expander_table() -> HashMap<~str, SyntaxExtension> {
ext::deriving::expand_deriving_iter_bytes));
// Quasi-quoting expanders
syntax_expanders.insert(
~"quote_tokens", builtin_normal_tt(ext::quote::expand_quote_tokens));
syntax_expanders.insert(~"quote_tokens",
builtin_normal_tt(ext::quote::expand_quote_tokens));
syntax_expanders.insert(~"quote_expr",
builtin_normal_tt(ext::quote::expand_quote_expr));
syntax_expanders.insert(~"quote_ty",

6
src/libsyntax/ext/tt/transcribe.rs
View file

@ -121,15 +121,15 @@ pure fn lookup_cur_matched_by_matched(r: @mut TtReader,
vec::foldl(start, r.repeat_idx, red)
}
fn lookup_cur_matched(r: @mut TtReader, name: ident) -> @named_match {
fn lookup_cur_matched(r: &TtReader, name: ident) -> @named_match {
lookup_cur_matched_by_matched(r, r.interpolations.get(&name))
}
enum lis {
lis_unconstrained, lis_constraint(uint, ident), lis_contradiction(~str)
}
fn lockstep_iter_size(t: token_tree, r: @mut TtReader) -> lis {
fn lis_merge(lhs: lis, rhs: lis, r: @mut TtReader) -> lis {
fn lockstep_iter_size(t: token_tree, r: &TtReader) -> lis {
fn lis_merge(lhs: lis, rhs: lis, r: &TtReader) -> lis {
match lhs {
lis_unconstrained => rhs,
lis_contradiction(_) => lhs,

26
src/libsyntax/parse/common.rs
View file

@ -20,6 +20,8 @@ use core::option::{None, Option, Some};
use core::option;
use std::oldmap::HashMap;
// seq_sep : a sequence separator (token)
// and whether a trailing separator is allowed.
pub type seq_sep = {
sep: Option<token::Token>,
trailing_sep_allowed: bool
@ -51,6 +53,8 @@ pub impl Parser {
+ token_to_str(self.reader, self.token) + ~"`");
}
// expect and consume the token t. Signal an error if
// the next token is not t.
fn expect(t: token::Token) {
if self.token == t {
self.bump();
@ -88,6 +92,8 @@ pub impl Parser {
return self.parse_ident();
}
// consume token 'tok' if it exists. Returns true if the given
// token was present, false otherwise.
fn eat(tok: token::Token) -> bool {
return if self.token == tok { self.bump(); true } else { false };
}
@ -185,6 +191,8 @@ pub impl Parser {
}
}
// expect and consume a GT. if a >> is seen, replace it
// with a single > and continue.
fn expect_gt() {
if self.token == token::GT {
self.bump();
@ -202,16 +210,19 @@ pub impl Parser {
}
}
// parse a sequence bracketed by '<' and '>', stopping
// before the '>'.
fn parse_seq_to_before_gt<T: Copy>(sep: Option<token::Token>,
f: fn(Parser) -> T) -> ~[T] {
let mut first = true;
let mut v = ~[];
while self.token != token::GT
// wait... isn't this going to eat a whole '>>' ?
&& self.token != token::BINOP(token::SHR) {
match sep {
Some(ref t) => {
if first { first = false; }
else { self.expect((*t)); }
else { self.expect(*t); }
}
_ => ()
}
@ -229,6 +240,7 @@ pub impl Parser {
return v;
}
// parse a sequence bracketed by '<' and '>'
fn parse_seq_lt_gt<T: Copy>(sep: Option<token::Token>,
f: fn(Parser) -> T) -> spanned<~[T]> {
let lo = self.span.lo;
@ -239,6 +251,9 @@ pub impl Parser {
return spanned(lo, hi, result);
}
// parse a sequence, including the closing delimiter. The function
// f must consume tokens until reaching the next separator or
// closing bracket.
fn parse_seq_to_end<T: Copy>(ket: token::Token, sep: seq_sep,
f: fn(Parser) -> T) -> ~[T] {
let val = self.parse_seq_to_before_end(ket, sep, f);
@ -246,7 +261,9 @@ pub impl Parser {
return val;
}
// parse a sequence, not including the closing delimiter. The function
// f must consume tokens until reaching the next separator or
// closing bracket.
fn parse_seq_to_before_end<T: Copy>(ket: token::Token, sep: seq_sep,
f: fn(Parser) -> T) -> ~[T] {
let mut first: bool = true;
@ -255,7 +272,7 @@ pub impl Parser {
match sep.sep {
Some(ref t) => {
if first { first = false; }
else { self.expect((*t)); }
else { self.expect(*t); }
}
_ => ()
}
@ -265,6 +282,9 @@ pub impl Parser {
return v;
}
// parse a sequence, including the closing delimiter. The function
// f must consume tokens until reaching the next separator or
// closing bracket.
fn parse_unspanned_seq<T: Copy>(bra: token::Token,
ket: token::Token,
sep: seq_sep,

47
src/libsyntax/parse/mod.rs
View file

@ -183,7 +183,6 @@ pub fn new_parser_from_file(sess: parse_sess,
let srdr = lexer::new_string_reader(sess.span_diagnostic,
filemap,
sess.interner);
Ok(Parser(sess, cfg, srdr as reader))
}
@ -222,3 +221,49 @@ pub fn new_parser_from_tts(sess: parse_sess, cfg: ast::crate_cfg,
return Parser(sess, cfg, trdr as reader)
}
#[cfg(test)]
mod test {
use super::*;
use std::serialize::Encodable;
use std;
use core::dvec;
use core::str;
use util::testing::*;
#[test] fn to_json_str (val: Encodable<std::json::Encoder>) -> ~str {
let bw = @io::BytesWriter {bytes: dvec::DVec(), pos: 0};
val.encode(~std::json::Encoder(bw as io::Writer));
str::from_bytes(bw.bytes.data)
}
#[test] fn alltts () {
let tts = parse_tts_from_source_str(
~"bogofile",
@~"fn foo (x : int) { x; }",
~[],
new_parse_sess(None));
check_equal(to_json_str(tts as Encodable::<std::json::Encoder>),
//[["tt_tok",["IDENT","fn"]]]
~"abc"
);
let ast1 = new_parser_from_tts(new_parse_sess(None),~[],tts)
.parse_item(~[]);
let ast2 = parse_item_from_source_str(
~"bogofile",
@~"fn foo (x : int) { x; }",
~[],~[],
new_parse_sess(None));
check_equal(ast1,ast2);
}
}
//
// Local Variables:
// mode: rust
// fill-column: 78;
// indent-tabs-mode: nil
// c-basic-offset: 4
// buffer-file-coding-system: utf-8-unix
// End:
//

47
src/libsyntax/parse/parser.rs
View file

@ -182,7 +182,8 @@ pure fn maybe_append(+lhs: ~[attribute], rhs: Option<~[attribute]>)
/* ident is handled by common.rs */
pub fn Parser(sess: parse_sess,
pub fn Parser(sess: parse_sess
,
cfg: ast::crate_cfg,
+rdr: reader) -> Parser {
@ -1238,6 +1239,8 @@ pub impl Parser {
return e;
}
// parse an optional separator followed by a kleene-style
// repetition token (+ or *).
fn parse_sep_and_zerok() -> (Option<token::Token>, bool) {
if self.token == token::BINOP(token::STAR)
|| self.token == token::BINOP(token::PLUS) {
@ -1258,20 +1261,18 @@ pub impl Parser {
}
}
// parse a single token tree from the input.
fn parse_token_tree() -> token_tree {
maybe_whole!(deref self, nt_tt);
fn parse_tt_tok(p: Parser, delim_ok: bool) -> token_tree {
fn parse_non_delim_tt_tok(p: Parser) -> token_tree {
maybe_whole!(deref p, nt_tt);
match p.token {
token::RPAREN | token::RBRACE | token::RBRACKET
if !delim_ok => {
=> {
p.fatal(~"incorrect close delimiter: `"
+ token_to_str(p.reader, p.token) + ~"`");
}
token::EOF => {
p.fatal(~"file ended in the middle of a macro invocation");
}
/* we ought to allow different depths of unquotation */
token::DOLLAR if p.quote_depth > 0u => {
p.bump();
@ -1282,32 +1283,43 @@ pub impl Parser {
seq_sep_none(),
|p| p.parse_token_tree());
let (s, z) = p.parse_sep_and_zerok();
return tt_seq(mk_sp(sp.lo ,p.span.hi), seq.node, s, z);
tt_seq(mk_sp(sp.lo ,p.span.hi), seq.node, s, z)
} else {
return tt_nonterminal(sp, p.parse_ident());
tt_nonterminal(sp, p.parse_ident())
}
}
_ => { /* ok */ }
_ => {
parse_any_tt_tok(p)
}
}
let res = tt_tok(p.span, p.token);
p.bump();
return res;
}
return match self.token {
// turn the next token into a tt_tok:
fn parse_any_tt_tok(p: Parser) -> token_tree{
let res = tt_tok(p.span, p.token);
p.bump();
res
}
match self.token {
token::EOF => {
self.fatal(~"file ended in the middle of a macro invocation");
}
token::LPAREN | token::LBRACE | token::LBRACKET => {
// tjc: ??????
let ket = token::flip_delimiter(copy self.token);
tt_delim(vec::append(
~[parse_tt_tok(self, true)],
// the open delimiter:
~[parse_any_tt_tok(self)],
vec::append(
self.parse_seq_to_before_end(
ket, seq_sep_none(),
|p| p.parse_token_tree()),
~[parse_tt_tok(self, true)])))
// the close delimiter:
~[parse_any_tt_tok(self)])))
}
_ => parse_tt_tok(self, false)
};
_ => parse_non_delim_tt_tok(self)
}
}
fn parse_all_token_trees() -> ~[token_tree] {
@ -3999,6 +4011,7 @@ pub impl Parser {
}
}
//
// Local Variables:
// mode: rust

1
src/libsyntax/parse/token.rs
View file

@ -86,6 +86,7 @@ pub enum Token {
LIT_STR(ast::ident),
/* Name components */
// an identifier contains an "is_mod_name" boolean.
IDENT(ast::ident, bool),
UNDERSCORE,
LIFETIME(ast::ident),