squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

Remove boxes from token.t.

Browse source
This commit is contained in:
Graydon Hoare 2011-05-09 14:17:28 -07:00
parent 70c759030c
commit b00f3074d4
3 changed files with 79 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
src/comp/front/lexer.rs
View file

@ -18,6 +18,8 @@ state type reader = state obj {
fn bump(); fn bump();
fn mark(); fn mark();
fn get_mark_chpos() -> uint; fn get_mark_chpos() -> uint;
fn add_str(str) -> token.str_num;
fn get_str(token.str_num) -> str;
fn get_chpos() -> uint; fn get_chpos() -> uint;
fn get_keywords() -> hashmap[str,token.token]; fn get_keywords() -> hashmap[str,token.token];
fn get_reserved() -> hashmap[str,()]; fn get_reserved() -> hashmap[str,()];
@ -32,6 +34,7 @@ fn new_reader(IO.reader rdr, str filename, codemap.filemap filemap)
mutable char ch, mutable char ch,
mutable uint mark_chpos, mutable uint mark_chpos,
mutable uint chpos, mutable uint chpos,
mutable vec[str] strs,
hashmap[str,token.token] keywords, hashmap[str,token.token] keywords,
hashmap[str,()] reserved, hashmap[str,()] reserved,
codemap.filemap fm) { codemap.filemap fm) {
@ -79,6 +82,15 @@ fn new_reader(IO.reader rdr, str filename, codemap.filemap filemap)
ret keywords; ret keywords;
} }
fn add_str(str s) -> token.str_num {
strs += vec(s);
ret Vec.len[str](strs) - 1u;
}
fn get_str(token.str_num i) -> str {
ret strs.(i);
}
fn get_reserved() -> hashmap[str,()] { fn get_reserved() -> hashmap[str,()] {
ret reserved; ret reserved;
} }
@ -88,9 +100,10 @@ fn new_reader(IO.reader rdr, str filename, codemap.filemap filemap)
} }
} }
auto file = Str.unsafe_from_bytes(rdr.read_whole_stream()); auto file = Str.unsafe_from_bytes(rdr.read_whole_stream());
let vec[str] strs = vec();
auto rd = reader(file, Str.byte_len(file), 0u, -1 as char, auto rd = reader(file, Str.byte_len(file), 0u, -1 as char,
filemap.start_pos, filemap.start_pos, filemap.start_pos, filemap.start_pos,
keyword_table(), strs, keyword_table(),
reserved_word_table(), reserved_word_table(),
filemap); filemap);
rd.init(); rd.init();
@ -500,25 +513,25 @@ fn scan_number(char c, reader rdr) -> token.token {
if (c == '3' && n == '2') { if (c == '3' && n == '2') {
rdr.bump(); rdr.bump(); rdr.bump(); rdr.bump();
ret token.LIT_MACH_FLOAT(util.common.ty_f32, ret token.LIT_MACH_FLOAT(util.common.ty_f32,
float_str); rdr.add_str(float_str));
} }
else if (c == '6' && n == '4') { else if (c == '6' && n == '4') {
rdr.bump(); rdr.bump(); rdr.bump(); rdr.bump();
ret token.LIT_MACH_FLOAT(util.common.ty_f64, ret token.LIT_MACH_FLOAT(util.common.ty_f64,
float_str); rdr.add_str(float_str));
/* FIXME: if this is out of range for either a 32-bit or /* FIXME: if this is out of range for either a 32-bit or
64-bit float, it won't be noticed till the back-end */ 64-bit float, it won't be noticed till the back-end */
} }
} }
else { else {
ret token.LIT_FLOAT(float_str); ret token.LIT_FLOAT(rdr.add_str(float_str));
} }
} }
auto maybe_exponent = scan_exponent(rdr); auto maybe_exponent = scan_exponent(rdr);
alt(maybe_exponent) { alt(maybe_exponent) {
case(some[str](?s)) { case(some[str](?s)) {
ret token.LIT_FLOAT(dec_str + s); ret token.LIT_FLOAT(rdr.add_str(dec_str + s));
} }
case(none[str]) { case(none[str]) {
ret token.LIT_INT(accum_int); ret token.LIT_INT(accum_int);
@ -594,7 +607,7 @@ fn next_token(reader rdr) -> token.token {
fail; fail;
} }
ret token.IDENT(accum_str); ret token.IDENT(rdr.add_str(accum_str));
} }
if (is_dec_digit(c)) { if (is_dec_digit(c)) {
@ -786,7 +799,7 @@ fn next_token(reader rdr) -> token.token {
rdr.bump(); rdr.bump();
} }
rdr.bump(); rdr.bump();
ret token.LIT_STR(accum_str); ret token.LIT_STR(rdr.add_str(accum_str));
} }
case ('-') { case ('-') {

65
src/comp/front/parser.rs
View file

@ -38,6 +38,8 @@ state type parser =
fn next_def_id() -> ast.def_id; fn next_def_id() -> ast.def_id;
fn set_def(ast.def_num); fn set_def(ast.def_num);
fn get_prec_table() -> vec[op_spec]; fn get_prec_table() -> vec[op_spec];
fn get_str(token.str_num) -> str;
fn get_reader() -> lexer.reader;
fn get_filemap() -> codemap.filemap; fn get_filemap() -> codemap.filemap;
fn get_chpos() -> uint; fn get_chpos() -> uint;
}; };
@ -111,6 +113,14 @@ fn new_parser(session.session sess,
ret precs; ret precs;
} }
fn get_str(token.str_num i) -> str {
ret rdr.get_str(i);
}
fn get_reader() -> lexer.reader {
ret rdr;
}
fn get_filemap() -> codemap.filemap { fn get_filemap() -> codemap.filemap {
ret rdr.get_filemap(); ret rdr.get_filemap();
} }
@ -135,7 +145,7 @@ fn new_parser(session.session sess,
fn unexpected(parser p, token.token t) { fn unexpected(parser p, token.token t) {
let str s = "unexpected token: "; let str s = "unexpected token: ";
s += token.to_str(t); s += token.to_str(p.get_reader(), t);
p.err(s); p.err(s);
} }
@ -144,9 +154,9 @@ fn expect(parser p, token.token t) {
p.bump(); p.bump();
} else { } else {
let str s = "expecting "; let str s = "expecting ";
s += token.to_str(t); s += token.to_str(p.get_reader(), t);
s += ", found "; s += ", found ";
s += token.to_str(p.peek()); s += token.to_str(p.get_reader(), p.peek());
p.err(s); p.err(s);
} }
} }
@ -157,7 +167,7 @@ fn spanned[T](uint lo, uint hi, &T node) -> ast.spanned[T] {
fn parse_ident(parser p) -> ast.ident { fn parse_ident(parser p) -> ast.ident {
alt (p.peek()) { alt (p.peek()) {
case (token.IDENT(?i)) { p.bump(); ret i; } case (token.IDENT(?i)) { p.bump(); ret p.get_str(i); }
case (_) { case (_) {
p.err("expecting ident"); p.err("expecting ident");
fail; fail;
@ -173,10 +183,10 @@ fn parse_ident(parser p) -> ast.ident {
*/ */
fn parse_str_lit_or_env_ident(parser p) -> ast.ident { fn parse_str_lit_or_env_ident(parser p) -> ast.ident {
alt (p.peek()) { alt (p.peek()) {
case (token.LIT_STR(?s)) { p.bump(); ret s; } case (token.LIT_STR(?s)) { p.bump(); ret p.get_str(s); }
case (token.IDENT(?i)) { case (token.IDENT(?i)) {
auto v = eval.lookup(p.get_session(), p.get_env(), auto v = eval.lookup(p.get_session(), p.get_env(),
p.get_span(), i); p.get_span(), p.get_str(i));
if (!eval.val_is_str(v)) { if (!eval.val_is_str(v)) {
p.err("expecting string-valued variable"); p.err("expecting string-valued variable");
} }
@ -549,7 +559,7 @@ fn parse_lit(parser p) -> ast.lit {
} }
case (token.LIT_FLOAT(?s)) { case (token.LIT_FLOAT(?s)) {
p.bump(); p.bump();
lit = ast.lit_float(s); lit = ast.lit_float(p.get_str(s));
} }
case (token.LIT_MACH_INT(?tm, ?i)) { case (token.LIT_MACH_INT(?tm, ?i)) {
p.bump(); p.bump();
@ -557,7 +567,7 @@ fn parse_lit(parser p) -> ast.lit {
} }
case (token.LIT_MACH_FLOAT(?tm, ?s)) { case (token.LIT_MACH_FLOAT(?tm, ?s)) {
p.bump(); p.bump();
lit = ast.lit_mach_float(tm, s); lit = ast.lit_mach_float(tm, p.get_str(s));
} }
case (token.LIT_CHAR(?c)) { case (token.LIT_CHAR(?c)) {
p.bump(); p.bump();
@ -569,7 +579,7 @@ fn parse_lit(parser p) -> ast.lit {
} }
case (token.LIT_STR(?s)) { case (token.LIT_STR(?s)) {
p.bump(); p.bump();
lit = ast.lit_str(s); lit = ast.lit_str(p.get_str(s));
} }
case (?t) { case (?t) {
unexpected(p, t); unexpected(p, t);
@ -617,7 +627,7 @@ fn parse_path(parser p, greed g) -> ast.path {
alt (p.peek()) { alt (p.peek()) {
case (token.IDENT(?i)) { case (token.IDENT(?i)) {
hi = p.get_hi_pos(); hi = p.get_hi_pos();
ids += vec(i); ids += vec(p.get_str(i));
p.bump(); p.bump();
if (p.peek() == token.DOT) { if (p.peek() == token.DOT) {
if (g == GREEDY) { if (g == GREEDY) {
@ -1025,7 +1035,7 @@ fn parse_dot_or_call_expr(parser p) -> @ast.expr {
case (token.IDENT(?i)) { case (token.IDENT(?i)) {
hi = p.get_hi_pos(); hi = p.get_hi_pos();
p.bump(); p.bump();
e = extend_expr_by_ident(p, lo, hi, e, i); e = extend_expr_by_ident(p, lo, hi, e, p.get_str(i));
} }
case (token.LPAREN) { case (token.LPAREN) {
@ -1373,7 +1383,7 @@ fn parse_alt_expr(parser p) -> @ast.expr {
case (token.RBRACE) { /* empty */ } case (token.RBRACE) { /* empty */ }
case (?tok) { case (?tok) {
p.err("expected 'case' or '}' when parsing 'alt' statement " + p.err("expected 'case' or '}' when parsing 'alt' statement " +
"but found " + token.to_str(tok)); "but found " + token.to_str(p.get_reader(), tok));
} }
} }
} }
@ -1483,16 +1493,17 @@ fn parse_pat(parser p) -> @ast.pat {
case (token.IDENT(?id)) { case (token.IDENT(?id)) {
hi = p.get_hi_pos(); hi = p.get_hi_pos();
p.bump(); p.bump();
pat = ast.pat_bind(id, p.next_def_id(), ast.ann_none); pat = ast.pat_bind(p.get_str(id), p.next_def_id(),
ast.ann_none);
} }
case (?tok) { case (?tok) {
p.err("expected identifier after '?' in pattern but " + p.err("expected identifier after '?' in pattern but " +
"found " + token.to_str(tok)); "found " + token.to_str(p.get_reader(), tok));
fail; fail;
} }
} }
} }
case (token.IDENT(?id)) { case (token.IDENT(_)) {
auto tag_path = parse_path(p, GREEDY); auto tag_path = parse_path(p, GREEDY);
hi = tag_path.span.hi; hi = tag_path.span.hi;
@ -1723,7 +1734,7 @@ fn parse_block(parser p) -> ast.block {
if (stmt_ends_with_semi(stmt)) { if (stmt_ends_with_semi(stmt)) {
p.err("expected ';' or '}' after " + p.err("expected ';' or '}' after " +
"expression but found " + "expression but found " +
token.to_str(t)); token.to_str(p.get_reader(), t));
fail; fail;
} }
stmts += vec(stmt); stmts += vec(stmt);
@ -2102,13 +2113,14 @@ fn parse_item_tag(parser p) -> @ast.item {
expect(p, token.SEMI); expect(p, token.SEMI);
auto id = p.next_def_id(); auto id = p.next_def_id();
auto vr = rec(name=name, args=args, id=id, ann=ast.ann_none); auto vr = rec(name=p.get_str(name), args=args,
id=id, ann=ast.ann_none);
variants += vec(spanned[ast.variant_](vlo, vhi, vr)); variants += vec(spanned[ast.variant_](vlo, vhi, vr));
} }
case (token.RBRACE) { /* empty */ } case (token.RBRACE) { /* empty */ }
case (_) { case (_) {
p.err("expected name of variant or '}' but found " + p.err("expected name of variant or '}' but found " +
token.to_str(tok)); token.to_str(p.get_reader(), tok));
} }
} }
} }
@ -2210,7 +2222,8 @@ fn parse_item(parser p) -> @ast.item {
ret parse_item_obj(p, lyr); ret parse_item_obj(p, lyr);
} }
case (?t) { case (?t) {
p.err("expected item but found " + token.to_str(t)); p.err("expected item but found " +
token.to_str(p.get_reader(), t));
} }
} }
fail; fail;
@ -2224,7 +2237,8 @@ fn parse_meta_item(parser p) -> @ast.meta_item {
case (token.LIT_STR(?s)) { case (token.LIT_STR(?s)) {
auto hi = p.get_hi_pos(); auto hi = p.get_hi_pos();
p.bump(); p.bump();
ret @spanned(lo, hi, rec(name = ident, value = s)); ret @spanned(lo, hi, rec(name = ident,
value = p.get_str(s)));
} }
case (_) { case (_) {
p.err("Metadata items must be string literals"); p.err("Metadata items must be string literals");
@ -2294,9 +2308,9 @@ fn parse_rest_import_name(parser p, ast.ident first,
fn parse_full_import_name(parser p, ast.ident def_ident) fn parse_full_import_name(parser p, ast.ident def_ident)
-> @ast.view_item { -> @ast.view_item {
alt (p.peek()) { alt (p.peek()) {
case (token.IDENT(?ident)) { case (token.IDENT(?i)) {
p.bump(); p.bump();
ret parse_rest_import_name(p, ident, some(def_ident)); ret parse_rest_import_name(p, p.get_str(i), some(def_ident));
} }
case (_) { case (_) {
p.err("expecting an identifier"); p.err("expecting an identifier");
@ -2308,15 +2322,16 @@ fn parse_full_import_name(parser p, ast.ident def_ident)
fn parse_import(parser p) -> @ast.view_item { fn parse_import(parser p) -> @ast.view_item {
expect(p, token.IMPORT); expect(p, token.IMPORT);
alt (p.peek()) { alt (p.peek()) {
case (token.IDENT(?ident)) { case (token.IDENT(?i)) {
p.bump(); p.bump();
alt (p.peek()) { alt (p.peek()) {
case (token.EQ) { case (token.EQ) {
p.bump(); p.bump();
ret parse_full_import_name(p, ident); ret parse_full_import_name(p, p.get_str(i));
} }
case (_) { case (_) {
ret parse_rest_import_name(p, ident, none[ast.ident]); ret parse_rest_import_name(p, p.get_str(i),
none[ast.ident]);
} }
} }
} }

29
src/comp/front/token.rs
View file

@ -5,6 +5,8 @@ import std.Int;
import std.UInt; import std.UInt;
import std.Str; import std.Str;
type str_num = uint;
tag binop { tag binop {
PLUS; PLUS;
MINUS; MINUS;
@ -127,14 +129,14 @@ tag token {
LIT_INT(int); LIT_INT(int);
LIT_UINT(uint); LIT_UINT(uint);
LIT_MACH_INT(ty_mach, int); LIT_MACH_INT(ty_mach, int);
LIT_FLOAT(str); LIT_FLOAT(str_num);
LIT_MACH_FLOAT(ty_mach, str); LIT_MACH_FLOAT(ty_mach, str_num);
LIT_STR(str); LIT_STR(str_num);
LIT_CHAR(char); LIT_CHAR(char);
LIT_BOOL(bool); LIT_BOOL(bool);
/* Name components */ /* Name components */
IDENT(str); IDENT(str_num);
IDX(int); IDX(int);
UNDERSCORE; UNDERSCORE;
@ -168,7 +170,7 @@ tag token {
PORT; PORT;
TASK; TASK;
BRACEQUOTE(str); BRACEQUOTE(str_num);
EOF; EOF;
} }
@ -188,7 +190,7 @@ fn binop_to_str(binop o) -> str {
} }
} }
fn to_str(token t) -> str { fn to_str(lexer.reader r, token t) -> str {
alt (t) { alt (t) {
case (EQ) { ret "="; } case (EQ) { ret "="; }
@ -301,10 +303,14 @@ fn to_str(token t) -> str {
ret Int.to_str(i, 10u) ret Int.to_str(i, 10u)
+ "_" + ty_mach_to_str(tm); + "_" + ty_mach_to_str(tm);
} }
case (LIT_FLOAT(?s)) { ret s; } case (LIT_MACH_FLOAT(?tm, ?s)) {
ret r.get_str(s) + "_" + ty_mach_to_str(tm);
}
case (LIT_FLOAT(?s)) { ret r.get_str(s); }
case (LIT_STR(?s)) { case (LIT_STR(?s)) {
// FIXME: escape. // FIXME: escape.
ret "\"" + s + "\""; ret "\"" + r.get_str(s) + "\"";
} }
case (LIT_CHAR(?c)) { case (LIT_CHAR(?c)) {
// FIXME: escape. // FIXME: escape.
@ -319,7 +325,11 @@ fn to_str(token t) -> str {
} }
/* Name components */ /* Name components */
case (IDENT(?s)) { auto si = "ident:"; si += s; ret si; } case (IDENT(?s)) {
auto si = "ident:";
si += r.get_str(s);
ret si;
}
case (IDX(?i)) { ret "_" + Int.to_str(i, 10u); } case (IDX(?i)) { ret "_" + Int.to_str(i, 10u); }
case (UNDERSCORE) { ret "_"; } case (UNDERSCORE) { ret "_"; }
@ -360,7 +370,6 @@ fn to_str(token t) -> str {
} }
// Local Variables: // Local Variables:
// fill-column: 78; // fill-column: 78;
// indent-tabs-mode: nil // indent-tabs-mode: nil