squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

rustc: Make the parser use the interner

Browse source
This commit is contained in:
Patrick Walton 2011-05-16 14:53:00 -07:00
parent b91ef944c8
commit 3ea3d9eed6
3 changed files with 31 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

42
src/comp/front/lexer.rs
View file

@ -10,6 +10,7 @@ import std::option::none;
import driver::session::session;
import util::common;
import util::common::new_str_hash;
import util::interner;
state type reader = state obj {
fn is_eof() -> bool;
@ -19,15 +20,15 @@ state type reader = state obj {
fn bump();
fn mark();
fn get_mark_chpos() -> uint;
fn add_str(str) -> token::str_num;
fn get_str(token::str_num) -> str;
fn get_interner() -> @interner::interner[str];
fn get_chpos() -> uint;
fn get_filemap() -> codemap::filemap;
fn err(str m);
};
fn new_reader(session sess, io::reader rdr,
codemap::filemap filemap) -> reader {
codemap::filemap filemap,
@interner::interner[str] itr) -> reader {
state obj reader(session sess,
str file,
@ -37,7 +38,8 @@ fn new_reader(session sess, io::reader rdr,
mutable uint mark_chpos,
mutable uint chpos,
mutable vec[str] strs,
codemap::filemap fm) {
codemap::filemap fm,
@interner::interner[str] itr) {
fn is_eof() -> bool {
ret ch == -1 as char;
@ -78,14 +80,7 @@ fn new_reader(session sess, io::reader rdr,
}
}
fn add_str(str s) -> token::str_num {
strs += vec(s);
ret _vec::len[str](strs) - 1u;
}
fn get_str(token::str_num i) -> str {
ret strs.(i);
}
fn get_interner() -> @interner::interner[str] { ret itr; }
fn get_filemap() -> codemap::filemap {
ret fm;
@ -99,7 +94,7 @@ fn new_reader(session sess, io::reader rdr,
let vec[str] strs = vec();
auto rd = reader(sess, file, _str::byte_len(file), 0u, -1 as char,
filemap.start_pos, filemap.start_pos,
strs, filemap);
strs, filemap, itr);
rd.init();
ret rd;
}
@ -387,28 +382,30 @@ fn scan_number(char c, reader rdr) -> token::token {
if (c == '3' && n == '2') {
rdr.bump(); rdr.bump();
ret token::LIT_MACH_FLOAT(util::common::ty_f32,
rdr.add_str(float_str));
interner::intern[str](*rdr.get_interner(), float_str));
}
else if (c == '6' && n == '4') {
rdr.bump(); rdr.bump();
ret token::LIT_MACH_FLOAT(util::common::ty_f64,
rdr.add_str(float_str));
interner::intern[str](*rdr.get_interner(), float_str));
/* FIXME: if this is out of range for either a 32-bit or
64-bit float, it won't be noticed till the back-end */
}
}
else {
ret token::LIT_FLOAT(rdr.add_str(float_str));
ret token::LIT_FLOAT(interner::intern[str](*rdr.get_interner(),
float_str));
}
}
auto maybe_exponent = scan_exponent(rdr);
alt(maybe_exponent) {
case(some[str](?s)) {
ret token::LIT_FLOAT(rdr.add_str(dec_str + s));
ret token::LIT_FLOAT(interner::intern[str](*rdr.get_interner(),
dec_str + s));
}
case(none[str]) {
ret token::LIT_INT(accum_int);
ret token::LIT_INT(accum_int);
}
}
}
@ -471,7 +468,8 @@ fn next_token(reader rdr) -> token::token {
}
auto is_mod_name = c == ':' && rdr.next() == ':';
ret token::IDENT(rdr.add_str(accum_str), is_mod_name);
ret token::IDENT(interner::intern[str](*rdr.get_interner(),
accum_str), is_mod_name);
}
if (is_dec_digit(c)) {
@ -671,7 +669,8 @@ fn next_token(reader rdr) -> token::token {
rdr.bump();
}
rdr.bump();
ret token::LIT_STR(rdr.add_str(accum_str));
ret token::LIT_STR(interner::intern[str](*rdr.get_interner(),
accum_str));
}
case ('-') {
@ -801,7 +800,8 @@ fn read_block_comment(reader rdr) -> cmnt {
fn gather_comments(session sess, str path) -> vec[cmnt] {
auto srdr = io::file_reader(path);
auto rdr = new_reader(sess, srdr, codemap::new_filemap(path, 0u));
auto itr = @interner::mk_interner[str](_str::hash, _str::eq);
auto rdr = new_reader(sess, srdr, codemap::new_filemap(path, 0u), itr);
let vec[cmnt] comments = vec();
while (!rdr.is_eof()) {
while (true) {

6
src/comp/front/parser.rs
View file

@ -11,6 +11,7 @@ import util::common;
import util::common::filename;
import util::common::span;
import util::common::new_str_hash;
import util::interner;
tag restriction {
UNRESTRICTED;
@ -123,7 +124,7 @@ fn new_parser(session::session sess,
}
fn get_str(token::str_num i) -> str {
ret rdr.get_str(i);
ret interner::get[str](*rdr.get_interner(), i);
}
fn get_reader() -> lexer::reader {
@ -156,7 +157,8 @@ fn new_parser(session::session sess,
auto srdr = io::file_reader(path);
auto filemap = codemap::new_filemap(path, pos);
_vec::push[codemap::filemap](sess.get_codemap().files, filemap);
auto rdr = lexer::new_reader(sess, srdr, filemap);
auto itr = @interner::mk_interner[str](_str::hash, _str::eq);
auto rdr = lexer::new_reader(sess, srdr, filemap, itr);
// Make sure npos points at first actual token:
lexer::consume_any_whitespace(rdr);
auto npos = rdr.get_chpos();

12
src/comp/front/token.rs
View file

@ -1,6 +1,7 @@
import util::common::ty_mach;
import util::common::ty_mach_to_str;
import util::common::new_str_hash;
import util::interner;
import std::_int;
import std::_uint;
import std::_str;
@ -139,13 +140,14 @@ fn to_str(lexer::reader r, token t) -> str {
+ "_" + ty_mach_to_str(tm);
}
case (LIT_MACH_FLOAT(?tm, ?s)) {
ret r.get_str(s) + "_" + ty_mach_to_str(tm);
ret interner::get[str](*r.get_interner(), s) + "_" +
ty_mach_to_str(tm);
}
case (LIT_FLOAT(?s)) { ret r.get_str(s); }
case (LIT_FLOAT(?s)) { ret interner::get[str](*r.get_interner(), s); }
case (LIT_STR(?s)) {
// FIXME: escape.
ret "\"" + r.get_str(s) + "\"";
ret "\"" + interner::get[str](*r.get_interner(), s) + "\"";
}
case (LIT_CHAR(?c)) {
// FIXME: escape.
@ -160,9 +162,7 @@ fn to_str(lexer::reader r, token t) -> str {
}
/* Name components */
case (IDENT(?s, _)) {
ret r.get_str(s);
}
case (IDENT(?s, _)) { ret interner::get[str](*r.get_interner(), s); }
case (IDX(?i)) { ret "_" + _int::to_str(i, 10u); }
case (UNDERSCORE) { ret "_"; }