diff --git a/src/comp/front/lexer.rs b/src/comp/front/lexer.rs index 18d211d94e8..16f75b4ab05 100644 --- a/src/comp/front/lexer.rs +++ b/src/comp/front/lexer.rs @@ -10,6 +10,7 @@ import std::option::none; import driver::session::session; import util::common; import util::common::new_str_hash; +import util::interner; state type reader = state obj { fn is_eof() -> bool; @@ -19,15 +20,15 @@ state type reader = state obj { fn bump(); fn mark(); fn get_mark_chpos() -> uint; - fn add_str(str) -> token::str_num; - fn get_str(token::str_num) -> str; + fn get_interner() -> @interner::interner[str]; fn get_chpos() -> uint; fn get_filemap() -> codemap::filemap; fn err(str m); }; fn new_reader(session sess, io::reader rdr, - codemap::filemap filemap) -> reader { + codemap::filemap filemap, + @interner::interner[str] itr) -> reader { state obj reader(session sess, str file, @@ -37,7 +38,8 @@ fn new_reader(session sess, io::reader rdr, mutable uint mark_chpos, mutable uint chpos, mutable vec[str] strs, - codemap::filemap fm) { + codemap::filemap fm, + @interner::interner[str] itr) { fn is_eof() -> bool { ret ch == -1 as char; @@ -78,14 +80,7 @@ fn new_reader(session sess, io::reader rdr, } } - fn add_str(str s) -> token::str_num { - strs += vec(s); - ret _vec::len[str](strs) - 1u; - } - - fn get_str(token::str_num i) -> str { - ret strs.(i); - } + fn get_interner() -> @interner::interner[str] { ret itr; } fn get_filemap() -> codemap::filemap { ret fm; @@ -99,7 +94,7 @@ fn new_reader(session sess, io::reader rdr, let vec[str] strs = vec(); auto rd = reader(sess, file, _str::byte_len(file), 0u, -1 as char, filemap.start_pos, filemap.start_pos, - strs, filemap); + strs, filemap, itr); rd.init(); ret rd; } @@ -387,28 +382,30 @@ fn scan_number(char c, reader rdr) -> token::token { if (c == '3' && n == '2') { rdr.bump(); rdr.bump(); ret token::LIT_MACH_FLOAT(util::common::ty_f32, - rdr.add_str(float_str)); + interner::intern[str](*rdr.get_interner(), float_str)); } else if (c == '6' && n == '4') { rdr.bump(); rdr.bump(); ret token::LIT_MACH_FLOAT(util::common::ty_f64, - rdr.add_str(float_str)); + interner::intern[str](*rdr.get_interner(), float_str)); /* FIXME: if this is out of range for either a 32-bit or 64-bit float, it won't be noticed till the back-end */ } } else { - ret token::LIT_FLOAT(rdr.add_str(float_str)); + ret token::LIT_FLOAT(interner::intern[str](*rdr.get_interner(), + float_str)); } } auto maybe_exponent = scan_exponent(rdr); alt(maybe_exponent) { case(some[str](?s)) { - ret token::LIT_FLOAT(rdr.add_str(dec_str + s)); + ret token::LIT_FLOAT(interner::intern[str](*rdr.get_interner(), + dec_str + s)); } case(none[str]) { - ret token::LIT_INT(accum_int); + ret token::LIT_INT(accum_int); } } } @@ -471,7 +468,8 @@ fn next_token(reader rdr) -> token::token { } auto is_mod_name = c == ':' && rdr.next() == ':'; - ret token::IDENT(rdr.add_str(accum_str), is_mod_name); + ret token::IDENT(interner::intern[str](*rdr.get_interner(), + accum_str), is_mod_name); } if (is_dec_digit(c)) { @@ -671,7 +669,8 @@ fn next_token(reader rdr) -> token::token { rdr.bump(); } rdr.bump(); - ret token::LIT_STR(rdr.add_str(accum_str)); + ret token::LIT_STR(interner::intern[str](*rdr.get_interner(), + accum_str)); } case ('-') { @@ -801,7 +800,8 @@ fn read_block_comment(reader rdr) -> cmnt { fn gather_comments(session sess, str path) -> vec[cmnt] { auto srdr = io::file_reader(path); - auto rdr = new_reader(sess, srdr, codemap::new_filemap(path, 0u)); + auto itr = @interner::mk_interner[str](_str::hash, _str::eq); + auto rdr = new_reader(sess, srdr, codemap::new_filemap(path, 0u), itr); let vec[cmnt] comments = vec(); while (!rdr.is_eof()) { while (true) { diff --git a/src/comp/front/parser.rs b/src/comp/front/parser.rs index 929bf6f0dd4..d0b49b10637 100644 --- a/src/comp/front/parser.rs +++ b/src/comp/front/parser.rs @@ -11,6 +11,7 @@ import util::common; import util::common::filename; import util::common::span; import util::common::new_str_hash; +import util::interner; tag restriction { UNRESTRICTED; @@ -123,7 +124,7 @@ fn new_parser(session::session sess, } fn get_str(token::str_num i) -> str { - ret rdr.get_str(i); + ret interner::get[str](*rdr.get_interner(), i); } fn get_reader() -> lexer::reader { @@ -156,7 +157,8 @@ fn new_parser(session::session sess, auto srdr = io::file_reader(path); auto filemap = codemap::new_filemap(path, pos); _vec::push[codemap::filemap](sess.get_codemap().files, filemap); - auto rdr = lexer::new_reader(sess, srdr, filemap); + auto itr = @interner::mk_interner[str](_str::hash, _str::eq); + auto rdr = lexer::new_reader(sess, srdr, filemap, itr); // Make sure npos points at first actual token: lexer::consume_any_whitespace(rdr); auto npos = rdr.get_chpos(); diff --git a/src/comp/front/token.rs b/src/comp/front/token.rs index 38f66a16dc5..b72d41e7d24 100644 --- a/src/comp/front/token.rs +++ b/src/comp/front/token.rs @@ -1,6 +1,7 @@ import util::common::ty_mach; import util::common::ty_mach_to_str; import util::common::new_str_hash; +import util::interner; import std::_int; import std::_uint; import std::_str; @@ -139,13 +140,14 @@ fn to_str(lexer::reader r, token t) -> str { + "_" + ty_mach_to_str(tm); } case (LIT_MACH_FLOAT(?tm, ?s)) { - ret r.get_str(s) + "_" + ty_mach_to_str(tm); + ret interner::get[str](*r.get_interner(), s) + "_" + + ty_mach_to_str(tm); } - case (LIT_FLOAT(?s)) { ret r.get_str(s); } + case (LIT_FLOAT(?s)) { ret interner::get[str](*r.get_interner(), s); } case (LIT_STR(?s)) { // FIXME: escape. - ret "\"" + r.get_str(s) + "\""; + ret "\"" + interner::get[str](*r.get_interner(), s) + "\""; } case (LIT_CHAR(?c)) { // FIXME: escape. @@ -160,9 +162,7 @@ fn to_str(lexer::reader r, token t) -> str { } /* Name components */ - case (IDENT(?s, _)) { - ret r.get_str(s); - } + case (IDENT(?s, _)) { ret interner::get[str](*r.get_interner(), s); } case (IDX(?i)) { ret "_" + _int::to_str(i, 10u); } case (UNDERSCORE) { ret "_"; }