Pull file reading out of the lexer
This commit is contained in:
parent
05390b4b38
commit
08f8753d80
2 changed files with 11 additions and 10 deletions
|
@ -29,10 +29,10 @@ type reader =
|
||||||
fn err(str) ;
|
fn err(str) ;
|
||||||
};
|
};
|
||||||
|
|
||||||
fn new_reader(&codemap::codemap cm, io::reader rdr, codemap::filemap filemap,
|
fn new_reader(&codemap::codemap cm, str src, codemap::filemap filemap,
|
||||||
@interner::interner[str] itr) -> reader {
|
@interner::interner[str] itr) -> reader {
|
||||||
obj reader(codemap::codemap cm,
|
obj reader(codemap::codemap cm,
|
||||||
str file,
|
str src,
|
||||||
uint len,
|
uint len,
|
||||||
mutable uint col,
|
mutable uint col,
|
||||||
mutable uint pos,
|
mutable uint pos,
|
||||||
|
@ -45,18 +45,18 @@ fn new_reader(&codemap::codemap cm, io::reader rdr, codemap::filemap filemap,
|
||||||
@interner::interner[str] itr) {
|
@interner::interner[str] itr) {
|
||||||
fn is_eof() -> bool { ret ch == -1 as char; }
|
fn is_eof() -> bool { ret ch == -1 as char; }
|
||||||
fn mark() { mark_pos = pos; mark_chpos = chpos; }
|
fn mark() { mark_pos = pos; mark_chpos = chpos; }
|
||||||
fn get_mark_str() -> str { ret str::slice(file, mark_pos, pos); }
|
fn get_mark_str() -> str { ret str::slice(src, mark_pos, pos); }
|
||||||
fn get_mark_chpos() -> uint { ret mark_chpos; }
|
fn get_mark_chpos() -> uint { ret mark_chpos; }
|
||||||
fn get_chpos() -> uint { ret chpos; }
|
fn get_chpos() -> uint { ret chpos; }
|
||||||
fn curr() -> char { ret ch; }
|
fn curr() -> char { ret ch; }
|
||||||
fn next() -> char {
|
fn next() -> char {
|
||||||
if (pos < len) {
|
if (pos < len) {
|
||||||
ret str::char_at(file, pos);
|
ret str::char_at(src, pos);
|
||||||
} else { ret -1 as char; }
|
} else { ret -1 as char; }
|
||||||
}
|
}
|
||||||
fn init() {
|
fn init() {
|
||||||
if (pos < len) {
|
if (pos < len) {
|
||||||
auto next = str::char_range_at(file, pos);
|
auto next = str::char_range_at(src, pos);
|
||||||
pos = next._1;
|
pos = next._1;
|
||||||
ch = next._0;
|
ch = next._0;
|
||||||
}
|
}
|
||||||
|
@ -66,7 +66,7 @@ fn new_reader(&codemap::codemap cm, io::reader rdr, codemap::filemap filemap,
|
||||||
col += 1u;
|
col += 1u;
|
||||||
chpos += 1u;
|
chpos += 1u;
|
||||||
if (ch == '\n') { codemap::next_line(fm, chpos); col = 0u; }
|
if (ch == '\n') { codemap::next_line(fm, chpos); col = 0u; }
|
||||||
auto next = str::char_range_at(file, pos);
|
auto next = str::char_range_at(src, pos);
|
||||||
pos = next._1;
|
pos = next._1;
|
||||||
ch = next._0;
|
ch = next._0;
|
||||||
} else { ch = -1 as char; }
|
} else { ch = -1 as char; }
|
||||||
|
@ -78,10 +78,9 @@ fn new_reader(&codemap::codemap cm, io::reader rdr, codemap::filemap filemap,
|
||||||
codemap::emit_error(some(rec(lo=chpos, hi=chpos)), m, cm);
|
codemap::emit_error(some(rec(lo=chpos, hi=chpos)), m, cm);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
auto file = str::unsafe_from_bytes(rdr.read_whole_stream());
|
|
||||||
let vec[str] strs = [];
|
let vec[str] strs = [];
|
||||||
auto rd =
|
auto rd =
|
||||||
reader(cm, file, str::byte_len(file), 0u, 0u, -1 as char, 0u,
|
reader(cm, src, str::byte_len(src), 0u, 0u, -1 as char, 0u,
|
||||||
filemap.start_pos, filemap.start_pos, strs, filemap, itr);
|
filemap.start_pos, filemap.start_pos, strs, filemap, itr);
|
||||||
rd.init();
|
rd.init();
|
||||||
ret rd;
|
ret rd;
|
||||||
|
@ -717,8 +716,9 @@ type lit = rec(str lit, uint pos);
|
||||||
fn gather_comments_and_literals(&codemap::codemap cm, str path) ->
|
fn gather_comments_and_literals(&codemap::codemap cm, str path) ->
|
||||||
rec(vec[cmnt] cmnts, vec[lit] lits) {
|
rec(vec[cmnt] cmnts, vec[lit] lits) {
|
||||||
auto srdr = io::file_reader(path);
|
auto srdr = io::file_reader(path);
|
||||||
|
auto src = str::unsafe_from_bytes(srdr.read_whole_stream());
|
||||||
auto itr = @interner::mk[str](str::hash, str::eq);
|
auto itr = @interner::mk[str](str::hash, str::eq);
|
||||||
auto rdr = new_reader(cm, srdr, codemap::new_filemap(path, 0u), itr);
|
auto rdr = new_reader(cm, src, codemap::new_filemap(path, 0u), itr);
|
||||||
let vec[cmnt] comments = [];
|
let vec[cmnt] comments = [];
|
||||||
let vec[lit] literals = [];
|
let vec[lit] literals = [];
|
||||||
let bool first_read = true;
|
let bool first_read = true;
|
||||||
|
|
|
@ -113,10 +113,11 @@ fn new_parser(parse_sess sess, ast::crate_cfg cfg,
|
||||||
auto ftype = SOURCE_FILE;
|
auto ftype = SOURCE_FILE;
|
||||||
if (str::ends_with(path, ".rc")) { ftype = CRATE_FILE; }
|
if (str::ends_with(path, ".rc")) { ftype = CRATE_FILE; }
|
||||||
auto srdr = io::file_reader(path);
|
auto srdr = io::file_reader(path);
|
||||||
|
auto src = str::unsafe_from_bytes(srdr.read_whole_stream());
|
||||||
auto filemap = codemap::new_filemap(path, pos);
|
auto filemap = codemap::new_filemap(path, pos);
|
||||||
vec::push(sess.cm.files, filemap);
|
vec::push(sess.cm.files, filemap);
|
||||||
auto itr = @interner::mk(str::hash, str::eq);
|
auto itr = @interner::mk(str::hash, str::eq);
|
||||||
auto rdr = lexer::new_reader(sess.cm, srdr, filemap, itr);
|
auto rdr = lexer::new_reader(sess.cm, src, filemap, itr);
|
||||||
// Make sure npos points at first actual token:
|
// Make sure npos points at first actual token:
|
||||||
|
|
||||||
lexer::consume_whitespace_and_comments(rdr);
|
lexer::consume_whitespace_and_comments(rdr);
|
||||||
|
|
Loading…
Reference in a new issue