squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

merge parse_impl and parser_api

Browse source
This commit is contained in:
Aleksey Kladov 2019-02-20 22:44:06 +03:00
parent e72ad0a2fa
commit 2acb21e8f7
2 changed files with 101 additions and 170 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
crates/ra_syntax/src/parsing/parser_api.rs
View file

@ -1,10 +1,14 @@
use std::cell::Cell;
use drop_bomb::DropBomb;
use crate::{
SyntaxKind::{self, ERROR},
syntax_error::ParseError,
SyntaxKind::{self, ERROR, EOF, TOMBSTONE},
parsing::{
TokenSource, TokenPos,
token_set::TokenSet,
parser_impl::ParserImpl,
parser_impl::event::Event,
},
};
@ -17,9 +21,22 @@ use crate::{
/// tree, but rather a flat stream of events of the form
/// "start expression, consume number literal,
/// finish expression". See `Event` docs for more.
pub(crate) struct Parser<'t>(pub(super) ParserImpl<'t>);
pub(crate) struct Parser<'t> {
token_source: &'t dyn TokenSource,
pos: TokenPos,
events: Vec<Event>,
steps: Cell<u32>,
}
impl<'t> Parser<'t> {
pub(super) fn new(token_source: &'t dyn TokenSource) -> Parser<'t> {
Parser { token_source, pos: TokenPos::default(), events: Vec::new(), steps: Cell::new(0) }
}
pub(crate) fn finish(self) -> Vec<Event> {
self.events
}
/// Returns the kind of the current token.
/// If parser has already reached the end of input,
/// the special `EOF` kind is returned.
@ -32,7 +49,13 @@ impl<'t> Parser<'t> {
///
/// Useful for parsing things like `>>`.
pub(crate) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> {
self.0.current2()
let c1 = self.token_source.token_kind(self.pos);
let c2 = self.token_source.token_kind(self.pos + 1);
if self.token_source.is_token_joint_to_next(self.pos) {
Some((c1, c2))
} else {
None
}
}
/// Returns the kinds of the current three tokens, if they are not separated
@ -40,13 +63,25 @@ impl<'t> Parser<'t> {
///
/// Useful for parsing things like `=>>`.
pub(crate) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> {
self.0.current3()
let c1 = self.token_source.token_kind(self.pos);
let c2 = self.token_source.token_kind(self.pos + 1);
let c3 = self.token_source.token_kind(self.pos + 2);
if self.token_source.is_token_joint_to_next(self.pos)
&& self.token_source.is_token_joint_to_next(self.pos + 1)
{
Some((c1, c2, c3))
} else {
None
}
}
/// Lookahead operation: returns the kind of the next nth
/// token.
pub(crate) fn nth(&self, n: u32) -> SyntaxKind {
self.0.nth(n)
let steps = self.steps.get();
assert!(steps <= 10_000_000, "the parser seems stuck");
self.steps.set(steps + 1);
self.token_source.token_kind(self.pos + n)
}
/// Checks if the current token is `kind`.
@ -60,20 +95,26 @@ impl<'t> Parser<'t> {
}
/// Checks if the current token is contextual keyword with text `t`.
pub(crate) fn at_contextual_kw(&self, t: &str) -> bool {
self.0.at_kw(t)
pub(crate) fn at_contextual_kw(&self, kw: &str) -> bool {
self.token_source.is_keyword(self.pos, kw)
}
/// Starts a new node in the syntax tree. All nodes and tokens
/// consumed between the `start` and the corresponding `Marker::complete`
/// belong to the same node.
pub(crate) fn start(&mut self) -> Marker {
Marker::new(self.0.start())
let pos = self.events.len() as u32;
self.push_event(Event::tombstone());
Marker::new(pos)
}
/// Advances the parser by one token unconditionally.
pub(crate) fn bump(&mut self) {
self.0.bump();
let kind = self.nth(0);
if kind == EOF {
return;
}
self.do_bump(kind, 1);
}
/// Advances the parser by one token, remapping its kind.
@ -83,14 +124,18 @@ impl<'t> Parser<'t> {
/// `union` keyword, and keyword is what ends up in the
/// final tree.
pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) {
self.0.bump_remap(kind);
if self.nth(0) == EOF {
// TODO: panic!?
return;
}
self.do_bump(kind, 1);
}
/// Advances the parser by `n` tokens, remapping its kind.
/// This is useful to create compound tokens from parts. For
/// example, an `<<` token is two consecutive remapped `<` tokens
pub(crate) fn bump_compound(&mut self, kind: SyntaxKind, n: u8) {
self.0.bump_compound(kind, n);
self.do_bump(kind, n);
}
/// Emit error with the `message`
@ -98,7 +143,8 @@ impl<'t> Parser<'t> {
/// structured errors with spans and notes, like rustc
/// does.
pub(crate) fn error<T: Into<String>>(&mut self, message: T) {
self.0.error(message.into())
let msg = ParseError(message.into());
self.push_event(Event::Error { msg })
}
/// Consume the next token if `kind` matches.
@ -136,6 +182,15 @@ impl<'t> Parser<'t> {
m.complete(self, ERROR);
};
}
fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) {
self.pos += u32::from(n_raw_tokens);
self.push_event(Event::Token { kind, n_raw_tokens });
}
fn push_event(&mut self, event: Event) {
self.events.push(event)
}
}
/// See `Parser::start`.
@ -154,7 +209,14 @@ impl Marker {
/// operation like `.precede()` to deal with forward_parent.
pub(crate) fn complete(mut self, p: &mut Parser, kind: SyntaxKind) -> CompletedMarker {
self.bomb.defuse();
p.0.complete(self.pos, kind);
let idx = self.pos as usize;
match p.events[idx] {
Event::Start { kind: ref mut slot, .. } => {
*slot = kind;
}
_ => unreachable!(),
}
p.push_event(Event::Finish);
CompletedMarker::new(self.pos, kind)
}
@ -162,7 +224,13 @@ impl Marker {
/// are attached to its parent instead.
pub(crate) fn abandon(mut self, p: &mut Parser) {
self.bomb.defuse();
p.0.abandon(self.pos);
let idx = self.pos as usize;
if idx == p.events.len() - 1 {
match p.events.pop() {
Some(Event::Start { kind: TOMBSTONE, forward_parent: None }) => (),
_ => unreachable!(),
}
}
}
}
@ -186,7 +254,15 @@ impl CompletedMarker {
/// then mark `NEWSTART` as `START`'s parent with saving its relative
/// distance to `NEWSTART` into forward_parent(=2 in this case);
pub(crate) fn precede(self, p: &mut Parser) -> Marker {
Marker::new(p.0.precede(self.0))
let new_pos = p.start();
let idx = self.0 as usize;
match p.events[idx] {
Event::Start { ref mut forward_parent, .. } => {
*forward_parent = Some(new_pos.pos - self.0);
}
_ => unreachable!(),
}
new_pos
}
pub(crate) fn kind(&self) -> SyntaxKind {

163
crates/ra_syntax/src/parsing/parser_impl.rs
View file

@ -1,20 +1,13 @@
mod event;
pub(crate) mod input;
pub(super) mod event;
pub(super) mod input;
use std::cell::Cell;
use crate::{
syntax_error::ParseError,
parsing::{
TreeSink, TokenSource, TokenPos,
lexer::Token,
parser_api::Parser,
parser_impl::event::{Event, EventProcessor},
},
use crate::parsing::{
TreeSink, TokenSource,
lexer::Token,
parser_api::Parser,
parser_impl::event::EventProcessor,
};
use crate::SyntaxKind::{self, EOF, TOMBSTONE};
/// Parse a sequence of tokens into the representative node tree
pub(super) fn parse_with<S: TreeSink>(
sink: S,
@ -24,147 +17,9 @@ pub(super) fn parse_with<S: TreeSink>(
) -> S::Tree {
let mut events = {
let input = input::ParserInput::new(text, tokens);
let parser_impl = ParserImpl::new(&input);
let mut parser_api = Parser(parser_impl);
let mut parser_api = Parser::new(&input);
parser(&mut parser_api);
parser_api.0.into_events()
parser_api.finish()
};
EventProcessor::new(sink, text, tokens, &mut events).process().finish()
}
/// Implementation details of `Parser`, extracted
/// to a separate struct in order not to pollute
/// the public API of the `Parser`.
pub(super) struct ParserImpl<'a> {
token_source: &'a dyn TokenSource,
pos: TokenPos,
events: Vec<Event>,
steps: Cell<u32>,
}
impl<'a> ParserImpl<'a> {
fn new(token_source: &'a dyn TokenSource) -> ParserImpl<'a> {
ParserImpl {
token_source,
pos: TokenPos::default(),
events: Vec::new(),
steps: Cell::new(0),
}
}
fn into_events(self) -> Vec<Event> {
assert_eq!(self.nth(0), EOF);
self.events
}
pub(super) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> {
let c1 = self.token_source.token_kind(self.pos);
let c2 = self.token_source.token_kind(self.pos + 1);
if self.token_source.is_token_joint_to_next(self.pos) {
Some((c1, c2))
} else {
None
}
}
pub(super) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> {
let c1 = self.token_source.token_kind(self.pos);
let c2 = self.token_source.token_kind(self.pos + 1);
let c3 = self.token_source.token_kind(self.pos + 2);
if self.token_source.is_token_joint_to_next(self.pos)
&& self.token_source.is_token_joint_to_next(self.pos + 1)
{
Some((c1, c2, c3))
} else {
None
}
}
/// Get the syntax kind of the nth token.
pub(super) fn nth(&self, n: u32) -> SyntaxKind {
let steps = self.steps.get();
assert!(steps <= 10_000_000, "the parser seems stuck");
self.steps.set(steps + 1);
self.token_source.token_kind(self.pos + n)
}
pub(super) fn at_kw(&self, kw: &str) -> bool {
self.token_source.is_keyword(self.pos, kw)
}
/// Start parsing right behind the last event.
pub(super) fn start(&mut self) -> u32 {
let pos = self.events.len() as u32;
self.push_event(Event::tombstone());
pos
}
/// Advances the parser by one token unconditionally.
pub(super) fn bump(&mut self) {
let kind = self.nth(0);
if kind == EOF {
return;
}
self.do_bump(kind, 1);
}
pub(super) fn bump_remap(&mut self, kind: SyntaxKind) {
if self.nth(0) == EOF {
// TODO: panic!?
return;
}
self.do_bump(kind, 1);
}
pub(super) fn bump_compound(&mut self, kind: SyntaxKind, n: u8) {
self.do_bump(kind, n);
}
fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) {
self.pos += u32::from(n_raw_tokens);
self.push_event(Event::Token { kind, n_raw_tokens });
}
/// Append one Error event to the back of events.
pub(super) fn error(&mut self, msg: String) {
self.push_event(Event::Error { msg: ParseError(msg) })
}
/// Complete an event with appending a `Finish` event.
pub(super) fn complete(&mut self, pos: u32, kind: SyntaxKind) {
match self.events[pos as usize] {
Event::Start { kind: ref mut slot, .. } => {
*slot = kind;
}
_ => unreachable!(),
}
self.push_event(Event::Finish);
}
/// Ignore the dummy `Start` event.
pub(super) fn abandon(&mut self, pos: u32) {
let idx = pos as usize;
if idx == self.events.len() - 1 {
match self.events.pop() {
Some(Event::Start { kind: TOMBSTONE, forward_parent: None }) => (),
_ => unreachable!(),
}
}
}
/// Save the relative distance of a completed event to its forward_parent.
pub(super) fn precede(&mut self, pos: u32) -> u32 {
let new_pos = self.start();
match self.events[pos as usize] {
Event::Start { ref mut forward_parent, .. } => {
*forward_parent = Some(new_pos - pos);
}
_ => unreachable!(),
}
new_pos
}
fn push_event(&mut self, event: Event) {
self.events.push(event)
}
}