squiid/rust
1
0
Fork 0
Code Issues Pull requests Projects Activity

Introduce EOF token

Browse source
This commit is contained in:
Aleksey Kladov 2018-01-07 21:09:05 +03:00
parent fc4d6cc298
commit 5562931e4f
3 changed files with 133 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
src/parser/event_parser/grammar.rs
View file

@ -1,5 +1,6 @@
use super::parser::Parser;
use {SyntaxKind};
use tree::EOF;
use syntax_kinds::*;
// Items //
@ -18,11 +19,7 @@ pub(crate) fn file(p: &mut Parser) {
}
fn item_first(p: &Parser) -> bool {
let current = match p.current() {
Some(c) => c,
None => return false,
};
match current {
match p.current() {
STRUCT_KW | FN_KW => true,
_ => false,
}
@ -79,7 +76,7 @@ fn visibility(_: &mut Parser) {
// Error recovery and high-order utils //
fn node_if<F: FnOnce(&mut Parser)>(p: &mut Parser, first: SyntaxKind, node_kind: SyntaxKind, rest: F) -> bool {
p.current_is(first) && { node(p, node_kind, |p| { p.bump(); rest(p); }); true }
p.current() == first && { node(p, node_kind, |p| { p.bump(); rest(p); }); true }
}
fn node<F: FnOnce(&mut Parser)>(p: &mut Parser, node_kind: SyntaxKind, rest: F) {
@ -95,7 +92,7 @@ fn many<F: Fn(&mut Parser) -> bool>(p: &mut Parser, f: F) {
fn comma_list<F: Fn(&mut Parser) -> bool>(p: &mut Parser, f: F) {
many(p, |p| {
f(p);
if p.is_eof() {
if p.current() == EOF {
false
} else {
p.expect(COMMA);
@ -119,7 +116,7 @@ where
f(p);
return true;
}
if p.is_eof() {
if p.current() == EOF {
if skipped {
p.finish();
}
@ -131,18 +128,14 @@ where
.message(message)
.emit();
}
p.bump().unwrap();
p.bump();
skipped = true;
}
}
impl<'p> Parser<'p> {
fn current_is(&self, kind: SyntaxKind) -> bool {
self.current() == Some(kind)
}
pub(crate) fn expect(&mut self, kind: SyntaxKind) -> bool {
if self.current_is(kind) {
if self.current() == kind {
self.bump();
true
} else {
@ -154,7 +147,7 @@ impl<'p> Parser<'p> {
}
fn optional(&mut self, kind: SyntaxKind) {
if self.current_is(kind) {
if self.current() == kind {
self.bump();
}
}

35
src/parser/event_parser/parser.rs
View file

@ -2,8 +2,7 @@ use {Token, SyntaxKind, TextUnit};
use super::{Event};
use super::super::is_insignificant;
use syntax_kinds::{L_CURLY, R_CURLY, ERROR};
pub(crate) const EOF: SyntaxKind = SyntaxKind(10000);
use tree::EOF;
pub(crate) struct Parser<'t> {
@ -46,19 +45,22 @@ impl<'t> Parser<'t> {
}
pub(crate) fn into_events(self) -> Vec<Event> {
assert!(self.is_eof());
assert!(self.curly_limit.is_none());
assert!(self.current() == EOF);
self.events
}
pub(crate) fn is_eof(&self) -> bool {
pub(crate) fn current(&self) -> SyntaxKind {
if self.pos == self.tokens.len() {
return true
return EOF;
}
let token = self.tokens[self.pos];
if let Some(limit) = self.curly_limit {
let token = self.tokens[self.pos];
return limit == self.curly_level && token.kind == R_CURLY;
if limit == self.curly_level && token.kind == R_CURLY {
return EOF
}
}
false
token.kind
}
pub(crate) fn start(&mut self, kind: SyntaxKind) {
@ -73,24 +75,17 @@ impl<'t> Parser<'t> {
ErrorBuilder::new(self)
}
pub(crate) fn current(&self) -> Option<SyntaxKind> {
if self.is_eof() {
return None;
}
let token = self.tokens[self.pos];
Some(token.kind)
}
pub(crate) fn bump(&mut self) -> Option<SyntaxKind> {
let kind = self.current()?;
pub(crate) fn bump(&mut self) -> SyntaxKind {
let kind = self.current();
match kind {
L_CURLY => self.curly_level += 1,
R_CURLY => self.curly_level -= 1,
EOF => return EOF,
_ => (),
}
self.pos += 1;
self.event(Event::Token { kind, n_raw_tokens: 1 });
Some(kind)
kind
}
pub(crate) fn lookahead(&self, kinds: &[SyntaxKind]) -> bool {
@ -114,7 +109,7 @@ impl<'t> Parser<'t> {
if !self.expect(R_CURLY) {
self.start(ERROR);
while self.curly_level > old_level {
if self.bump().is_none() {
if self.bump() == EOF {
break;
}
}

212
src/tree/mod.rs
View file

@ -10,8 +10,16 @@ pub use self::file_builder::{FileBuilder, Sink};
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct SyntaxKind(pub(crate) u32);
pub(crate) const EOF: SyntaxKind = SyntaxKind(10000);
pub(crate) const EOF_INFO: SyntaxInfo = SyntaxInfo {
name: "EOF"
};
impl SyntaxKind {
fn info(self) -> &'static SyntaxInfo {
if self == EOF {
return &EOF_INFO;
}
syntax_info(self)
}
}
@ -35,72 +43,72 @@ pub struct Token {
}
pub struct File {
text: String,
nodes: Vec<NodeData>,
errors: Vec<SyntaxErrorData>,
text: String,
nodes: Vec<NodeData>,
errors: Vec<SyntaxErrorData>,
}
impl File {
pub fn root<'f>(&'f self) -> Node<'f> {
assert!(!self.nodes.is_empty());
Node { file: self, idx: NodeIdx(0) }
}
pub fn root<'f>(&'f self) -> Node<'f> {
assert!(!self.nodes.is_empty());
Node { file: self, idx: NodeIdx(0) }
}
}
#[derive(Clone, Copy)]
pub struct Node<'f> {
file: &'f File,
idx: NodeIdx,
file: &'f File,
idx: NodeIdx,
}
impl<'f> Node<'f> {
pub fn kind(&self) -> SyntaxKind {
self.data().kind
}
pub fn kind(&self) -> SyntaxKind {
self.data().kind
}
pub fn range(&self) -> TextRange {
self.data().range
}
pub fn range(&self) -> TextRange {
self.data().range
}
pub fn text(&self) -> &'f str {
&self.file.text.as_str()[self.range()]
}
pub fn text(&self) -> &'f str {
&self.file.text.as_str()[self.range()]
}
pub fn parent(&self) -> Option<Node<'f>> {
self.as_node(self.data().parent)
}
pub fn parent(&self) -> Option<Node<'f>> {
self.as_node(self.data().parent)
}
pub fn children(&self) -> Children<'f> {
Children { next: self.as_node(self.data().first_child) }
}
pub fn children(&self) -> Children<'f> {
Children { next: self.as_node(self.data().first_child) }
}
pub fn errors(&self) -> SyntaxErrors<'f> {
let pos = self.file.errors.iter().position(|e| e.node == self.idx);
let next = pos
.map(|i| ErrorIdx(i as u32))
.map(|idx| SyntaxError { file: self.file, idx });
SyntaxErrors { next }
}
pub fn errors(&self) -> SyntaxErrors<'f> {
let pos = self.file.errors.iter().position(|e| e.node == self.idx);
let next = pos
.map(|i| ErrorIdx(i as u32))
.map(|idx| SyntaxError { file: self.file, idx });
SyntaxErrors { next }
}
fn data(&self) -> &'f NodeData {
&self.file.nodes[self.idx]
}
fn data(&self) -> &'f NodeData {
&self.file.nodes[self.idx]
}
fn as_node(&self, idx: Option<NodeIdx>) -> Option<Node<'f>> {
idx.map(|idx| Node { file: self.file, idx })
}
fn as_node(&self, idx: Option<NodeIdx>) -> Option<Node<'f>> {
idx.map(|idx| Node { file: self.file, idx })
}
}
impl<'f> fmt::Debug for Node<'f> {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "{:?}@{:?}", self.kind(), self.range())
}
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "{:?}@{:?}", self.kind(), self.range())
}
}
impl<'f> cmp::PartialEq<Node<'f>> for Node<'f> {
fn eq(&self, other: &Node<'f>) -> bool {
self.idx == other.idx && ::std::ptr::eq(self.file, other.file)
}
fn eq(&self, other: &Node<'f>) -> bool {
self.idx == other.idx && ::std::ptr::eq(self.file, other.file)
}
}
impl<'f> cmp::Eq for Node<'f> {
@ -108,66 +116,66 @@ impl<'f> cmp::Eq for Node<'f> {
#[derive(Clone, Copy)]
pub struct SyntaxError<'f> {
file: &'f File,
idx: ErrorIdx,
file: &'f File,
idx: ErrorIdx,
}
impl<'f> SyntaxError<'f> {
pub fn message(&self) -> &'f str {
self.data().message.as_str()
}
pub fn message(&self) -> &'f str {
self.data().message.as_str()
}
pub fn after_child(&self) -> Option<Node<'f>> {
let idx = self.data().after_child?;
Some(Node { file: self.file, idx })
}
pub fn after_child(&self) -> Option<Node<'f>> {
let idx = self.data().after_child?;
Some(Node { file: self.file, idx })
}
fn data(&self) -> &'f SyntaxErrorData {
&self.file.errors[self.idx]
}
fn data(&self) -> &'f SyntaxErrorData {
&self.file.errors[self.idx]
}
fn next(&self) -> Option<SyntaxError<'f>> {
let next_idx = self.idx.0 + 1;
if !((next_idx as usize) < self.file.errors.len()) {
return None;
}
let result = SyntaxError {
file: self.file,
idx: ErrorIdx(next_idx)
};
if result.data().node != self.data().node {
return None;
}
Some(result)
}
fn next(&self) -> Option<SyntaxError<'f>> {
let next_idx = self.idx.0 + 1;
if !((next_idx as usize) < self.file.errors.len()) {
return None;
}
let result = SyntaxError {
file: self.file,
idx: ErrorIdx(next_idx)
};
if result.data().node != self.data().node {
return None;
}
Some(result)
}
}
pub struct Children<'f> {
next: Option<Node<'f>>,
next: Option<Node<'f>>,
}
impl<'f> Iterator for Children<'f> {
type Item = Node<'f>;
type Item = Node<'f>;
fn next(&mut self) -> Option<Node<'f>> {
let next = self.next;
self.next = next.and_then(|node| node.as_node(node.data().next_sibling));
next
}
fn next(&mut self) -> Option<Node<'f>> {
let next = self.next;
self.next = next.and_then(|node| node.as_node(node.data().next_sibling));
next
}
}
pub struct SyntaxErrors<'f> {
next: Option<SyntaxError<'f>>,
next: Option<SyntaxError<'f>>,
}
impl<'f> Iterator for SyntaxErrors<'f> {
type Item = SyntaxError<'f>;
type Item = SyntaxError<'f>;
fn next(&mut self) -> Option<SyntaxError<'f>> {
let next = self.next;
self.next = next.as_ref().and_then(SyntaxError::next);
next
}
fn next(&mut self) -> Option<SyntaxError<'f>> {
let next = self.next;
self.next = next.as_ref().and_then(SyntaxError::next);
next
}
}
@ -175,40 +183,40 @@ impl<'f> Iterator for SyntaxErrors<'f> {
struct NodeIdx(u32);
struct NodeData {
kind: SyntaxKind,
range: TextRange,
parent: Option<NodeIdx>,
first_child: Option<NodeIdx>,
next_sibling: Option<NodeIdx>,
kind: SyntaxKind,
range: TextRange,
parent: Option<NodeIdx>,
first_child: Option<NodeIdx>,
next_sibling: Option<NodeIdx>,
}
impl ::std::ops::Index<NodeIdx> for Vec<NodeData> {
type Output = NodeData;
type Output = NodeData;
fn index(&self, NodeIdx(idx): NodeIdx) -> &NodeData {
&self[idx as usize]
}
fn index(&self, NodeIdx(idx): NodeIdx) -> &NodeData {
&self[idx as usize]
}
}
impl ::std::ops::IndexMut<NodeIdx> for Vec<NodeData> {
fn index_mut(&mut self, NodeIdx(idx): NodeIdx) -> &mut NodeData {
&mut self[idx as usize]
}
fn index_mut(&mut self, NodeIdx(idx): NodeIdx) -> &mut NodeData {
&mut self[idx as usize]
}
}
#[derive(Clone, Copy)]
struct ErrorIdx(u32);
struct SyntaxErrorData {
node: NodeIdx,
message: String,
after_child: Option<NodeIdx>,
node: NodeIdx,
message: String,
after_child: Option<NodeIdx>,
}
impl ::std::ops::Index<ErrorIdx> for Vec<SyntaxErrorData> {
type Output = SyntaxErrorData;
type Output = SyntaxErrorData;
fn index(&self, ErrorIdx(idx): ErrorIdx) -> &SyntaxErrorData {
&self[idx as usize]
}
fn index(&self, ErrorIdx(idx): ErrorIdx) -> &SyntaxErrorData {
&self[idx as usize]
}
}