kill old tree

This commit is contained in:
Aleksey Kladov 2018-07-29 14:37:48 +03:00
parent c12450fb4e
commit ad188d4c3d
11 changed files with 21 additions and 444 deletions

View file

@ -26,52 +26,21 @@ mod yellow;
pub mod syntax_kinds;
pub use text_unit::{TextRange, TextUnit};
pub use tree::{File, Node, SyntaxKind, Token};
pub(crate) use tree::{ErrorMsg, FileBuilder, Sink, GreenBuilder};
pub use tree::{SyntaxKind, Token};
pub(crate) use tree::{Sink, GreenBuilder};
pub use lexer::{next_token, tokenize};
pub use yellow::SyntaxNode;
pub(crate) use yellow::SError;
pub use parser::{parse, parse_green};
pub use parser::{parse_green};
/// Utilities for simple uses of the parser.
pub mod utils {
use std::fmt::Write;
use {File, Node, SyntaxNode};
use {SyntaxNode};
use std::collections::BTreeSet;
use SError;
/// Parse a file and create a string representation of the resulting parse tree.
pub fn dump_tree(file: &File) -> String {
let mut result = String::new();
go(file.root(), &mut result, 0);
return result;
fn go(node: Node, buff: &mut String, level: usize) {
buff.push_str(&String::from(" ").repeat(level));
write!(buff, "{:?}\n", node).unwrap();
let my_errors = node.errors().filter(|e| e.after_child().is_none());
let parent_errors = node.parent()
.into_iter()
.flat_map(|n| n.errors())
.filter(|e| e.after_child() == Some(node));
for err in my_errors {
buff.push_str(&String::from(" ").repeat(level));
write!(buff, "err: `{}`\n", err.message()).unwrap();
}
for child in node.children() {
go(child, buff, level + 1)
}
for err in parent_errors {
buff.push_str(&String::from(" ").repeat(level));
write!(buff, "err: `{}`\n", err.message()).unwrap();
}
}
}
/// Parse a file and create a string representation of the resulting parse tree.
pub fn dump_tree_green(syntax: &SyntaxNode) -> String {
let mut errors: BTreeSet<_> = syntax.root.errors.iter().cloned().collect();
@ -82,12 +51,6 @@ pub mod utils {
fn go(node: &SyntaxNode, buff: &mut String, level: usize, errors: &mut BTreeSet<SError>) {
buff.push_str(&String::from(" ").repeat(level));
write!(buff, "{:?}\n", node).unwrap();
// let my_errors = node.errors().filter(|e| e.after_child().is_none());
// let parent_errors = node.parent()
// .into_iter()
// .flat_map(|n| n.errors())
// .filter(|e| e.after_child() == Some(node));
//
let my_errors: Vec<_> = errors.iter().filter(|e| e.offset == node.range().start())
.cloned().collect();
for err in my_errors {

View file

@ -1,5 +1,5 @@
use {
ErrorMsg, File, FileBuilder, Sink, SyntaxKind, Token, GreenBuilder,
Sink, SyntaxKind, Token,
syntax_kinds::TOMBSTONE,
};
use super::is_insignificant;
@ -67,12 +67,6 @@ pub(crate) enum Event {
},
}
pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File {
let mut builder = FileBuilder::new(text);
process(&mut builder, tokens, events);
builder.finish()
}
pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec<Event>) {
let mut idx = 0;
@ -147,7 +141,7 @@ pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec<Event>)
}
builder.leaf(kind, len);
}
&Event::Error { ref msg } => builder.error(ErrorMsg { msg: msg.clone() }),
&Event::Error { ref msg } => builder.error(msg.clone()),
}
}
}

View file

@ -7,7 +7,7 @@ mod grammar;
use std::sync::Arc;
use {
File, SyntaxKind, Token,
Token,
yellow::SyntaxNode,
syntax_kinds::*
};
@ -15,18 +15,6 @@ use GreenBuilder;
use parser::event::process;
/// Parse a sequence of tokens into the representative node tree
pub fn parse(text: String, tokens: &[Token]) -> File {
let events = {
let input = input::ParserInput::new(&text, tokens);
let parser_impl = parser::imp::ParserImpl::new(&input);
let mut parser = parser::Parser(parser_impl);
grammar::file(&mut parser);
parser.0.into_events()
};
event::to_file(text, tokens, events)
}
/// Parse a sequence of tokens into the representative node tree
pub fn parse_green(text: String, tokens: &[Token]) -> SyntaxNode {
let events = {

View file

@ -12,157 +12,13 @@ use {
SyntaxKind, TextRange, TextUnit,
yellow::GreenNode
};
use super::{File, NodeData, NodeIdx, SyntaxErrorData};
use SError;
pub(crate) trait Sink {
fn leaf(&mut self, kind: SyntaxKind, len: TextUnit);
fn start_internal(&mut self, kind: SyntaxKind);
fn finish_internal(&mut self);
fn error(&mut self, err: ErrorMsg);
}
#[derive(Debug)]
pub(crate) struct FileBuilder {
text: String,
nodes: Vec<NodeData>,
errors: Vec<SyntaxErrorData>,
in_progress: Vec<(NodeIdx, Option<NodeIdx>)>,
// (parent, last_child)
pos: TextUnit,
}
impl Sink for FileBuilder {
fn leaf(&mut self, kind: SyntaxKind, len: TextUnit) {
let leaf = NodeData {
kind,
range: TextRange::offset_len(self.pos, len),
parent: None,
first_child: None,
next_sibling: None,
};
self.pos += len;
let id = self.push_child(leaf);
self.add_len(id);
}
fn start_internal(&mut self, kind: SyntaxKind) {
let node = NodeData {
kind,
range: TextRange::offset_len(self.pos, 0.into()),
parent: None,
first_child: None,
next_sibling: None,
};
let id = if self.in_progress.is_empty() {
self.new_node(node)
} else {
self.push_child(node)
};
self.in_progress.push((id, None))
}
fn finish_internal(&mut self) {
let (id, _) = self.in_progress
.pop()
.expect("trying to complete a node, but there are no in-progress nodes");
if !self.in_progress.is_empty() {
self.add_len(id);
}
}
fn error(&mut self, err: ErrorMsg) {
let &(node, after_child) = self.in_progress.last().unwrap();
self.errors.push(SyntaxErrorData {
node,
message: err.msg,
after_child,
})
}
}
impl FileBuilder {
pub fn new(text: String) -> FileBuilder {
FileBuilder {
text,
nodes: Vec::new(),
errors: Vec::new(),
in_progress: Vec::new(),
pos: 0.into(),
}
}
pub fn finish(self) -> File {
assert!(
self.in_progress.is_empty(),
"some nodes in FileBuilder are unfinished: {:?}",
self.in_progress
.iter()
.map(|&(idx, _)| self.nodes[idx].kind)
.collect::<Vec<_>>()
);
assert_eq!(
self.pos,
(self.text.len() as u32).into(),
"nodes in FileBuilder do not cover the whole file"
);
File {
text: self.text,
nodes: self.nodes,
errors: self.errors,
}
}
fn new_node(&mut self, data: NodeData) -> NodeIdx {
let id = NodeIdx(self.nodes.len() as u32);
self.nodes.push(data);
id
}
fn push_child(&mut self, mut child: NodeData) -> NodeIdx {
child.parent = Some(self.current_id());
let id = self.new_node(child);
{
let (parent, sibling) = *self.in_progress.last().unwrap();
let slot = if let Some(idx) = sibling {
&mut self.nodes[idx].next_sibling
} else {
&mut self.nodes[parent].first_child
};
fill(slot, id);
}
self.in_progress.last_mut().unwrap().1 = Some(id);
id
}
fn add_len(&mut self, child: NodeIdx) {
let range = self.nodes[child].range;
grow(&mut self.current_parent().range, range);
}
fn current_id(&self) -> NodeIdx {
self.in_progress.last().unwrap().0
}
fn current_parent(&mut self) -> &mut NodeData {
let idx = self.current_id();
&mut self.nodes[idx]
}
}
fn fill<T>(slot: &mut Option<T>, value: T) {
assert!(slot.is_none());
*slot = Some(value);
}
fn grow(left: &mut TextRange, right: TextRange) {
assert_eq!(left.end(), right.start());
*left = TextRange::from_to(left.start(), right.end())
}
#[derive(Default)]
pub(crate) struct ErrorMsg {
pub(crate) msg: String,
fn error(&mut self, err: String);
}
pub(crate) struct GreenBuilder {
@ -216,8 +72,8 @@ impl Sink for GreenBuilder {
}
}
fn error(&mut self, err: ErrorMsg) {
self.errors.push(SError { message: err.msg, offset: self.pos })
fn error(&mut self, message: String) {
self.errors.push(SError { message, offset: self.pos })
}
}
impl SyntaxKind {

View file

@ -1,8 +1,8 @@
mod file_builder;
use ::{TextRange, TextUnit};
use std::{fmt, cmp};
pub(crate) use self::file_builder::{ErrorMsg, FileBuilder, Sink, GreenBuilder};
use ::{TextUnit};
use std::{fmt};
pub(crate) use self::file_builder::{Sink, GreenBuilder};
pub use syntax_kinds::SyntaxKind;
@ -25,218 +25,3 @@ pub struct Token {
/// The length of the token.
pub len: TextUnit,
}
/// The contents of a Rust source file.
#[derive(Debug)]
pub struct File {
text: String,
nodes: Vec<NodeData>,
errors: Vec<SyntaxErrorData>,
}
impl File {
/// The root node of this source file.
pub fn root<'f>(&'f self) -> Node<'f> {
assert!(!self.nodes.is_empty());
Node {
file: self,
idx: NodeIdx(0),
}
}
}
/// A reference to a token in a Rust source file.
#[derive(Clone, Copy)]
pub struct Node<'f> {
file: &'f File,
idx: NodeIdx,
}
impl<'f> Node<'f> {
/// The kind of the token at this node.
pub fn kind(&self) -> SyntaxKind {
self.data().kind
}
/// The text range covered by the token at this node.
pub fn range(&self) -> TextRange {
self.data().range
}
/// The text at this node.
pub fn text(&self) -> &'f str {
&self.file.text.as_str()[self.range()]
}
/// The parent node to this node.
pub fn parent(&self) -> Option<Node<'f>> {
self.as_node(self.data().parent)
}
/// The children nodes of this node.
pub fn children(&self) -> Children<'f> {
Children {
next: self.as_node(self.data().first_child),
}
}
/// Any errors contained in this node.
pub fn errors(&self) -> SyntaxErrors<'f> {
let pos = self.file.errors.iter().position(|e| e.node == self.idx);
let next = pos.map(|i| ErrorIdx(i as u32)).map(|idx| SyntaxError {
file: self.file,
idx,
});
SyntaxErrors { next }
}
fn data(&self) -> &'f NodeData {
&self.file.nodes[self.idx]
}
fn as_node(&self, idx: Option<NodeIdx>) -> Option<Node<'f>> {
idx.map(|idx| Node {
file: self.file,
idx,
})
}
}
impl<'f> fmt::Debug for Node<'f> {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "{:?}@{:?}", self.kind(), self.range())?;
if has_short_text(self.kind()) {
write!(fmt, " \"{}\"", self.text())?;
}
Ok(())
}
}
fn has_short_text(kind: SyntaxKind) -> bool {
use syntax_kinds::*;
match kind {
IDENT | LIFETIME => true,
_ => false,
}
}
impl<'f> cmp::PartialEq<Node<'f>> for Node<'f> {
fn eq(&self, other: &Node<'f>) -> bool {
self.idx == other.idx && ::std::ptr::eq(self.file, other.file)
}
}
impl<'f> cmp::Eq for Node<'f> {}
#[derive(Clone, Copy, Debug)]
pub struct SyntaxError<'f> {
file: &'f File,
idx: ErrorIdx,
}
impl<'f> SyntaxError<'f> {
pub fn message(&self) -> &'f str {
self.data().message.as_str()
}
pub fn after_child(&self) -> Option<Node<'f>> {
let idx = self.data().after_child?;
Some(Node {
file: self.file,
idx,
})
}
fn data(&self) -> &'f SyntaxErrorData {
&self.file.errors[self.idx]
}
fn next(&self) -> Option<SyntaxError<'f>> {
let next_idx = self.idx.0 + 1;
if !((next_idx as usize) < self.file.errors.len()) {
return None;
}
let result = SyntaxError {
file: self.file,
idx: ErrorIdx(next_idx),
};
if result.data().node != self.data().node {
return None;
}
Some(result)
}
}
#[derive(Debug)]
pub struct Children<'f> {
next: Option<Node<'f>>,
}
impl<'f> Iterator for Children<'f> {
type Item = Node<'f>;
fn next(&mut self) -> Option<Node<'f>> {
let next = self.next;
self.next = next.and_then(|node| node.as_node(node.data().next_sibling));
next
}
}
#[derive(Debug)]
pub struct SyntaxErrors<'f> {
next: Option<SyntaxError<'f>>,
}
impl<'f> Iterator for SyntaxErrors<'f> {
type Item = SyntaxError<'f>;
fn next(&mut self) -> Option<SyntaxError<'f>> {
let next = self.next;
self.next = next.as_ref().and_then(SyntaxError::next);
next
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct NodeIdx(u32);
#[derive(Debug)]
struct NodeData {
kind: SyntaxKind,
range: TextRange,
parent: Option<NodeIdx>,
first_child: Option<NodeIdx>,
next_sibling: Option<NodeIdx>,
}
impl ::std::ops::Index<NodeIdx> for Vec<NodeData> {
type Output = NodeData;
fn index(&self, NodeIdx(idx): NodeIdx) -> &NodeData {
&self[idx as usize]
}
}
impl ::std::ops::IndexMut<NodeIdx> for Vec<NodeData> {
fn index_mut(&mut self, NodeIdx(idx): NodeIdx) -> &mut NodeData {
&mut self[idx as usize]
}
}
#[derive(Clone, Copy, Debug)]
struct ErrorIdx(u32);
#[derive(Debug)]
struct SyntaxErrorData {
node: NodeIdx,
message: String,
after_child: Option<NodeIdx>,
}
impl ::std::ops::Index<ErrorIdx> for Vec<SyntaxErrorData> {
type Output = SyntaxErrorData;
fn index(&self, ErrorIdx(idx): ErrorIdx) -> &SyntaxErrorData {
&self[idx as usize]
}
}

View file

@ -116,13 +116,6 @@ impl GreenNode {
GreenNodeData::Branch(_) => false
}
}
pub(crate) fn leaf_text(&self) -> &str {
match &self.data {
GreenNodeData::Leaf(l) => l.text.as_str(),
GreenNodeData::Branch(_) => panic!("not a leaf")
}
}
}
#[derive(Debug)]

View file

@ -4,7 +4,6 @@ mod syntax;
use std::{
sync::{Arc, Weak},
ops::Deref,
mem
};
pub(crate) use self::{

View file

@ -1,6 +1,6 @@
use std::sync::{Arc, Weak, RwLock};
use std::sync::{Arc, RwLock};
use {
TextUnit, SyntaxKind, TextRange,
TextUnit,
yellow::{Ptr, GreenNode, TextLen}
};

View file

@ -7,7 +7,6 @@ use {
TextRange, TextUnit, SyntaxKind,
yellow::{Ptr, RedNode, GreenNode, TextLen},
};
use yellow::green::GreenTrivia;
#[derive(Clone)]
pub struct SyntaxNode {

View file

@ -1,8 +1,8 @@
extern crate libsyntax2;
extern crate testutils;
use libsyntax2::{parse, tokenize, parse_green};
use libsyntax2::utils::{dump_tree, dump_tree_green};
use libsyntax2::{tokenize, parse_green};
use libsyntax2::utils::{dump_tree_green};
use testutils::dir_tests;
#[test]

View file

@ -2,14 +2,14 @@ extern crate libsyntax2;
use std::io::Read;
use libsyntax2::{parse, tokenize};
use libsyntax2::utils::dump_tree;
use libsyntax2::{parse_green, tokenize};
use libsyntax2::utils::dump_tree_green;
fn main() {
let text = read_input();
let tokens = tokenize(&text);
let file = parse(text, &tokens);
let tree = dump_tree(&file);
let file = parse_green(text, &tokens);
let tree = dump_tree_green(&file);
println!("{}", tree);
}