Merge #861

861: Move parsing to a separate module r=matklad a=matklad



Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

crates/ra_syntax/src/lib.rs

@@ -16,30 +16,29 @@
 #![allow(missing_docs)]
 //#![warn(unreachable_pub)] // rust-lang/rust#47816
 
+mod syntax_kinds;
+mod syntax_node;
+mod syntax_text;
+mod syntax_error;
+mod parsing;
+mod string_lexing;
+mod validation;
+mod ptr;
+
 pub mod algo;
 pub mod ast;
-mod lexer;
-#[macro_use]
-mod token_set;
-mod grammar;
-mod parser_api;
-mod parser_impl;
-mod reparsing;
-mod string_lexing;
-mod syntax_kinds;
 /// Utilities for simple uses of the parser.
 pub mod utils;
-mod validation;
-mod syntax_node;
-mod ptr;
 
 pub use rowan::{SmolStr, TextRange, TextUnit};
 pub use crate::{
     ast::AstNode,
-    lexer::{tokenize, Token},
     syntax_kinds::SyntaxKind,
-    syntax_node::{Direction, SyntaxError, SyntaxNode, WalkEvent, Location, TreeArc},
+    syntax_error::{SyntaxError, SyntaxErrorKind, Location},
+    syntax_text::SyntaxText,
+    syntax_node::{Direction,  SyntaxNode, WalkEvent, TreeArc},
     ptr::{SyntaxNodePtr, AstPtr},
+    parsing::{tokenize, Token},
 };
 
 use ra_text_edit::AtomTextEdit;
@@ -59,9 +58,7 @@ impl SourceFile {
     }
 
     pub fn parse(text: &str) -> TreeArc<SourceFile> {
-        let tokens = tokenize(&text);
-        let (green, errors) =
-            parser_impl::parse_with(syntax_node::GreenBuilder::new(), text, &tokens, grammar::root);
+        let (green, errors) = parsing::parse_text(text);
         SourceFile::new(green, errors)
     }
 
@@ -70,7 +67,7 @@ impl SourceFile {
     }
 
     pub fn incremental_reparse(&self, edit: &AtomTextEdit) -> Option<TreeArc<SourceFile>> {
-        reparsing::incremental_reparse(self.syntax(), edit, self.errors())
+        parsing::incremental_reparse(self.syntax(), edit, self.errors())
             .map(|(green_node, errors)| SourceFile::new(green_node, errors))
     }
 

crates/ra_syntax/src/parsing.rs

@@ -0,0 +1,25 @@
+#[macro_use]
+mod token_set;
+mod builder;
+mod lexer;
+mod parser_impl;
+mod parser_api;
+mod grammar;
+mod reparsing;
+
+use crate::{
+    SyntaxError,
+    parsing::builder::GreenBuilder,
+    syntax_node::GreenNode,
+};
+
+pub use self::lexer::{tokenize, Token};
+
+pub(crate) use self::reparsing::incremental_reparse;
+
+pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
+    let tokens = tokenize(&text);
+    let (green, errors) =
+        parser_impl::parse_with(GreenBuilder::new(), text, &tokens, grammar::root);
+    (green, errors)
+}

crates/ra_syntax/src/parsing/builder.rs

@@ -1,8 +1,9 @@
 use crate::{
-    parser_impl::Sink,
-    syntax_node::{GreenNode, RaTypes, SyntaxError},
-    SmolStr, SyntaxKind,
+    parsing::parser_impl::Sink,
+    syntax_node::{GreenNode, RaTypes},
+    SmolStr, SyntaxKind, SyntaxError,
 };
+
 use rowan::GreenNodeBuilder;
 
 pub(crate) struct GreenBuilder {

crates/ra_syntax/src/parsing/grammar.rs

@@ -36,26 +36,46 @@ mod type_args;
 mod type_params;
 mod types;
 
-pub(crate) use self::{
-    expressions::block,
-    items::{
-        enum_variant_list, extern_item_list, impl_item_list, match_arm_list, mod_item_list,
-        named_field_def_list, named_field_list, token_tree, trait_item_list, use_tree_list,
+use crate::{
+    SyntaxNode,
+    SyntaxKind::{self, *},
+    parsing::{
+        token_set::TokenSet,
+        parser_api::{CompletedMarker, Marker, Parser}
     },
 };
-use crate::{
-    parser_api::{CompletedMarker, Marker, Parser},
-    token_set::TokenSet,
-    SyntaxKind::{self, *},
-};
 
-pub(crate) fn root(p: &mut Parser) {
+pub(super) fn root(p: &mut Parser) {
     let m = p.start();
     p.eat(SHEBANG);
     items::mod_contents(p, false);
     m.complete(p, SOURCE_FILE);
 }
 
+pub(super) fn reparser(node: &SyntaxNode) -> Option<fn(&mut Parser)> {
+    let res = match node.kind() {
+        BLOCK => expressions::block,
+        NAMED_FIELD_DEF_LIST => items::named_field_def_list,
+        NAMED_FIELD_LIST => items::named_field_list,
+        ENUM_VARIANT_LIST => items::enum_variant_list,
+        MATCH_ARM_LIST => items::match_arm_list,
+        USE_TREE_LIST => items::use_tree_list,
+        EXTERN_ITEM_LIST => items::extern_item_list,
+        TOKEN_TREE if node.first_child().unwrap().kind() == L_CURLY => items::token_tree,
+        ITEM_LIST => {
+            let parent = node.parent().unwrap();
+            match parent.kind() {
+                IMPL_BLOCK => items::impl_item_list,
+                TRAIT_DEF => items::trait_item_list,
+                MODULE => items::mod_item_list,
+                _ => return None,
+            }
+        }
+        _ => return None,
+    };
+    Some(res)
+}
+
 #[derive(Clone, Copy, PartialEq, Eq)]
 enum BlockLike {
     Block,

crates/ra_syntax/src/parsing/grammar/expressions.rs

@@ -138,7 +138,7 @@ enum Op {
 }
 
 fn current_op(p: &Parser) -> (u8, Op) {
-    if let Some(t) = p.next3() {
+    if let Some(t) = p.current3() {
         match t {
             (L_ANGLE, L_ANGLE, EQ) => return (1, Op::Composite(SHLEQ, 3)),
             (R_ANGLE, R_ANGLE, EQ) => return (1, Op::Composite(SHREQ, 3)),
@@ -146,7 +146,7 @@ fn current_op(p: &Parser) -> (u8, Op) {
         }
     }
 
-    if let Some(t) = p.next2() {
+    if let Some(t) = p.current2() {
         match t {
             (PLUS, EQ) => return (1, Op::Composite(PLUSEQ, 2)),
             (MINUS, EQ) => return (1, Op::Composite(MINUSEQ, 2)),

crates/ra_syntax/src/parsing/lexer/comments.rs

@@ -1,4 +1,4 @@
-use crate::lexer::ptr::Ptr;
+use crate::parsing::lexer::ptr::Ptr;
 
 use crate::SyntaxKind::{self, *};
 

crates/ra_syntax/src/parsing/lexer/numbers.rs

@@ -1,5 +1,7 @@
-use crate::lexer::classes::*;
-use crate::lexer::ptr::Ptr;
+use crate::parsing::lexer::{
+    ptr::Ptr,
+    classes::*,
+};
 
 use crate::SyntaxKind::{self, *};
 

crates/ra_syntax/src/parsing/lexer/strings.rs

@@ -1,6 +1,7 @@
-use crate::SyntaxKind::{self, *};
-
-use crate::lexer::ptr::Ptr;
+use crate::{
+    parsing::lexer::ptr::Ptr,
+    SyntaxKind::{self, *},
+};
 
 pub(crate) fn is_string_literal_start(c: char, c1: Option<char>, c2: Option<char>) -> bool {
     match (c, c1, c2) {

crates/ra_syntax/src/parsing/parser_api.rs

@@ -1,9 +1,11 @@
 use drop_bomb::DropBomb;
 
 use crate::{
-    parser_impl::ParserImpl,
-    token_set::TokenSet,
     SyntaxKind::{self, ERROR},
+    parsing::{
+        token_set::TokenSet,
+        parser_impl::ParserImpl
+    },
 };
 
 /// `Parser` struct provides the low-level API for
@@ -25,6 +27,22 @@ impl<'t> Parser<'t> {
         self.nth(0)
     }
 
+    /// Returns the kinds of the current two tokens, if they are not separated
+    /// by trivia.
+    ///
+    /// Useful for parsing things like `>>`.
+    pub(crate) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> {
+        self.0.current2()
+    }
+
+    /// Returns the kinds of the current three tokens, if they are not separated
+    /// by trivia.
+    ///
+    /// Useful for parsing things like `=>>`.
+    pub(crate) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> {
+        self.0.current3()
+    }
+
     /// Lookahead operation: returns the kind of the next nth
     /// token.
     pub(crate) fn nth(&self, n: u32) -> SyntaxKind {
@@ -41,14 +59,6 @@ impl<'t> Parser<'t> {
         kinds.contains(self.current())
     }
 
-    pub(crate) fn next2(&self) -> Option<(SyntaxKind, SyntaxKind)> {
-        self.0.next2()
-    }
-
-    pub(crate) fn next3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> {
-        self.0.next3()
-    }
-
     /// Checks if the current token is contextual keyword with text `t`.
     pub(crate) fn at_contextual_kw(&self, t: &str) -> bool {
         self.0.at_kw(t)

crates/ra_syntax/src/parsing/parser_impl.rs

@@ -4,22 +4,21 @@ mod input;
 use std::cell::Cell;
 
 use crate::{
-    lexer::Token,
-    parser_api::Parser,
-    parser_impl::{
-        event::{Event, EventProcessor},
-        input::{InputPosition, ParserInput},
-    },
     SmolStr,
-    syntax_node::syntax_error::{
-        ParseError,
-        SyntaxError,
+    syntax_error::{ParseError, SyntaxError},
+    parsing::{
+        lexer::Token,
+        parser_api::Parser,
+        parser_impl::{
+            event::{Event, EventProcessor},
+            input::{InputPosition, ParserInput},
+        },
     },
 };
 
 use crate::SyntaxKind::{self, EOF, TOMBSTONE};
 
-pub(crate) trait Sink {
+pub(super) trait Sink {
     type Tree;
 
     /// Adds new leaf to the current branch.
@@ -41,7 +40,7 @@ pub(crate) trait Sink {
 }
 
 /// Parse a sequence of tokens into the representative node tree
-pub(crate) fn parse_with<S: Sink>(
+pub(super) fn parse_with<S: Sink>(
     sink: S,
     text: &str,
     tokens: &[Token],
@@ -60,7 +59,7 @@ pub(crate) fn parse_with<S: Sink>(
 /// Implementation details of `Parser`, extracted
 /// to a separate struct in order not to pollute
 /// the public API of the `Parser`.
-pub(crate) struct ParserImpl<'t> {
+pub(super) struct ParserImpl<'t> {
     parser_input: &'t ParserInput<'t>,
     pos: InputPosition,
     events: Vec<Event>,
@@ -68,7 +67,7 @@ pub(crate) struct ParserImpl<'t> {
 }
 
 impl<'t> ParserImpl<'t> {
-    pub(crate) fn new(inp: &'t ParserInput<'t>) -> ParserImpl<'t> {
+    fn new(inp: &'t ParserInput<'t>) -> ParserImpl<'t> {
         ParserImpl {
             parser_input: inp,
             pos: InputPosition::new(),
@@ -77,12 +76,12 @@ impl<'t> ParserImpl<'t> {
         }
     }
 
-    pub(crate) fn into_events(self) -> Vec<Event> {
+    fn into_events(self) -> Vec<Event> {
         assert_eq!(self.nth(0), EOF);
         self.events
     }
 
-    pub(super) fn next2(&self) -> Option<(SyntaxKind, SyntaxKind)> {
+    pub(super) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> {
         let c1 = self.parser_input.kind(self.pos);
         let c2 = self.parser_input.kind(self.pos + 1);
         if self.parser_input.token_start_at(self.pos + 1)
@@ -94,7 +93,7 @@ impl<'t> ParserImpl<'t> {
         }
     }
 
-    pub(super) fn next3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> {
+    pub(super) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> {
         let c1 = self.parser_input.kind(self.pos);
         let c2 = self.parser_input.kind(self.pos + 1);
         let c3 = self.parser_input.kind(self.pos + 2);

crates/ra_syntax/src/parsing/parser_impl/event.rs

@@ -7,19 +7,22 @@
 //! tree builder: the parser produces a stream of events like
 //! `start node`, `finish node`, and `FileBuilder` converts
 //! this stream to a real tree.
+use std::mem;
+
 use crate::{
-    lexer::Token,
-    parser_impl::Sink,
     SmolStr,
     SyntaxKind::{self, *},
     TextRange, TextUnit,
-    syntax_node::syntax_error::{
+    syntax_error::{
         ParseError,
         SyntaxError,
         SyntaxErrorKind,
     },
+    parsing::{
+        lexer::Token,
+        parser_impl::Sink,
+    },
 };
-use std::mem;
 
 /// `Parser` produces a flat list of `Event`s.
 /// They are converted to a tree-structure in

crates/ra_syntax/src/parsing/parser_impl/input.rs

@@ -1,4 +1,7 @@
-use crate::{lexer::Token, SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit};
+use crate::{
+    SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit,
+    parsing::lexer::Token,
+};
 
 use std::ops::{Add, AddAssign};
 

crates/ra_syntax/src/parsing/reparsing.rs

@@ -1,10 +1,17 @@
-use crate::algo;
-use crate::grammar;
-use crate::lexer::{tokenize, Token};
-use crate::parser_api::Parser;
-use crate::parser_impl;
-use crate::syntax_node::{self, GreenNode, SyntaxError, SyntaxNode};
-use crate::{SyntaxKind::*, TextRange, TextUnit};
+use crate::{
+    SyntaxKind::*, TextRange, TextUnit,
+    algo,
+    syntax_node::{GreenNode, SyntaxNode},
+    syntax_error::SyntaxError,
+    parsing::{
+        grammar,
+        parser_impl,
+        builder::GreenBuilder,
+        parser_api::Parser,
+        lexer::{tokenize, Token},
+    }
+};
+
 use ra_text_edit::AtomTextEdit;
 
 pub(crate) fn incremental_reparse(
@@ -56,7 +63,7 @@ fn reparse_block<'node>(
         return None;
     }
     let (green, new_errors) =
-        parser_impl::parse_with(syntax_node::GreenBuilder::new(), &text, &tokens, reparser);
+        parser_impl::parse_with(GreenBuilder::new(), &text, &tokens, reparser);
     Some((node, green, new_errors))
 }
 
@@ -72,34 +79,12 @@ fn is_contextual_kw(text: &str) -> bool {
     }
 }
 
-type ParseFn = fn(&mut Parser);
-fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(&SyntaxNode, ParseFn)> {
+fn find_reparsable_node(
+    node: &SyntaxNode,
+    range: TextRange,
+) -> Option<(&SyntaxNode, fn(&mut Parser))> {
     let node = algo::find_covering_node(node, range);
-    return node.ancestors().filter_map(|node| reparser(node).map(|r| (node, r))).next();
-
-    fn reparser(node: &SyntaxNode) -> Option<ParseFn> {
-        let res = match node.kind() {
-            BLOCK => grammar::block,
-            NAMED_FIELD_DEF_LIST => grammar::named_field_def_list,
-            NAMED_FIELD_LIST => grammar::named_field_list,
-            ENUM_VARIANT_LIST => grammar::enum_variant_list,
-            MATCH_ARM_LIST => grammar::match_arm_list,
-            USE_TREE_LIST => grammar::use_tree_list,
-            EXTERN_ITEM_LIST => grammar::extern_item_list,
-            TOKEN_TREE if node.first_child().unwrap().kind() == L_CURLY => grammar::token_tree,
-            ITEM_LIST => {
-                let parent = node.parent().unwrap();
-                match parent.kind() {
-                    IMPL_BLOCK => grammar::impl_item_list,
-                    TRAIT_DEF => grammar::trait_item_list,
-                    MODULE => grammar::mod_item_list,
-                    _ => return None,
-                }
-            }
-            _ => return None,
-        };
-        Some(res)
-    }
+    node.ancestors().find_map(|node| grammar::reparser(node).map(|r| (node, r)))
 }
 
 fn is_balanced(tokens: &[Token]) -> bool {

crates/ra_syntax/src/parsing/token_set.rs

@@ -4,19 +4,19 @@ use crate::SyntaxKind;
 pub(crate) struct TokenSet(u128);
 
 impl TokenSet {
-    pub const fn empty() -> TokenSet {
+    pub(crate) const fn empty() -> TokenSet {
         TokenSet(0)
     }
 
-    pub const fn singleton(kind: SyntaxKind) -> TokenSet {
+    pub(crate) const fn singleton(kind: SyntaxKind) -> TokenSet {
         TokenSet(mask(kind))
     }
 
-    pub const fn union(self, other: TokenSet) -> TokenSet {
+    pub(crate) const fn union(self, other: TokenSet) -> TokenSet {
         TokenSet(self.0 | other.0)
     }
 
-    pub fn contains(&self, kind: SyntaxKind) -> bool {
+    pub(crate) fn contains(&self, kind: SyntaxKind) -> bool {
         self.0 & mask(kind) != 0
     }
 }

crates/ra_syntax/src/syntax_kinds.rs

@@ -1,8 +1,9 @@
 mod generated;
 
-use crate::SyntaxKind::*;
 use std::fmt;
 
+use crate::SyntaxKind::*;
+
 pub use self::generated::SyntaxKind;
 
 impl fmt::Debug for SyntaxKind {

crates/ra_syntax/src/syntax_node.rs

@@ -1,15 +1,12 @@
-mod builder;
-pub mod syntax_error;
-mod syntax_text;
-
 use std::{fmt, borrow::Borrow};
 
-use self::syntax_text::SyntaxText;
-use crate::{SmolStr, SyntaxKind, TextRange};
 use rowan::{Types, TransparentNewType};
 
-pub(crate) use self::builder::GreenBuilder;
-pub use self::syntax_error::{SyntaxError, SyntaxErrorKind, Location};
+use crate::{
+    SmolStr, SyntaxKind, TextRange, SyntaxText,
+    syntax_error::SyntaxError,
+};
+
 pub use rowan::WalkEvent;
 
 #[derive(Debug, Clone, Copy)]

crates/ra_syntax/src/validation.rs

@@ -5,7 +5,7 @@ mod string;
 mod block;
 
 use crate::{
-    SourceFile, syntax_node::SyntaxError, AstNode,
+    SourceFile, SyntaxError, AstNode,
     ast,
     algo::visit::{visitor_ctx, VisitorCtx},
 };

crates/ra_syntax/src/validation/block.rs

@@ -1,9 +1,7 @@
 use crate::{SyntaxKind::*,
     ast::{self, AttrsOwner, AstNode},
-    syntax_node::{
-        SyntaxError,
-        SyntaxErrorKind::*,
-    },
+    SyntaxError,
+    SyntaxErrorKind::*,
 };
 
 pub(crate) fn validate_block_node(node: &ast::Block, errors: &mut Vec<SyntaxError>) {

crates/ra_syntax/src/validation/byte.rs

@@ -5,10 +5,8 @@ use crate::{
     string_lexing::{self, StringComponentKind},
     TextRange,
     validation::char,
-    syntax_node::{
-        SyntaxError,
-        SyntaxErrorKind::*,
-    },
+    SyntaxError,
+    SyntaxErrorKind::*,
 };
 
 pub(super) fn validate_byte_node(node: &ast::Byte, errors: &mut Vec<SyntaxError>) {

crates/ra_syntax/src/validation/byte_string.rs

@@ -1,10 +1,8 @@
 use crate::{
     ast::{self, AstNode, AstToken},
     string_lexing::{self, StringComponentKind},
-    syntax_node::{
-        SyntaxError,
-        SyntaxErrorKind::*,
-    },
+    SyntaxError,
+    SyntaxErrorKind::*,
 };
 
 use super::byte;

crates/ra_syntax/src/validation/char.rs

@@ -8,10 +8,8 @@ use crate::{
     ast::{self, AstNode, AstToken},
     string_lexing::{self, StringComponentKind},
     TextRange,
-    syntax_node::{
-        SyntaxError,
-        SyntaxErrorKind::*,
-    },
+    SyntaxError,
+    SyntaxErrorKind::*,
 };
 
 pub(super) fn validate_char_node(node: &ast::Char, errors: &mut Vec<SyntaxError>) {

crates/ra_syntax/src/validation/string.rs

@@ -1,10 +1,8 @@
 use crate::{
     ast::{self, AstNode, AstToken},
     string_lexing,
-    syntax_node::{
-        SyntaxError,
-        SyntaxErrorKind::*,
-    },
+    SyntaxError,
+    SyntaxErrorKind::*,
 };
 
 use super::char;

crates/tools/src/lib.rs

@@ -14,7 +14,7 @@ pub use teraron::{Mode, Overwrite, Verify};
 pub type Result<T> = std::result::Result<T, failure::Error>;
 
 pub const GRAMMAR: &str = "crates/ra_syntax/src/grammar.ron";
-const GRAMMAR_DIR: &str = "crates/ra_syntax/src/grammar";
+const GRAMMAR_DIR: &str = "crates/ra_syntax/src/parsing/grammar";
 const OK_INLINE_TESTS_DIR: &str = "crates/ra_syntax/tests/data/parser/inline/ok";
 const ERR_INLINE_TESTS_DIR: &str = "crates/ra_syntax/tests/data/parser/inline/err";