Merge #616

616: Lex raw idents correctly r=matklad a=jrobsonchase

One question:

What's the intent of the text by the `IDENT` node under `NAME`? Should it be the actual token text or the semantic name?

Closes #611 

Co-authored-by: Josh Robson Chase <josh@robsonchase.com>

crates/ra_syntax/src/lexer.rs

@@ -190,13 +190,12 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
 }
 
 fn scan_ident(c: char, ptr: &mut Ptr) -> SyntaxKind {
-    let is_single_letter = match ptr.current() {
-        None => true,
-        Some(c) if !is_ident_continue(c) => true,
-        _ => false,
-    };
-    if is_single_letter {
-        return if c == '_' { UNDERSCORE } else { IDENT };
+    match (c, ptr.current()) {
+        ('r', Some('#')) => {
+            ptr.bump();
+        }
+        ('_', Some(c)) if !is_ident_continue(c) => return UNDERSCORE,
+        _ => {}
     }
     ptr.bump_while(is_ident_continue);
     if let Some(kind) = SyntaxKind::from_keyword(ptr.current_token_text()) {

crates/ra_syntax/src/lexer/strings.rs

@@ -5,7 +5,8 @@ use crate::lexer::ptr::Ptr;
 pub(crate) fn is_string_literal_start(c: char, c1: Option<char>, c2: Option<char>) -> bool {
     match (c, c1, c2) {
         ('r', Some('"'), _)
-        | ('r', Some('#'), _)
+        | ('r', Some('#'), Some('"'))
+        | ('r', Some('#'), Some('#'))
         | ('b', Some('"'), _)
         | ('b', Some('\''), _)
         | ('b', Some('r'), Some('"'))

crates/ra_syntax/tests/data/lexer/0016_raw_ident.rs

@@ -0,0 +1 @@
+r#raw_ident

crates/ra_syntax/tests/data/lexer/0016_raw_ident.txt

@@ -0,0 +1,2 @@
+IDENT 11 "r#raw_ident"
+WHITESPACE 1 "\n"

crates/ra_syntax/tests/data/parser/ok/0039_raw_fn_item.rs

@@ -0,0 +1,2 @@
+fn r#foo() {
+}

crates/ra_syntax/tests/data/parser/ok/0039_raw_fn_item.txt

@@ -0,0 +1,15 @@
+SOURCE_FILE@[0; 15)
+  FN_DEF@[0; 14)
+    FN_KW@[0; 2)
+    WHITESPACE@[2; 3)
+    NAME@[3; 8)
+      IDENT@[3; 8) "r#foo"
+    PARAM_LIST@[8; 10)
+      L_PAREN@[8; 9)
+      R_PAREN@[9; 10)
+    WHITESPACE@[10; 11)
+    BLOCK@[11; 14)
+      L_CURLY@[11; 12)
+      WHITESPACE@[12; 13)
+      R_CURLY@[13; 14)
+  WHITESPACE@[14; 15)

crates/ra_syntax/tests/data/parser/ok/0040_raw_struct_item_field.rs

@@ -0,0 +1,3 @@
+struct S {
+    r#foo: u32
+}

crates/ra_syntax/tests/data/parser/ok/0040_raw_struct_item_field.txt

@@ -0,0 +1,22 @@
+SOURCE_FILE@[0; 27)
+  STRUCT_DEF@[0; 27)
+    STRUCT_KW@[0; 6)
+    WHITESPACE@[6; 7)
+    NAME@[7; 8)
+      IDENT@[7; 8) "S"
+    WHITESPACE@[8; 9)
+    NAMED_FIELD_DEF_LIST@[9; 27)
+      L_CURLY@[9; 10)
+      WHITESPACE@[10; 15)
+      NAMED_FIELD_DEF@[15; 25)
+        NAME@[15; 20)
+          IDENT@[15; 20) "r#foo"
+        COLON@[20; 21)
+        WHITESPACE@[21; 22)
+        PATH_TYPE@[22; 25)
+          PATH@[22; 25)
+            PATH_SEGMENT@[22; 25)
+              NAME_REF@[22; 25)
+                IDENT@[22; 25) "u32"
+      WHITESPACE@[25; 26)
+      R_CURLY@[26; 27)