Add support for marking doctest items as distinct from normal code, add default tag to all doctest elements

crates/ra_ide/src/lib.rs

@@ -443,13 +443,13 @@ impl Analysis {
 
     /// Computes syntax highlighting for the given file
     pub fn highlight(&self, file_id: FileId) -> Cancelable<Vec<HighlightedRange>> {
-        self.with_db(|db| syntax_highlighting::highlight(db, file_id, None, false))
+        self.with_db(|db| syntax_highlighting::highlight(db, file_id, None, false, None))
     }
 
     /// Computes syntax highlighting for the given file range.
     pub fn highlight_range(&self, frange: FileRange) -> Cancelable<Vec<HighlightedRange>> {
         self.with_db(|db| {
-            syntax_highlighting::highlight(db, frange.file_id, Some(frange.range), false)
+            syntax_highlighting::highlight(db, frange.file_id, Some(frange.range), false, None)
         })
     }
 

crates/ra_ide/src/prime_caches.rs

@@ -7,6 +7,6 @@ use crate::{FileId, RootDatabase};
 
 pub(crate) fn prime_caches(db: &RootDatabase, files: Vec<FileId>) {
     for file in files {
-        let _ = crate::syntax_highlighting::highlight(db, file, None, false);
+        let _ = crate::syntax_highlighting::highlight(db, file, None, false, None);
     }
 }

crates/ra_ide/src/snapshots/highlight_doctest.html

@@ -47,9 +47,9 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
     <span class="comment documentation">/// # Examples</span>
     <span class="comment documentation">///</span>
     <span class="comment documentation">/// ```</span>
-    <span class="comment documentation">/// #</span> <span class="attribute">#![</span><span class="function attribute">allow</span><span class="attribute">(unused_mut)]</span>
-    <span class="comment documentation">/// </span><span class="keyword">let</span> <span class="keyword">mut</span> <span class="variable declaration mutable">foo</span>: <span class="struct">Foo</span> = <span class="struct">Foo</span>::<span class="function">new</span>();
-    <span class="comment documentation">/// ```</span>
+    <span class="comment documentation">/// #</span> <span class="operator injected">#</span><span class="operator injected">!</span><span class="operator injected">[</span><span class="operator injected">allow</span><span class="operator injected">(</span><span class="operator injected">unused_mut</span><span class="operator injected">)</span><span class="operator injected">]</span><span class="operator injected">
+</span>    <span class="comment documentation">/// </span><span class="keyword injected">let</span><span class="operator injected"> </span><span class="keyword injected">mut</span><span class="operator injected"> </span><span class="operator injected">foo</span><span class="operator injected">:</span><span class="operator injected"> </span><span class="operator injected">Foo</span><span class="operator injected"> </span><span class="operator injected">=</span><span class="operator injected"> </span><span class="operator injected">Foo</span><span class="operator injected">::</span><span class="operator injected">new</span><span class="operator injected">(</span><span class="operator injected">)</span><span class="operator injected">;</span><span class="operator injected">
+</span><span class="operator injected"> </span>   <span class="comment documentation">/// ```</span>
     <span class="keyword">pub</span> <span class="keyword">const</span> <span class="keyword">fn</span> <span class="function declaration">new</span>() -&gt; <span class="struct">Foo</span> {
         <span class="struct">Foo</span> { <span class="field">bar</span>: <span class="bool_literal">true</span> }
     }
@@ -59,27 +59,27 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
     <span class="comment documentation">/// # Examples</span>
     <span class="comment documentation">///</span>
     <span class="comment documentation">/// ```</span>
-    <span class="comment documentation">/// </span><span class="keyword">use</span> <span class="module">x</span>::<span class="module">y</span>;
-    <span class="comment documentation">///</span>
-    <span class="comment documentation">/// </span><span class="keyword">let</span> <span class="variable declaration">foo</span> = <span class="struct">Foo</span>::<span class="function">new</span>();
-    <span class="comment documentation">///</span>
-    <span class="comment documentation">/// </span><span class="comment">// calls bar on foo</span>
-    <span class="comment documentation">/// </span><span class="macro">assert!</span>(foo.bar());
-    <span class="comment documentation">///</span>
-    <span class="comment documentation">/// </span><span class="keyword">let</span> <span class="variable declaration">bar</span> = <span class="variable">foo</span>.<span class="field">bar</span> || <span class="struct">Foo</span>::<span class="constant">bar</span>;
-    <span class="comment documentation">///</span>
-    <span class="comment documentation">/// </span><span class="comment">/* multi-line
-    </span><span class="comment documentation">/// </span><span class="comment">       comment */</span>
-    <span class="comment documentation">///</span>
-    <span class="comment documentation">/// </span><span class="keyword">let</span> <span class="variable declaration">multi_line_string</span> = <span class="string_literal">"Foo
-    </span><span class="comment documentation">/// </span><span class="string_literal">  bar
-    </span><span class="comment documentation">/// </span><span class="string_literal">         "</span>;
-    <span class="comment documentation">///</span>
-    <span class="comment documentation">/// ```</span>
+    <span class="comment documentation">/// </span><span class="keyword injected">use</span><span class="operator injected"> </span><span class="operator injected">x</span><span class="operator injected">::</span><span class="operator injected">y</span><span class="operator injected">;</span><span class="operator injected">
+    </span><span class="comment documentation">///</span><span class="operator injected">
+</span>    <span class="comment documentation">/// </span><span class="keyword injected">let</span><span class="operator injected"> </span><span class="operator injected">foo</span><span class="operator injected"> </span><span class="operator injected">=</span><span class="operator injected"> </span><span class="operator injected">Foo</span><span class="operator injected">::</span><span class="operator injected">new</span><span class="operator injected">(</span><span class="operator injected">)</span><span class="operator injected">;</span><span class="operator injected">
+    </span><span class="comment documentation">///</span><span class="operator injected">
+</span>    <span class="comment documentation">/// </span><span class="comment injected">// calls bar on foo</span><span class="operator injected">
+</span>    <span class="comment documentation">/// </span><span class="macro injected">assert!</span><span class="operator injected">(</span><span class="operator injected">foo</span><span class="operator injected">.</span><span class="operator injected">bar</span><span class="operator injected">(</span><span class="operator injected">)</span><span class="operator injected">)</span><span class="operator injected">;</span><span class="operator injected">
+    </span><span class="comment documentation">///</span><span class="operator injected">
+</span>    <span class="comment documentation">/// </span><span class="keyword injected">let</span><span class="operator injected"> </span><span class="operator injected">bar</span><span class="operator injected"> </span><span class="operator injected">=</span><span class="operator injected"> </span><span class="operator injected">foo</span><span class="operator injected">.</span><span class="operator injected">bar</span><span class="operator injected"> </span><span class="operator injected">||</span><span class="operator injected"> </span><span class="operator injected">Foo</span><span class="operator injected">::</span><span class="operator injected">bar</span><span class="operator injected">;</span><span class="operator injected">
+    </span><span class="comment documentation">///</span><span class="operator injected">
+</span>    <span class="comment documentation">/// </span><span class="comment injected">/* multi-line
+    </span><span class="comment documentation">/// </span><span class="comment injected">       comment */</span><span class="operator injected">
+    </span><span class="comment documentation">///</span><span class="operator injected">
+</span>    <span class="comment documentation">/// </span><span class="keyword injected">let</span><span class="operator injected"> </span><span class="operator injected">multi_line_string</span><span class="operator injected"> </span><span class="operator injected">=</span><span class="operator injected"> </span><span class="string_literal injected">"Foo
+    </span><span class="comment documentation">/// </span><span class="string_literal injected">  bar
+    </span><span class="comment documentation">/// </span><span class="string_literal injected">         "</span><span class="operator injected">;</span><span class="operator injected">
+    </span><span class="comment documentation">///</span><span class="operator injected">
+</span>    <span class="comment documentation">/// ```</span>
     <span class="comment documentation">///</span>
     <span class="comment documentation">/// ```rust,no_run</span>
-    <span class="comment documentation">/// </span><span class="keyword">let</span> <span class="variable declaration">foobar</span> = <span class="struct">Foo</span>::<span class="function">new</span>().<span class="function">bar</span>();
-    <span class="comment documentation">/// ```</span>
+    <span class="comment documentation">/// </span><span class="keyword injected">let</span><span class="operator injected"> </span><span class="operator injected">foobar</span><span class="operator injected"> </span><span class="operator injected">=</span><span class="operator injected"> </span><span class="operator injected">Foo</span><span class="operator injected">::</span><span class="operator injected">new</span><span class="operator injected">(</span><span class="operator injected">)</span><span class="operator injected">.</span><span class="operator injected">bar</span><span class="operator injected">(</span><span class="operator injected">)</span><span class="operator injected">;</span><span class="operator injected">
+</span><span class="operator injected"> </span>   <span class="comment documentation">/// ```</span>
     <span class="comment documentation">///</span>
     <span class="comment documentation">/// ```sh</span>
     <span class="comment documentation">/// echo 1</span>

crates/ra_ide/src/syntax_highlighting.rs

@@ -45,6 +45,7 @@ pub(crate) fn highlight(
     file_id: FileId,
     range_to_highlight: Option<TextRange>,
     syntactic_name_ref_highlighting: bool,
+    default_tag: Option<HighlightTag>,
 ) -> Vec<HighlightedRange> {
     let _p = profile("highlight");
     let sema = Semantics::new(db);
@@ -107,6 +108,7 @@ pub(crate) fn highlight(
                         &mut bindings_shadow_count,
                         syntactic_name_ref_highlighting,
                         name.syntax().clone().into(),
+                        default_tag,
                     ) {
                         stack.add(HighlightedRange {
                             range: name.syntax().text_range(),
@@ -206,6 +208,7 @@ pub(crate) fn highlight(
             &mut bindings_shadow_count,
             syntactic_name_ref_highlighting,
             element_to_highlight.clone(),
+            default_tag,
         ) {
             stack.add(HighlightedRange { range, highlight, binding_hash });
             if let Some(string) =
@@ -430,13 +433,14 @@ fn highlight_element(
     bindings_shadow_count: &mut FxHashMap<Name, u32>,
     syntactic_name_ref_highlighting: bool,
     element: SyntaxElement,
+    default_tag: Option<HighlightTag>,
 ) -> Option<(Highlight, Option<u64>)> {
     let db = sema.db;
     let mut binding_hash = None;
     let highlight: Highlight = match element.kind() {
         FN_DEF => {
             bindings_shadow_count.clear();
-            return None;
+            default_tag?.into()
         }
 
         // Highlight definitions depending on the "type" of the definition.
@@ -515,12 +519,12 @@ fn highlight_element(
             let expr = prefix_expr.expr()?;
             let ty = sema.type_of_expr(&expr)?;
             if !ty.is_raw_ptr() {
-                return None;
+                default_tag?.into()
+            } else {
+                let mut h = Highlight::new(HighlightTag::Operator);
+                h |= HighlightModifier::Unsafe;
+                h
             }
-
-            let mut h = Highlight::new(HighlightTag::Operator);
-            h |= HighlightModifier::Unsafe;
-            h
         }
         T![!] if element.parent().and_then(ast::MacroCall::cast).is_some() => {
             Highlight::new(HighlightTag::Macro)
@@ -546,7 +550,7 @@ fn highlight_element(
             }
         }
 
-        _ => return None,
+        _ => default_tag?.into(),
     };
 
     return Some((highlight, binding_hash));

crates/ra_ide/src/syntax_highlighting/html.rs

@@ -19,7 +19,7 @@ pub(crate) fn highlight_as_html(db: &RootDatabase, file_id: FileId, rainbow: boo
         )
     }
 
-    let ranges = highlight(db, file_id, None, false);
+    let ranges = highlight(db, file_id, None, false, None);
     let text = parse.tree().syntax().to_string();
     let mut prev_pos = TextSize::from(0);
     let mut buf = String::new();

crates/ra_ide/src/syntax_highlighting/injection.rs

@@ -150,7 +150,10 @@ pub(super) fn highlight_doc_comment(
     let (analysis, tmp_file_id) = Analysis::from_single_file(text);
 
     stack.push();
-    for mut h in analysis.with_db(|db| super::highlight(db, tmp_file_id, None, true)).unwrap() {
+    for mut h in analysis
+        .with_db(|db| super::highlight(db, tmp_file_id, None, true, Some(HighlightTag::Operator)))
+        .unwrap()
+    {
         // Determine start offset and end offset in case of multi-line ranges
         let mut start_offset = None;
         let mut end_offset = None;
@@ -172,6 +175,7 @@ pub(super) fn highlight_doc_comment(
                 h.range.end() + end_offset.unwrap_or(start_offset) - h.range.start(),
             );
 
+            h.highlight |= HighlightModifier::Injected;
             stack.add(h);
         }
     }

crates/ra_ide/src/syntax_highlighting/tags.rs

@@ -57,6 +57,7 @@ pub enum HighlightModifier {
     /// not.
     Definition,
     Documentation,
+    Injected,
     Mutable,
     Unsafe,
 }
@@ -110,6 +111,7 @@ impl HighlightModifier {
         HighlightModifier::ControlFlow,
         HighlightModifier::Definition,
         HighlightModifier::Documentation,
+        HighlightModifier::Injected,
         HighlightModifier::Mutable,
         HighlightModifier::Unsafe,
     ];
@@ -120,6 +122,7 @@ impl HighlightModifier {
             HighlightModifier::ControlFlow => "control",
             HighlightModifier::Definition => "declaration",
             HighlightModifier::Documentation => "documentation",
+            HighlightModifier::Injected => "injected",
             HighlightModifier::Mutable => "mutable",
             HighlightModifier::Unsafe => "unsafe",
         }

crates/rust-analyzer/src/semantic_tokens.rs

@@ -68,6 +68,7 @@ macro_rules! define_semantic_token_modifiers {
 define_semantic_token_modifiers![
     (CONSTANT, "constant"),
     (CONTROL_FLOW, "controlFlow"),
+    (INJECTED, "injected"),
     (MUTABLE, "mutable"),
     (UNSAFE, "unsafe"),
     (ATTRIBUTE_MODIFIER, "attribute"),

crates/rust-analyzer/src/to_proto.rs

@@ -331,6 +331,7 @@ fn semantic_token_type_and_modifiers(
             HighlightModifier::Attribute => semantic_tokens::ATTRIBUTE_MODIFIER,
             HighlightModifier::Definition => lsp_types::SemanticTokenModifier::DECLARATION,
             HighlightModifier::Documentation => lsp_types::SemanticTokenModifier::DOCUMENTATION,
+            HighlightModifier::Injected => semantic_tokens::INJECTED,
             HighlightModifier::ControlFlow => semantic_tokens::CONTROL_FLOW,
             HighlightModifier::Mutable => semantic_tokens::MUTABLE,
             HighlightModifier::Unsafe => semantic_tokens::UNSAFE,