Auto merge of #24884 - michaelsproul:extended-errors, r=nrc

I've been working on improving the diagnostic registration system so that it can:

* Check uniqueness of error codes *across the whole compiler*. The current method using `errorck.py` is prone to failure as it relies on simple text search - I found that it breaks when referencing an error's ident within a string (e.g. `"See also E0303"`).
* Provide JSON output of error metadata, to eventually facilitate HTML output, as well as tracking of which errors need descriptions. The current schema is:

```
<error code>: {
    "description": <long description>,
    "use_site": {
        "filename": <filename where error is used>,
        "line": <line in file where error is used>
    }
}
```

[Here's][metadata-dump] a pretty-printed sample dump for `librustc`.

One thing to note is that I had to move the diagnostics arrays out of the diagnostics modules. I really wanted to be able to capture error usage information, which only becomes available as a crate is compiled. Hence all invocations of `__build_diagnostics_array!` have been moved to the ends of their respective `lib.rs` files. I tried to avoid moving the array by making a plugin that expands to nothing but couldn't invoke it in item position and gave up on hackily generating a fake item. I also briefly considered using a lint, but it seemed like it would impossible to get access to the data stored in the thread-local storage.

The next step will be to generate a web page that lists each error with its rendered description and use site. Simple mapping and filtering of the metadata files also allows us to work out which error numbers are absent, which errors are unused and which need descriptions.

[metadata-dump]: https://gist.github.com/michaelsproul/3246846ff1bea71bd049

src/librustc/diagnostics.rs

@@ -550,5 +550,3 @@ register_diagnostics! {
     E0316, // nested quantification of lifetimes
     E0370  // discriminant overflow
 }
-
-__build_diagnostic_array! { DIAGNOSTICS }

src/librustc/lib.rs

@@ -161,3 +161,9 @@ pub mod lib {
 mod rustc {
     pub use lint;
 }
+
+// Build the diagnostics array at the end so that the metadata includes error use sites.
+#[cfg(stage0)]
+__build_diagnostic_array! { DIAGNOSTICS }
+#[cfg(not(stage0))]
+__build_diagnostic_array! { librustc, DIAGNOSTICS }

src/librustc_borrowck/diagnostics.rs

@@ -13,5 +13,3 @@
 register_diagnostics! {
     E0373 // closure may outlive current fn, but it borrows {}, which is owned by current fn
 }
-
-__build_diagnostic_array! { DIAGNOSTICS }

src/librustc_borrowck/lib.rs

@@ -46,3 +46,8 @@ pub mod diagnostics;
 mod borrowck;
 
 pub mod graphviz;
+
+#[cfg(stage0)]
+__build_diagnostic_array! { DIAGNOSTICS }
+#[cfg(not(stage0))]
+__build_diagnostic_array! { librustc_borrowck, DIAGNOSTICS }

src/librustc_driver/lib.rs

@@ -853,9 +853,10 @@ pub fn diagnostics_registry() -> diagnostics::registry::Registry {
     use syntax::diagnostics::registry::Registry;
 
     let all_errors = Vec::new() +
-        &rustc::diagnostics::DIAGNOSTICS[..] +
-        &rustc_typeck::diagnostics::DIAGNOSTICS[..] +
-        &rustc_resolve::diagnostics::DIAGNOSTICS[..];
+        &rustc::DIAGNOSTICS[..] +
+        &rustc_typeck::DIAGNOSTICS[..] +
+        &rustc_borrowck::DIAGNOSTICS[..] +
+        &rustc_resolve::DIAGNOSTICS[..];
 
     Registry::new(&*all_errors)
 }

src/librustc_resolve/diagnostics.rs

@@ -28,5 +28,3 @@ register_diagnostics! {
     E0364, // item is private
     E0365  // item is private
 }
-
-__build_diagnostic_array! { DIAGNOSTICS }

src/librustc_resolve/lib.rs

@@ -3706,3 +3706,8 @@ pub fn resolve_crate<'a, 'tcx>(session: &'a Session,
                     },
     }
 }
+
+#[cfg(stage0)]
+__build_diagnostic_array! { DIAGNOSTICS }
+#[cfg(not(stage0))]
+__build_diagnostic_array! { librustc_resolve, DIAGNOSTICS }

src/librustc_typeck/diagnostics.rs

@@ -188,5 +188,3 @@ register_diagnostics! {
     E0371, // impl Trait for Trait is illegal
     E0372  // impl Trait for Trait where Trait is not object safe
 }
-
-__build_diagnostic_array! { DIAGNOSTICS }

src/librustc_typeck/lib.rs

@@ -343,3 +343,8 @@ pub fn check_crate(tcx: &ty::ctxt, trait_map: ty::TraitMap) {
     check_for_entry_fn(&ccx);
     tcx.sess.abort_if_errors();
 }
+
+#[cfg(stage0)]
+__build_diagnostic_array! { DIAGNOSTICS }
+#[cfg(not(stage0))]
+__build_diagnostic_array! { librustc_typeck, DIAGNOSTICS }

src/libsyntax/diagnostics/metadata.rs

@@ -0,0 +1,155 @@
+// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! This module contains utilities for outputting metadata for diagnostic errors.
+//!
+//! Each set of errors is mapped to a metadata file by a name, which is
+//! currently always a crate name.
+
+use std::collections::BTreeMap;
+use std::env;
+use std::path::PathBuf;
+use std::fs::{read_dir, create_dir_all, OpenOptions, File};
+use std::io::{Read, Write};
+use std::error::Error;
+use rustc_serialize::json::{self, as_json};
+
+use codemap::Span;
+use ext::base::ExtCtxt;
+use diagnostics::plugin::{ErrorMap, ErrorInfo};
+
+pub use self::Uniqueness::*;
+
+// Default metadata directory to use for extended error JSON.
+const ERROR_METADATA_DIR_DEFAULT: &'static str = "tmp/extended-errors";
+
+// The name of the environment variable that sets the metadata dir.
+const ERROR_METADATA_VAR: &'static str = "ERROR_METADATA_DIR";
+
+/// JSON encodable/decodable version of `ErrorInfo`.
+#[derive(PartialEq, RustcDecodable, RustcEncodable)]
+pub struct ErrorMetadata {
+    pub description: Option<String>,
+    pub use_site: Option<ErrorLocation>
+}
+
+/// Mapping from error codes to metadata that can be (de)serialized.
+pub type ErrorMetadataMap = BTreeMap<String, ErrorMetadata>;
+
+/// JSON encodable error location type with filename and line number.
+#[derive(PartialEq, RustcDecodable, RustcEncodable)]
+pub struct ErrorLocation {
+    pub filename: String,
+    pub line: usize
+}
+
+impl ErrorLocation {
+    /// Create an error location from a span.
+    pub fn from_span(ecx: &ExtCtxt, sp: Span) -> ErrorLocation {
+        let loc = ecx.codemap().lookup_char_pos_adj(sp.lo);
+        ErrorLocation {
+            filename: loc.filename,
+            line: loc.line
+        }
+    }
+}
+
+/// Type for describing the uniqueness of a set of error codes, as returned by `check_uniqueness`.
+pub enum Uniqueness {
+    /// All errors in the set checked are unique according to the metadata files checked.
+    Unique,
+    /// One or more errors in the set occur in another metadata file.
+    /// This variant contains the first duplicate error code followed by the name
+    /// of the metadata file where the duplicate appears.
+    Duplicate(String, String)
+}
+
+/// Get the directory where metadata files should be stored.
+pub fn get_metadata_dir() -> PathBuf {
+    match env::var(ERROR_METADATA_VAR) {
+        Ok(v) => From::from(v),
+        Err(_) => From::from(ERROR_METADATA_DIR_DEFAULT)
+    }
+}
+
+/// Get the path where error metadata for the set named by `name` should be stored.
+fn get_metadata_path(name: &str) -> PathBuf {
+    get_metadata_dir().join(format!("{}.json", name))
+}
+
+/// Check that the errors in `err_map` aren't present in any metadata files in the
+/// metadata directory except the metadata file corresponding to `name`.
+pub fn check_uniqueness(name: &str, err_map: &ErrorMap) -> Result<Uniqueness, Box<Error>> {
+    let metadata_dir = get_metadata_dir();
+    let metadata_path = get_metadata_path(name);
+
+    // Create the error directory if it does not exist.
+    try!(create_dir_all(&metadata_dir));
+
+    // Check each file in the metadata directory.
+    for entry in try!(read_dir(&metadata_dir)) {
+        let path = try!(entry).path();
+
+        // Skip any existing file for this set.
+        if path == metadata_path {
+            continue;
+        }
+
+        // Read the metadata file into a string.
+        let mut metadata_str = String::new();
+        try!(
+            File::open(&path).and_then(|mut f|
+            f.read_to_string(&mut metadata_str))
+        );
+
+        // Parse the JSON contents.
+        let metadata: ErrorMetadataMap = try!(json::decode(&metadata_str));
+
+        // Check for duplicates.
+        for err in err_map.keys() {
+            let err_code = err.as_str();
+            if metadata.contains_key(err_code) {
+                return Ok(Duplicate(
+                    err_code.to_string(),
+                    path.to_string_lossy().into_owned()
+                ));
+            }
+        }
+    }
+
+    Ok(Unique)
+}
+
+/// Write metadata for the errors in `err_map` to disk, to a file corresponding to `name`.
+pub fn output_metadata(ecx: &ExtCtxt, name: &str, err_map: &ErrorMap)
+    -> Result<(), Box<Error>>
+{
+    let metadata_path = get_metadata_path(name);
+
+    // Open the dump file.
+    let mut dump_file = try!(OpenOptions::new()
+        .write(true)
+        .create(true)
+        .open(&metadata_path)
+    );
+
+    // Construct a serializable map.
+    let json_map = err_map.iter().map(|(k, &ErrorInfo { description, use_site })| {
+        let key = k.as_str().to_string();
+        let value = ErrorMetadata {
+            description: description.map(|n| n.as_str().to_string()),
+            use_site: use_site.map(|sp| ErrorLocation::from_span(ecx, sp))
+        };
+        (key, value)
+    }).collect::<ErrorMetadataMap>();
+
+    try!(write!(&mut dump_file, "{}", as_json(&json_map)));
+    Ok(())
+}

src/libsyntax/diagnostics/plugin.rs

@@ -14,6 +14,7 @@ use std::collections::BTreeMap;
 use ast;
 use ast::{Ident, Name, TokenTree};
 use codemap::Span;
+use diagnostics::metadata::{check_uniqueness, output_metadata, Duplicate};
 use ext::base::{ExtCtxt, MacEager, MacResult};
 use ext::build::AstBuilder;
 use parse::token;
@@ -24,32 +25,28 @@ use util::small_vector::SmallVector;
 const MAX_DESCRIPTION_WIDTH: usize = 80;
 
 thread_local! {
-    static REGISTERED_DIAGNOSTICS: RefCell<BTreeMap<Name, Option<Name>>> = {
+    static REGISTERED_DIAGNOSTICS: RefCell<ErrorMap> = {
         RefCell::new(BTreeMap::new())
     }
 }
-thread_local! {
-    static USED_DIAGNOSTICS: RefCell<BTreeMap<Name, Span>> = {
-        RefCell::new(BTreeMap::new())
-    }
+
+/// Error information type.
+pub struct ErrorInfo {
+    pub description: Option<Name>,
+    pub use_site: Option<Span>
 }
 
+/// Mapping from error codes to metadata.
+pub type ErrorMap = BTreeMap<Name, ErrorInfo>;
+
 fn with_registered_diagnostics<T, F>(f: F) -> T where
-    F: FnOnce(&mut BTreeMap<Name, Option<Name>>) -> T,
+    F: FnOnce(&mut ErrorMap) -> T,
 {
     REGISTERED_DIAGNOSTICS.with(move |slot| {
         f(&mut *slot.borrow_mut())
     })
 }
 
-fn with_used_diagnostics<T, F>(f: F) -> T where
-    F: FnOnce(&mut BTreeMap<Name, Span>) -> T,
-{
-    USED_DIAGNOSTICS.with(move |slot| {
-        f(&mut *slot.borrow_mut())
-    })
-}
-
 pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt,
                                    span: Span,
                                    token_tree: &[TokenTree])
@@ -58,23 +55,26 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt,
         (1, Some(&ast::TtToken(_, token::Ident(code, _)))) => code,
         _ => unreachable!()
     };
-    with_used_diagnostics(|diagnostics| {
-        match diagnostics.insert(code.name, span) {
-            Some(previous_span) => {
+
+    with_registered_diagnostics(|diagnostics| {
+        match diagnostics.get_mut(&code.name) {
+            // Previously used errors.
+            Some(&mut ErrorInfo { description: _, use_site: Some(previous_span) }) => {
                 ecx.span_warn(span, &format!(
                     "diagnostic code {} already used", &token::get_ident(code)
                 ));
                 ecx.span_note(previous_span, "previous invocation");
-            },
-            None => ()
-        }
-        ()
-    });
-    with_registered_diagnostics(|diagnostics| {
-        if !diagnostics.contains_key(&code.name) {
-            ecx.span_err(span, &format!(
-                "used diagnostic code {} not registered", &token::get_ident(code)
-            ));
+            }
+            // Newly used errors.
+            Some(ref mut info) => {
+                info.use_site = Some(span);
+            }
+            // Unregistered errors.
+            None => {
+                ecx.span_err(span, &format!(
+                    "used diagnostic code {} not registered", &token::get_ident(code)
+                ));
+            }
         }
     });
     MacEager::expr(ecx.expr_tuple(span, Vec::new()))
@@ -116,10 +116,14 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt,
                 token::get_ident(*code), MAX_DESCRIPTION_WIDTH
             ));
         }
-        raw_msg
     });
+    // Add the error to the map.
     with_registered_diagnostics(|diagnostics| {
-        if diagnostics.insert(code.name, description).is_some() {
+        let info = ErrorInfo {
+            description: description,
+            use_site: None
+        };
+        if diagnostics.insert(code.name, info).is_some() {
             ecx.span_err(span, &format!(
                 "diagnostic code {} already registered", &token::get_ident(*code)
             ));
@@ -143,19 +147,43 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt,
                                           span: Span,
                                           token_tree: &[TokenTree])
                                           -> Box<MacResult+'cx> {
-    let name = match (token_tree.len(), token_tree.get(0)) {
-        (1, Some(&ast::TtToken(_, token::Ident(ref name, _)))) => name,
+    assert_eq!(token_tree.len(), 3);
+    let (crate_name, name) = match (&token_tree[0], &token_tree[2]) {
+        (
+            // Crate name.
+            &ast::TtToken(_, token::Ident(ref crate_name, _)),
+            // DIAGNOSTICS ident.
+            &ast::TtToken(_, token::Ident(ref name, _))
+        ) => (crate_name.as_str(), name),
         _ => unreachable!()
     };
 
+    // Check uniqueness of errors and output metadata.
+    with_registered_diagnostics(|diagnostics| {
+        match check_uniqueness(crate_name, &*diagnostics) {
+            Ok(Duplicate(err, location)) => {
+                ecx.span_err(span, &format!(
+                    "error {} from `{}' also found in `{}'",
+                    err, crate_name, location
+                ));
+            },
+            Ok(_) => (),
+            Err(e) => panic!("{}", e.description())
+        }
+
+        output_metadata(&*ecx, crate_name, &*diagnostics).ok().expect("metadata output error");
+    });
+
+    // Construct the output expression.
     let (count, expr) =
         with_registered_diagnostics(|diagnostics| {
             let descriptions: Vec<P<ast::Expr>> =
-                diagnostics.iter().filter_map(|(code, description)| {
-                    description.map(|description| {
+                diagnostics.iter().filter_map(|(code, info)| {
+                    info.description.map(|description| {
                         ecx.expr_tuple(span, vec![
                             ecx.expr_str(span, token::get_name(*code)),
-                            ecx.expr_str(span, token::get_name(description))])
+                            ecx.expr_str(span, token::get_name(description))
+                        ])
                     })
                 }).collect();
             (descriptions.len(), ecx.expr_vec(span, descriptions))

src/libsyntax/lib.rs

@@ -70,6 +70,7 @@ pub mod diagnostics {
     pub mod macros;
     pub mod plugin;
     pub mod registry;
+    pub mod metadata;
 }
 
 pub mod syntax {

src/test/run-make/issue-19371/foo.rs

@@ -49,7 +49,7 @@ fn basic_sess(sysroot: PathBuf) -> Session {
     opts.output_types = vec![OutputTypeExe];
     opts.maybe_sysroot = Some(sysroot);
 
-    let descriptions = Registry::new(&rustc::diagnostics::DIAGNOSTICS);
+    let descriptions = Registry::new(&rustc::DIAGNOSTICS);
     let sess = build_session(opts, None, descriptions);
     rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
     sess