Rollup merge of #63721 - Mark-Simulacrum:decouple-error-index, r=matthewjasper

Do not emit JSON dumps of diagnostic codes

This decouples the error index generator from libsyntax for the most part (though it still depends on librustdoc for the markdown parsing and generation).

Fixes #34588

Cargo.lock

@@ -946,6 +946,7 @@ name = "error_index_generator"
 version = "0.0.0"
 dependencies = [
  "rustdoc",
+ "walkdir",
 ]
 
 [[package]]

src/bootstrap/doc.rs

@@ -825,8 +825,7 @@ impl Step for ErrorIndex {
         index.arg(crate::channel::CFG_RELEASE_NUM);
 
         // FIXME: shouldn't have to pass this env var
-        index.env("CFG_BUILD", &builder.config.build)
-             .env("RUSTC_ERROR_METADATA_DST", builder.extended_error_dir());
+        index.env("CFG_BUILD", &builder.config.build);
 
         builder.run(&mut index);
     }

src/bootstrap/test.rs

@@ -1535,8 +1535,7 @@ impl Step for ErrorIndex {
         );
         tool.arg("markdown")
             .arg(&output)
-            .env("CFG_BUILD", &builder.config.build)
-            .env("RUSTC_ERROR_METADATA_DST", builder.extended_error_dir());
+            .env("CFG_BUILD", &builder.config.build);
 
         builder.info(&format!("Testing error-index stage{}", compiler.stage));
         let _time = util::timeit(&builder);

src/libsyntax/diagnostics/metadata.rs

@@ -1,93 +0,0 @@
-//! This module contains utilities for outputting metadata for diagnostic errors.
-//!
-//! Each set of errors is mapped to a metadata file by a name, which is
-//! currently always a crate name.
-
-use std::collections::BTreeMap;
-use std::env;
-use std::fs::{remove_file, create_dir_all, File};
-use std::io::Write;
-use std::path::PathBuf;
-use std::error::Error;
-use rustc_serialize::json::as_json;
-
-use syntax_pos::{Span, FileName};
-
-use crate::ext::base::ExtCtxt;
-use crate::diagnostics::plugin::{ErrorMap, ErrorInfo};
-
-/// JSON encodable/decodable version of `ErrorInfo`.
-#[derive(PartialEq, RustcDecodable, RustcEncodable)]
-pub struct ErrorMetadata {
-    pub description: Option<String>,
-    pub use_site: Option<ErrorLocation>
-}
-
-/// Mapping from error codes to metadata that can be (de)serialized.
-pub type ErrorMetadataMap = BTreeMap<String, ErrorMetadata>;
-
-/// JSON encodable error location type with filename and line number.
-#[derive(PartialEq, RustcDecodable, RustcEncodable)]
-pub struct ErrorLocation {
-    pub filename: FileName,
-    pub line: usize
-}
-
-impl ErrorLocation {
-    /// Creates an error location from a span.
-    pub fn from_span(ecx: &ExtCtxt<'_>, sp: Span) -> ErrorLocation {
-        let loc = ecx.source_map().lookup_char_pos(sp.lo());
-        ErrorLocation {
-            filename: loc.file.name.clone(),
-            line: loc.line
-        }
-    }
-}
-
-/// Gets the directory where metadata for a given `prefix` should be stored.
-///
-/// See `output_metadata`.
-pub fn get_metadata_dir(prefix: &str) -> PathBuf {
-    env::var_os("RUSTC_ERROR_METADATA_DST")
-        .map(PathBuf::from)
-        .expect("env var `RUSTC_ERROR_METADATA_DST` isn't set")
-        .join(prefix)
-}
-
-/// Map `name` to a path in the given directory: <directory>/<name>.json
-fn get_metadata_path(directory: PathBuf, name: &str) -> PathBuf {
-    directory.join(format!("{}.json", name))
-}
-
-/// Write metadata for the errors in `err_map` to disk, to a file corresponding to `prefix/name`.
-///
-/// For our current purposes the prefix is the target architecture and the name is a crate name.
-/// If an error occurs steps will be taken to ensure that no file is created.
-pub fn output_metadata(ecx: &ExtCtxt<'_>, prefix: &str, name: &str, err_map: &ErrorMap)
-    -> Result<(), Box<dyn Error>>
-{
-    // Create the directory to place the file in.
-    let metadata_dir = get_metadata_dir(prefix);
-    create_dir_all(&metadata_dir)?;
-
-    // Open the metadata file.
-    let metadata_path = get_metadata_path(metadata_dir, name);
-    let mut metadata_file = File::create(&metadata_path)?;
-
-    // Construct a serializable map.
-    let json_map = err_map.iter().map(|(k, &ErrorInfo { description, use_site })| {
-        let key = k.as_str().to_string();
-        let value = ErrorMetadata {
-            description: description.map(|n| n.as_str().to_string()),
-            use_site: use_site.map(|sp| ErrorLocation::from_span(ecx, sp))
-        };
-        (key, value)
-    }).collect::<ErrorMetadataMap>();
-
-    // Write the data to the file, deleting it if the write fails.
-    let result = write!(&mut metadata_file, "{}", as_json(&json_map));
-    if result.is_err() {
-        remove_file(&metadata_path)?;
-    }
-    Ok(result?)
-}

src/libsyntax/diagnostics/plugin.rs

@@ -1,5 +1,4 @@
 use std::collections::BTreeMap;
-use std::env;
 
 use crate::ast::{self, Ident, Name};
 use crate::source_map;
@@ -12,8 +11,6 @@ use crate::tokenstream::{TokenTree};
 use smallvec::smallvec;
 use syntax_pos::Span;
 
-use crate::diagnostics::metadata::output_metadata;
-
 pub use errors::*;
 
 // Maximum width of any line in an extended error description (inclusive).
@@ -127,36 +124,13 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>,
                                           token_tree: &[TokenTree])
                                           -> Box<dyn MacResult+'cx> {
     assert_eq!(token_tree.len(), 3);
-    let (crate_name, ident) = match (&token_tree[0], &token_tree[2]) {
-        (
-            // Crate name.
-            &TokenTree::Token(Token { kind: token::Ident(crate_name, _), .. }),
-            // DIAGNOSTICS ident.
-            &TokenTree::Token(Token { kind: token::Ident(name, _), span })
-        ) => (crate_name, Ident::new(name, span)),
+    let ident = match &token_tree[2] {
+        // DIAGNOSTICS ident.
+        &TokenTree::Token(Token { kind: token::Ident(name, _), span })
+        => Ident::new(name, span),
         _ => unreachable!()
     };
 
-    // Output error metadata to `tmp/extended-errors/<target arch>/<crate name>.json`
-    if let Ok(target_triple) = env::var("CFG_COMPILER_HOST_TRIPLE") {
-        ecx.parse_sess.registered_diagnostics.with_lock(|diagnostics| {
-            if let Err(e) = output_metadata(ecx,
-                                            &target_triple,
-                                            &crate_name.as_str(),
-                                            diagnostics) {
-                ecx.span_bug(span, &format!(
-                    "error writing metadata for triple `{}` and crate `{}`, error: {}, \
-                     cause: {:?}",
-                    target_triple, crate_name, e.description(), e.source()
-                ));
-            }
-        });
-    } else {
-        ecx.span_err(span, &format!(
-            "failed to write metadata for crate `{}` because $CFG_COMPILER_HOST_TRIPLE is not set",
-            crate_name));
-    }
-
     // Construct the output expression.
     let (count, expr) =
         ecx.parse_sess.registered_diagnostics.with_lock(|diagnostics| {

src/libsyntax/lib.rs

@@ -124,7 +124,6 @@ pub mod diagnostics {
     #[macro_use]
     pub mod macros;
     pub mod plugin;
-    pub mod metadata;
 }
 
 // N.B., this module needs to be declared first so diagnostics are

src/tools/error_index_generator/Cargo.toml

@@ -3,10 +3,14 @@ authors = ["The Rust Project Developers"]
 name = "error_index_generator"
 version = "0.0.0"
 edition = "2018"
+build = "build.rs"
 
 [dependencies]
 rustdoc = { path = "../../librustdoc" }
 
+[build-dependencies]
+walkdir = "2"
+
 [[bin]]
 name = "error_index_generator"
 path = "main.rs"

src/tools/error_index_generator/build.rs

@@ -0,0 +1,64 @@
+use walkdir::WalkDir;
+use std::path::PathBuf;
+use std::{env, fs};
+
+fn main() {
+    // The src directory (we are in src/tools/error_index_generator)
+    // Note that we could skip one of the .. but this ensures we at least loosely find the right
+    // directory.
+    let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+    let dest = out_dir.join("error_codes.rs");
+    let mut idx = 0;
+    for entry in WalkDir::new("../../../src") {
+        let entry = entry.unwrap();
+        if entry.file_name() == "error_codes.rs" {
+            println!("cargo:rerun-if-changed={}", entry.path().to_str().unwrap());
+            let file = fs::read_to_string(entry.path()).unwrap()
+                .replace("use syntax::{register_diagnostics, register_long_diagnostics};", "")
+                .replace("use syntax::register_diagnostics;", "")
+                .replace("use syntax::register_long_diagnostics;", "");
+            let contents = format!("(|| {{\n{}\n}})();", file);
+
+            fs::write(&out_dir.join(&format!("error_{}.rs", idx)), &contents).unwrap();
+
+            idx += 1;
+        }
+    }
+
+    let mut all = String::new();
+    all.push_str("fn register_all() -> Vec<(&'static str, Option<&'static str>)> {\n");
+    all.push_str("let mut long_codes: Vec<(&'static str, Option<&'static str>)> = Vec::new();\n");
+    all.push_str(r#"
+macro_rules! register_diagnostics {
+    ($($code:tt),*) => {{
+        long_codes.extend([$(
+            stringify!($code),
+        )*].iter().cloned().map(|s| (s, None)).collect::<Vec<_>>());
+    }};
+    ($($code:tt),*,) => {{
+        long_codes.extend([$(
+            stringify!($code),
+        )*].iter().cloned().map(|s| (s, None)));
+    }}
+}
+
+macro_rules! register_long_diagnostics {
+    ($($code:tt: $description:tt),*) => {
+        {long_codes.extend([$(
+            (stringify!($code), Some(stringify!($description))),
+        )*].iter());}
+    };
+    ($($code:tt: $description:tt),*,) => {
+        {long_codes.extend([$(
+            (stringify!($code), Some(stringify!($description))),
+        )*].iter());}
+    }
+}"#);
+    for idx in 0..idx {
+        all.push_str(&format!(r#"include!(concat!(env!("OUT_DIR"), "/error_{}.rs"));"#, idx));
+    }
+    all.push_str("\nlong_codes\n");
+    all.push_str("}\n");
+
+    fs::write(&dest, all).unwrap();
+}

src/tools/error_index_generator/main.rs

@@ -2,22 +2,26 @@
 
 extern crate env_logger;
 extern crate syntax;
-extern crate serialize as rustc_serialize;
 
 use std::collections::BTreeMap;
 use std::env;
 use std::error::Error;
-use std::fs::{self, read_dir, File};
+use std::fs::File;
 use std::io::Write;
 use std::path::Path;
 use std::path::PathBuf;
 use std::cell::RefCell;
 
 use syntax::edition::DEFAULT_EDITION;
-use syntax::diagnostics::metadata::{get_metadata_dir, ErrorMetadataMap, ErrorMetadata};
 
 use rustdoc::html::markdown::{Markdown, IdMap, ErrorCodes, Playground};
-use rustc_serialize::json;
+
+pub struct ErrorMetadata {
+    pub description: Option<String>,
+}
+
+/// Mapping from error codes to metadata that can be (de)serialized.
+pub type ErrorMetadataMap = BTreeMap<String, ErrorMetadata>;
 
 enum OutputFormat {
     HTML(HTMLFormatter),
@@ -80,11 +84,7 @@ impl Formatter for HTMLFormatter {
             Some(_) => "error-described",
             None => "error-undescribed",
         };
-        let use_desc = match info.use_site {
-            Some(_) => "error-used",
-            None => "error-unused",
-        };
-        write!(output, "<div class=\"{} {}\">", desc_desc, use_desc)?;
+        write!(output, "<div class=\"{}\">", desc_desc)?;
 
         // Error title (with self-link).
         write!(output,
@@ -199,25 +199,6 @@ impl Formatter for MarkdownFormatter {
     }
 }
 
-/// Loads all the metadata files from `metadata_dir` into an in-memory map.
-fn load_all_errors(metadata_dir: &Path) -> Result<ErrorMetadataMap, Box<dyn Error>> {
-    let mut all_errors = BTreeMap::new();
-
-    for entry in read_dir(metadata_dir)? {
-        let path = entry?.path();
-
-        let metadata_str = fs::read_to_string(&path)?;
-
-        let some_errors: ErrorMetadataMap = json::decode(&metadata_str)?;
-
-        for (err_code, info) in some_errors {
-            all_errors.insert(err_code, info);
-        }
-    }
-
-    Ok(all_errors)
-}
-
 /// Output an HTML page for the errors in `err_map` to `output_path`.
 fn render_error_page<T: Formatter>(err_map: &ErrorMetadataMap, output_path: &Path,
                                    formatter: T) -> Result<(), Box<dyn Error>> {
@@ -234,9 +215,13 @@ fn render_error_page<T: Formatter>(err_map: &ErrorMetadataMap, output_path: &Pat
 }
 
 fn main_with_result(format: OutputFormat, dst: &Path) -> Result<(), Box<dyn Error>> {
-    let build_arch = env::var("CFG_BUILD")?;
-    let metadata_dir = get_metadata_dir(&build_arch);
-    let err_map = load_all_errors(&metadata_dir)?;
+    let long_codes = register_all();
+    let mut err_map = BTreeMap::new();
+    for (code, desc) in long_codes {
+        err_map.insert(code.to_string(), ErrorMetadata {
+            description: desc.map(String::from),
+        });
+    }
     match format {
         OutputFormat::Unknown(s)  => panic!("Unknown output format: {}", s),
         OutputFormat::HTML(h)     => render_error_page(&err_map, dst, h)?,
@@ -272,3 +257,5 @@ fn main() {
         panic!("{}", e.description());
     }
 }
+
+include!(concat!(env!("OUT_DIR"), "/error_codes.rs"));