From 69b8ef806b3742ba7d41a77cd216713c84f36254 Mon Sep 17 00:00:00 2001 From: Huon Wilson Date: Fri, 7 Mar 2014 14:31:41 +1100 Subject: [PATCH] rustdoc: run on plain Markdown files. This theoretically gives rustdoc the ability to render our guides, tutorial and manual (not in practice, since the files themselves need to be adjusted slightly to use Sundown-compatible functionality). Fixes #11392. --- src/doc/rustdoc.md | 25 +++++ src/librustdoc/html/markdown.rs | 18 +++- src/librustdoc/lib.rs | 38 ++++++- src/librustdoc/markdown.rs | 171 ++++++++++++++++++++++++++++++++ src/librustdoc/test.rs | 68 +++++++++---- 5 files changed, 292 insertions(+), 28 deletions(-) create mode 100644 src/librustdoc/markdown.rs diff --git a/src/doc/rustdoc.md b/src/doc/rustdoc.md index 545cafd7f31..415db46be5b 100644 --- a/src/doc/rustdoc.md +++ b/src/doc/rustdoc.md @@ -181,3 +181,28 @@ rustdoc will implicitly add `extern crate ;` where `` is the name the crate being tested to the top of each code example. This means that rustdoc must be able to find a compiled version of the library crate being tested. Extra search paths may be added via the `-L` flag to `rustdoc`. + +# Standalone Markdown files + +As well as Rust crates, rustdoc supports rendering pure Markdown files +into HTML and testing the code snippets from them. A Markdown file is +detected by a `.md` or `.markdown` extension. + +There are 4 options to modify the output that Rustdoc creates. +- `--markdown-css PATH`: adds a `` tag pointing to `PATH`. +- `--markdown-in-header FILE`: includes the contents of `FILE` at the + end of the `...` section. +- `--markdown-before-content FILE`: includes the contents of `FILE` + directly after ``, before the rendered content (including the + title). +- `--markdown-after-content FILE`: includes the contents of `FILE` + directly before ``, after all the rendered content. + +All of these can be specified multiple times, and they are output in +the order in which they are specified. The first line of the file must +be the title, prefixed with `%` (e.g. this page has `% Rust +Documentation` on the first line). + +Like with a Rust crate, the `--test` argument will run the code +examples to check they compile, and obeys any `--test-args` flags. The +tests are named after the last `#` heading. diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index 19a28931a8a..30040a1846c 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -28,7 +28,6 @@ use std::cast; use std::fmt; -use std::intrinsics; use std::io; use std::libc; use std::local_data; @@ -258,7 +257,7 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) { }; if ignore { return } vec::raw::buf_as_slice((*text).data, (*text).size as uint, |text| { - let tests: &mut ::test::Collector = intrinsics::transmute(opaque); + let tests = &mut *(opaque as *mut ::test::Collector); let text = str::from_utf8(text).unwrap(); let mut lines = text.lines().map(|l| stripped_filtered_line(l).unwrap_or(l)); let text = lines.to_owned_vec().connect("\n"); @@ -266,6 +265,19 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) { }) } } + extern fn header(_ob: *buf, text: *buf, level: libc::c_int, opaque: *libc::c_void) { + unsafe { + let tests = &mut *(opaque as *mut ::test::Collector); + if text.is_null() { + tests.register_header("", level as u32); + } else { + vec::raw::buf_as_slice((*text).data, (*text).size as uint, |text| { + let text = str::from_utf8(text).unwrap(); + tests.register_header(text, level as u32); + }) + } + } + } unsafe { let ob = bufnew(OUTPUT_UNIT); @@ -276,7 +288,7 @@ pub fn find_testable_code(doc: &str, tests: &mut ::test::Collector) { blockcode: Some(block), blockquote: None, blockhtml: None, - header: None, + header: Some(header), other: mem::init() }; diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index 19e3aed6462..94bc5ed2526 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -14,7 +14,7 @@ #[crate_type = "dylib"]; #[crate_type = "rlib"]; -#[feature(globs, struct_variant, managed_boxes)]; +#[feature(globs, struct_variant, managed_boxes, macro_rules)]; extern crate syntax; extern crate rustc; @@ -26,6 +26,7 @@ extern crate collections; extern crate testing = "test"; extern crate time; +use std::cell::RefCell; use std::local_data; use std::io; use std::io::{File, MemWriter}; @@ -44,6 +45,7 @@ pub mod html { pub mod markdown; pub mod render; } +pub mod markdown; pub mod passes; pub mod plugins; pub mod visit_ast; @@ -105,6 +107,19 @@ pub fn opts() -> ~[getopts::OptGroup] { optflag("", "test", "run code examples as tests"), optmulti("", "test-args", "arguments to pass to the test runner", "ARGS"), + optmulti("", "markdown-css", "CSS files to include via in a rendered Markdown file", + "FILES"), + optmulti("", "markdown-in-header", + "files to include inline in the section of a rendered Markdown file", + "FILES"), + optmulti("", "markdown-before-content", + "files to include inline between and the content of a rendered \ + Markdown file", + "FILES"), + optmulti("", "markdown-after-content", + "files to include inline between the content and of a rendered \ + Markdown file", + "FILES"), ] } @@ -137,8 +152,24 @@ pub fn main_args(args: &[~str]) -> int { } let input = matches.free[0].as_slice(); - if matches.opt_present("test") { - return test::run(input, &matches); + let libs = matches.opt_strs("L").map(|s| Path::new(s.as_slice())); + let libs = @RefCell::new(libs.move_iter().collect()); + + let test_args = matches.opt_strs("test-args"); + let test_args = test_args.iter().flat_map(|s| s.words()).map(|s| s.to_owned()).to_owned_vec(); + + let should_test = matches.opt_present("test"); + let markdown_input = input.ends_with(".md") || input.ends_with(".markdown"); + + let output = matches.opt_str("o").map(|s| Path::new(s)); + + match (should_test, markdown_input) { + (true, true) => return markdown::test(input, libs, test_args), + (true, false) => return test::run(input, libs, test_args), + + (false, true) => return markdown::render(input, output.unwrap_or(Path::new("doc")), + &matches), + (false, false) => {} } if matches.opt_strs("passes") == ~[~"list"] { @@ -163,7 +194,6 @@ pub fn main_args(args: &[~str]) -> int { info!("going to format"); let started = time::precise_time_ns(); - let output = matches.opt_str("o").map(|s| Path::new(s)); match matches.opt_str("w") { Some(~"html") | None => { match html::render::run(krate, output.unwrap_or(Path::new("doc"))) { diff --git a/src/librustdoc/markdown.rs b/src/librustdoc/markdown.rs new file mode 100644 index 00000000000..a998e3d6994 --- /dev/null +++ b/src/librustdoc/markdown.rs @@ -0,0 +1,171 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::{str, io}; +use std::cell::RefCell; +use std::vec_ng::Vec; + +use collections::HashSet; + +use getopts; +use testing; + +use html::escape::Escape; +use html::markdown::{Markdown, find_testable_code, reset_headers}; +use test::Collector; + +fn load_string(input: &Path) -> io::IoResult> { + let mut f = try!(io::File::open(input)); + let d = try!(f.read_to_end()); + Ok(str::from_utf8_owned(d)) +} +macro_rules! load_or_return { + ($input: expr, $cant_read: expr, $not_utf8: expr) => { + { + let input = Path::new($input); + match load_string(&input) { + Err(e) => { + let _ = writeln!(&mut io::stderr(), + "error reading `{}`: {}", input.display(), e); + return $cant_read; + } + Ok(None) => { + let _ = writeln!(&mut io::stderr(), + "error reading `{}`: not UTF-8", input.display()); + return $not_utf8; + } + Ok(Some(s)) => s + } + } + } +} + +/// Separate any lines at the start of the file that begin with `%`. +fn extract_leading_metadata<'a>(s: &'a str) -> (Vec<&'a str>, &'a str) { + let mut metadata = Vec::new(); + for line in s.lines() { + if line.starts_with("%") { + // remove % + metadata.push(line.slice_from(1).trim_left()) + } else { + let line_start_byte = s.subslice_offset(line); + return (metadata, s.slice_from(line_start_byte)); + } + } + // if we're here, then all lines were metadata % lines. + (metadata, "") +} + +fn load_external_files(names: &[~str]) -> Option<~str> { + let mut out = ~""; + for name in names.iter() { + out.push_str(load_or_return!(name.as_slice(), None, None)); + out.push_char('\n'); + } + Some(out) +} + +/// Render `input` (e.g. "foo.md") into an HTML file in `output` +/// (e.g. output = "bar" => "bar/foo.html"). +pub fn render(input: &str, mut output: Path, matches: &getopts::Matches) -> int { + let input_p = Path::new(input); + output.push(input_p.filestem().unwrap()); + output.set_extension("html"); + + let mut css = ~""; + for name in matches.opt_strs("markdown-css").iter() { + let s = format!("\n", name); + css.push_str(s) + } + + let input_str = load_or_return!(input, 1, 2); + + let (in_header, before_content, after_content) = + match (load_external_files(matches.opt_strs("markdown-in-header")), + load_external_files(matches.opt_strs("markdown-before-content")), + load_external_files(matches.opt_strs("markdown-after-content"))) { + (Some(a), Some(b), Some(c)) => (a,b,c), + _ => return 3 + }; + + let mut out = match io::File::create(&output) { + Err(e) => { + let _ = writeln!(&mut io::stderr(), + "error opening `{}` for writing: {}", + output.display(), e); + return 4; + } + Ok(f) => f + }; + + let (metadata, text) = extract_leading_metadata(input_str); + if metadata.len() == 0 { + let _ = writeln!(&mut io::stderr(), + "invalid markdown file: expecting initial line with `% ...TITLE...`"); + return 5; + } + let title = metadata.get(0).as_slice(); + + reset_headers(); + + let err = write!( + &mut out, + r#" + + + + + {title} + + {css} + {in_header} + + + + + {before_content} +

{title}

+ {text} + {after_content} + +"#, + title = Escape(title), + css = css, + in_header = in_header, + before_content = before_content, + text = Markdown(text), + after_content = after_content); + + match err { + Err(e) => { + let _ = writeln!(&mut io::stderr(), + "error writing to `{}`: {}", + output.display(), e); + 6 + } + Ok(_) => 0 + } +} + +/// Run any tests/code examples in the markdown file `input`. +pub fn test(input: &str, libs: @RefCell>, mut test_args: ~[~str]) -> int { + let input_str = load_or_return!(input, 1, 2); + + let mut collector = Collector::new(input.to_owned(), libs, true); + find_testable_code(input_str, &mut collector); + test_args.unshift(~"rustdoctest"); + testing::test_main(test_args, collector.tests); + 0 +} diff --git a/src/librustdoc/test.rs b/src/librustdoc/test.rs index 5edc24c6066..640a3304094 100644 --- a/src/librustdoc/test.rs +++ b/src/librustdoc/test.rs @@ -9,6 +9,7 @@ // except according to those terms. use std::cell::RefCell; +use std::char; use std::io; use std::io::Process; use std::local_data; @@ -22,7 +23,6 @@ use rustc::back::link; use rustc::driver::driver; use rustc::driver::session; use rustc::metadata::creader::Loader; -use getopts; use syntax::diagnostic; use syntax::parse; use syntax::codemap::CodeMap; @@ -35,11 +35,9 @@ use html::markdown; use passes; use visit_ast::RustdocVisitor; -pub fn run(input: &str, matches: &getopts::Matches) -> int { +pub fn run(input: &str, libs: @RefCell>, mut test_args: ~[~str]) -> int { let input_path = Path::new(input); let input = driver::FileInput(input_path.clone()); - let libs = matches.opt_strs("L").map(|s| Path::new(s.as_slice())); - let libs = @RefCell::new(libs.move_iter().collect()); let sessopts = @session::Options { maybe_sysroot: Some(@os::self_exe_path().unwrap().dir_path()), @@ -79,21 +77,12 @@ pub fn run(input: &str, matches: &getopts::Matches) -> int { let (krate, _) = passes::unindent_comments(krate); let (krate, _) = passes::collapse_docs(krate); - let mut collector = Collector { - tests: ~[], - names: ~[], - cnt: 0, - libs: libs, - cratename: krate.name.to_owned(), - }; + let mut collector = Collector::new(krate.name.to_owned(), libs, false); collector.fold_crate(krate); - let args = matches.opt_strs("test-args"); - let mut args = args.iter().flat_map(|s| s.words()).map(|s| s.to_owned()); - let mut args = args.to_owned_vec(); - args.unshift(~"rustdoctest"); + test_args.unshift(~"rustdoctest"); - testing::test_main(args, collector.tests); + testing::test_main(test_args, collector.tests); 0 } @@ -198,17 +187,35 @@ fn maketest(s: &str, cratename: &str) -> ~str { } pub struct Collector { - priv tests: ~[testing::TestDescAndFn], + tests: ~[testing::TestDescAndFn], priv names: ~[~str], priv libs: @RefCell>, priv cnt: uint, + priv use_headers: bool, + priv current_header: Option<~str>, priv cratename: ~str, } impl Collector { - pub fn add_test(&mut self, test: &str, should_fail: bool, no_run: bool) { - let test = test.to_owned(); - let name = format!("{}_{}", self.names.connect("::"), self.cnt); + pub fn new(cratename: ~str, libs: @RefCell>, use_headers: bool) -> Collector { + Collector { + tests: ~[], + names: ~[], + libs: libs, + cnt: 0, + use_headers: use_headers, + current_header: None, + cratename: cratename + } + } + + pub fn add_test(&mut self, test: ~str, should_fail: bool, no_run: bool) { + let name = if self.use_headers { + let s = self.current_header.as_ref().map(|s| s.as_slice()).unwrap_or(""); + format!("{}_{}", s, self.cnt) + } else { + format!("{}_{}", self.names.connect("::"), self.cnt) + }; self.cnt += 1; let libs = self.libs.borrow(); let libs = (*libs.get()).clone(); @@ -225,6 +232,25 @@ impl Collector { }), }); } + + pub fn register_header(&mut self, name: &str, level: u32) { + if self.use_headers && level == 1 { + // we use these headings as test names, so it's good if + // they're valid identifiers. + let name = name.chars().enumerate().map(|(i, c)| { + if (i == 0 && char::is_XID_start(c)) || + (i != 0 && char::is_XID_continue(c)) { + c + } else { + '_' + } + }).collect::<~str>(); + + // new header => reset count. + self.cnt = 0; + self.current_header = Some(name); + } + } } impl DocFolder for Collector { @@ -237,7 +263,7 @@ impl DocFolder for Collector { match item.doc_value() { Some(doc) => { self.cnt = 0; - markdown::find_testable_code(doc, self); + markdown::find_testable_code(doc, &mut *self); } None => {} }