compiletest: Add support for metrics and ratchet modes.
This commit is contained in:
parent
f2f7fb3ae1
commit
bc4c89c10a
3 changed files with 68 additions and 9 deletions
|
@ -58,6 +58,15 @@ pub struct config {
|
|||
// Write out a parseable log of tests that were run
|
||||
logfile: Option<Path>,
|
||||
|
||||
// Write out a json file containing any metrics of the run
|
||||
save_metrics: Option<Path>,
|
||||
|
||||
// Write and ratchet a metrics file
|
||||
ratchet_metrics: Option<Path>,
|
||||
|
||||
// Percent change in metrics to consider noise
|
||||
ratchet_noise_percent: Option<f64>,
|
||||
|
||||
// A command line to prefix program execution with,
|
||||
// for running under valgrind
|
||||
runtool: Option<~str>,
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
extern mod extra;
|
||||
|
||||
use std::os;
|
||||
use std::f64;
|
||||
|
||||
use extra::getopts;
|
||||
use extra::getopts::groups::{optopt, optflag, reqopt};
|
||||
|
@ -66,6 +67,10 @@ pub fn parse_config(args: ~[~str]) -> config {
|
|||
optopt("", "rustcflags", "flags to pass to rustc", "FLAGS"),
|
||||
optflag("", "verbose", "run tests verbosely, showing all output"),
|
||||
optopt("", "logfile", "file to log test execution to", "FILE"),
|
||||
optopt("", "save-metrics", "file to save metrics to", "FILE"),
|
||||
optopt("", "ratchet-metrics", "file to ratchet metrics against", "FILE"),
|
||||
optopt("", "ratchet-noise-percent",
|
||||
"percent change in metrics to consider noise", "N"),
|
||||
optflag("", "jit", "run tests under the JIT"),
|
||||
optflag("", "newrt", "run tests on the new runtime / scheduler"),
|
||||
optopt("", "target", "the target to build for", "TARGET"),
|
||||
|
@ -116,6 +121,13 @@ pub fn parse_config(args: ~[~str]) -> config {
|
|||
Some(copy matches.free[0])
|
||||
} else { None },
|
||||
logfile: getopts::opt_maybe_str(matches, "logfile").map(|s| Path(*s)),
|
||||
save_metrics: getopts::opt_maybe_str(matches, "save-metrics").map(|s| Path(*s)),
|
||||
ratchet_metrics:
|
||||
getopts::opt_maybe_str(matches, "ratchet-metrics").map(|s| Path(*s)),
|
||||
ratchet_noise_percent:
|
||||
getopts::opt_maybe_str(matches,
|
||||
"ratchet-noise-percent").map(|s|
|
||||
f64::from_str(*s).get()),
|
||||
runtool: getopts::opt_maybe_str(matches, "runtool"),
|
||||
rustcflags: getopts::opt_maybe_str(matches, "rustcflags"),
|
||||
jit: getopts::opt_present(matches, "jit"),
|
||||
|
@ -215,10 +227,10 @@ pub fn test_opts(config: &config) -> test::TestOpts {
|
|||
run_ignored: config.run_ignored,
|
||||
logfile: copy config.logfile,
|
||||
run_tests: true,
|
||||
run_benchmarks: false,
|
||||
ratchet_metrics: None,
|
||||
ratchet_noise_percent: None,
|
||||
save_metrics: None,
|
||||
run_benchmarks: true,
|
||||
ratchet_metrics: copy config.ratchet_metrics,
|
||||
ratchet_noise_percent: copy config.ratchet_noise_percent,
|
||||
save_metrics: copy config.save_metrics,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -231,7 +243,13 @@ pub fn make_tests(config: &config) -> ~[test::TestDescAndFn] {
|
|||
let file = copy *file;
|
||||
debug!("inspecting file %s", file.to_str());
|
||||
if is_test(config, file) {
|
||||
tests.push(make_test(config, file))
|
||||
let t = do make_test(config, file) {
|
||||
match config.mode {
|
||||
mode_codegen => make_metrics_test_closure(config, file),
|
||||
_ => make_test_closure(config, file)
|
||||
}
|
||||
};
|
||||
tests.push(t)
|
||||
}
|
||||
}
|
||||
tests
|
||||
|
@ -260,14 +278,15 @@ pub fn is_test(config: &config, testfile: &Path) -> bool {
|
|||
return valid;
|
||||
}
|
||||
|
||||
pub fn make_test(config: &config, testfile: &Path) -> test::TestDescAndFn {
|
||||
pub fn make_test(config: &config, testfile: &Path,
|
||||
f: &fn()->test::TestFn) -> test::TestDescAndFn {
|
||||
test::TestDescAndFn {
|
||||
desc: test::TestDesc {
|
||||
name: make_test_name(config, testfile),
|
||||
ignore: header::is_test_ignored(config, testfile),
|
||||
should_fail: false
|
||||
},
|
||||
testfn: make_test_closure(config, testfile),
|
||||
testfn: f(),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -291,3 +310,10 @@ pub fn make_test_closure(config: &config, testfile: &Path) -> test::TestFn {
|
|||
let testfile = Cell::new(testfile.to_str());
|
||||
test::DynTestFn(|| { runtest::run(config.take(), testfile.take()) })
|
||||
}
|
||||
|
||||
pub fn make_metrics_test_closure(config: &config, testfile: &Path) -> test::TestFn {
|
||||
use std::cell::Cell;
|
||||
let config = Cell::new(copy *config);
|
||||
let testfile = Cell::new(testfile.to_str());
|
||||
test::DynMetricFn(|mm| { runtest::run_metrics(config.take(), testfile.take(), mm) })
|
||||
}
|
||||
|
|
|
@ -25,7 +25,14 @@ use std::os;
|
|||
use std::uint;
|
||||
use std::vec;
|
||||
|
||||
use extra::test::MetricMap;
|
||||
|
||||
pub fn run(config: config, testfile: ~str) {
|
||||
let mut _mm = MetricMap::new();
|
||||
run_metrics(config, testfile, &mut _mm);
|
||||
}
|
||||
|
||||
pub fn run_metrics(config: config, testfile: ~str, mm: &mut MetricMap) {
|
||||
if config.verbose {
|
||||
// We're going to be dumping a lot of info. Start on a new line.
|
||||
io::stdout().write_str("\n\n");
|
||||
|
@ -40,7 +47,7 @@ pub fn run(config: config, testfile: ~str) {
|
|||
mode_run_pass => run_rpass_test(&config, &props, &testfile),
|
||||
mode_pretty => run_pretty_test(&config, &props, &testfile),
|
||||
mode_debug_info => run_debuginfo_test(&config, &props, &testfile),
|
||||
mode_codegen => run_codegen_test(&config, &props, &testfile)
|
||||
mode_codegen => run_codegen_test(&config, &props, &testfile, mm)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -906,7 +913,14 @@ fn disassemble_extract(config: &config, _props: &TestProps,
|
|||
}
|
||||
|
||||
|
||||
fn run_codegen_test(config: &config, props: &TestProps, testfile: &Path) {
|
||||
fn count_extracted_lines(p: &Path) -> uint {
|
||||
let x = io::read_whole_file_str(&p.with_filetype("ll")).get();
|
||||
x.line_iter().len_()
|
||||
}
|
||||
|
||||
|
||||
fn run_codegen_test(config: &config, props: &TestProps,
|
||||
testfile: &Path, mm: &mut MetricMap) {
|
||||
|
||||
if config.llvm_bin_path.is_none() {
|
||||
fatal(~"missing --llvm-bin-path");
|
||||
|
@ -947,7 +961,17 @@ fn run_codegen_test(config: &config, props: &TestProps, testfile: &Path) {
|
|||
fatal_ProcRes(~"disassembling extract failed", &ProcRes);
|
||||
}
|
||||
|
||||
let base = output_base_name(config, testfile);
|
||||
let base_extract = append_suffix_to_stem(&base, "extract");
|
||||
|
||||
let base_clang = append_suffix_to_stem(&base, "clang");
|
||||
let base_clang_extract = append_suffix_to_stem(&base_clang, "extract");
|
||||
|
||||
let base_lines = count_extracted_lines(&base_extract);
|
||||
let clang_lines = count_extracted_lines(&base_clang_extract);
|
||||
|
||||
mm.insert_metric("clang-codegen-ratio",
|
||||
(base_lines as f64) / (clang_lines as f64),
|
||||
0.001);
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue