Introduce test result output file option for libtest

Separating test results from other output **avoids contamination**. If `libtest` only writes output to stdout, any non-test output (log messages, debug prints, etc.) may corrupt the stream and break parsers. Rust’s `println`’s are wrapped by libtest, but anything can (and does, in real world) use `libc`, or have C code using `libc` that corrupts stdout. Also, in practice, projects often resort to external post-processing to filter test output. As one [tracking discussion notes](rust-lang#85563), *“due to limitations of Rust libtest formatters, Rust developers often use a separate tool to postprocess the test results output”*. By writing test results directly to a file, we can guarantee the test results are isolated and parseable, without third-party noise.

Author: patskovn

library/test/src/cli.rs

@@ -18,6 +18,7 @@ pub struct TestOpts {
     pub run_tests: bool,
     pub bench_benchmarks: bool,
     pub logfile: Option<PathBuf>,
+    pub test_results_file: Option<PathBuf>,
     pub nocapture: bool,
     pub color: ColorConfig,
     pub format: OutputFormat,
@@ -59,6 +60,7 @@ fn optgroups() -> getopts::Options {
         .optflag("", "list", "List all tests and benchmarks")
         .optflag("h", "help", "Display this message")
         .optopt("", "logfile", "Write logs to the specified file (deprecated)", "PATH")
+        .optopt("", "test-results-file", "Write test results to the specified file", "PATH")
         .optflag(
             "",
             "no-capture",
@@ -275,6 +277,7 @@ fn parse_opts_impl(matches: getopts::Matches) -> OptRes {
     let run_tests = !bench_benchmarks || matches.opt_present("test");
 
     let logfile = get_log_file(&matches)?;
+    let test_results_file = get_test_results_file(&matches)?;
     let run_ignored = get_run_ignored(&matches, include_ignored)?;
     let filters = matches.free.clone();
     let nocapture = get_nocapture(&matches)?;
@@ -298,6 +301,7 @@ fn parse_opts_impl(matches: getopts::Matches) -> OptRes {
         run_tests,
         bench_benchmarks,
         logfile,
+        test_results_file,
         nocapture,
         color,
         format,
@@ -500,3 +504,9 @@ fn get_log_file(matches: &getopts::Matches) -> OptPartRes<Option<PathBuf>> {
 
     Ok(logfile)
 }
+
+fn get_test_results_file(matches: &getopts::Matches) -> OptPartRes<Option<PathBuf>> {
+    let test_results_file = matches.opt_str("test-results-file").map(|s| PathBuf::from(&s));
+
+    Ok(test_results_file)
+}

library/test/src/console.rs

@@ -1,8 +1,9 @@
 //! Module providing interface for running tests in the console.
 
-use std::fs::File;
+use std::fs::{File, OpenOptions};
 use std::io;
 use std::io::prelude::Write;
+use std::path::PathBuf;
 use std::time::Instant;
 
 use super::bench::fmt_bench_samples;
@@ -171,11 +172,7 @@ impl ConsoleTestState {
 
 // List the tests to console, and optionally to logfile. Filters are honored.
 pub(crate) fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<()> {
-    let output = match term::stdout() {
-        None => OutputLocation::Raw(io::stdout().lock()),
-        Some(t) => OutputLocation::Pretty(t),
-    };
-
+    let output = build_output(&opts.test_results_file)?;
     let mut out: Box<dyn OutputFormatter> = match opts.format {
         OutputFormat::Pretty | OutputFormat::Junit => {
             Box::new(PrettyFormatter::new(output, false, 0, false, None))
@@ -211,6 +208,24 @@ pub(crate) fn list_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) ->
     out.write_discovery_finish(&st)
 }
 
+pub(crate) fn build_output(
+    test_results_file: &Option<PathBuf>,
+) -> io::Result<OutputLocation<Box<dyn Write>>> {
+    let output: OutputLocation<Box<dyn Write>> = match test_results_file {
+        Some(results_file_path) => {
+            let file_output =
+                OpenOptions::new().write(true).create_new(true).open(results_file_path)?;
+
+            OutputLocation::Raw(Box::new(file_output))
+        }
+        None => match term::stdout() {
+            None => OutputLocation::Raw(Box::new(io::stdout().lock())),
+            Some(t) => OutputLocation::Pretty(t),
+        },
+    };
+    Ok(output)
+}
+
 // Updates `ConsoleTestState` depending on result of the test execution.
 fn handle_test_result(st: &mut ConsoleTestState, completed_test: CompletedTest) {
     let test = completed_test.desc;
@@ -284,10 +299,7 @@ fn on_test_event(
 /// A simple console test runner.
 /// Runs provided tests reporting process and results to the stdout.
 pub fn run_tests_console(opts: &TestOpts, tests: Vec<TestDescAndFn>) -> io::Result<bool> {
-    let output = match term::stdout() {
-        None => OutputLocation::Raw(io::stdout()),
-        Some(t) => OutputLocation::Pretty(t),
-    };
+    let output = build_output(&opts.test_results_file)?;
 
     let max_name_len = tests
         .iter()

library/test/src/tests.rs

@@ -1,3 +1,5 @@
+use std::path::PathBuf;
+
 use super::*;
 use crate::{
     console::OutputLocation,
@@ -24,6 +26,7 @@ impl TestOpts {
             run_tests: false,
             bench_benchmarks: false,
             logfile: None,
+            test_results_file: None,
             nocapture: false,
             color: AutoColor,
             format: OutputFormat::Pretty,
@@ -468,6 +471,29 @@ fn parse_include_ignored_flag() {
     assert_eq!(opts.run_ignored, RunIgnored::Yes);
 }
 
+#[test]
+fn parse_test_results_file_flag_reads_value() {
+    let args = vec![
+        "progname".to_string(),
+        "filter".to_string(),
+        "--test-results-file".to_string(),
+        "expected_path_to_results_file".to_string(),
+    ];
+    let opts = parse_opts(&args).unwrap().unwrap();
+    assert_eq!(opts.test_results_file, Some(PathBuf::from("expected_path_to_results_file")));
+}
+
+#[test]
+fn parse_test_results_file_requires_value() {
+    let args =
+        vec!["progname".to_string(), "filter".to_string(), "--test-results-file".to_string()];
+    let maybe_opts = parse_opts(&args).unwrap();
+    assert_eq!(
+        maybe_opts.err(),
+        Some("Argument to option 'test-results-file' missing".to_string())
+    );
+}
+
 #[test]
 fn filter_for_ignored_option() {
     // When we run ignored tests the test filter should filter out all the

src/tools/compiletest/src/executor/libtest.rs

@@ -94,6 +94,7 @@ fn test_opts(config: &Config) -> test::TestOpts {
         run_ignored: if config.run_ignored { test::RunIgnored::Yes } else { test::RunIgnored::No },
         format: config.format.to_libtest(),
         logfile: None,
+        test_results_file: None,
         run_tests: true,
         bench_benchmarks: true,
         nocapture: config.nocapture,