Auto merge of #56189 - rep-nop:keep_doc_test_executable, r=QuietMisdreavus

rustdoc: Add option to persist doc test executables

Fixes #37048.

This is the initial version of the code so the doctest executables can be used for stuff like code coverage (specifically xd009642/tarpaulin#13) the folders it goes into were just a first idea, so any better ones are welcome.

Right now it creates a directory structure like:
```
  given_path/
          |_____ <filename>_rs_<linenum>/
          |_____ ...
          |_____ <filename>_rs_<linenum>/
                 |_____ rust_out

```
I couldn't figure out where it actually outputs the file w/ the name, I suspect its somewhere deeper in the compiler.

It also adds the unstable `--persist-doctests` flag to `rustdoc` that enables this behavior.

Author: bors

Diff for: src/doc/rustdoc/src/unstable-features.md

@@ -417,3 +417,15 @@ JavaScript, and font files in a single location, rather than duplicating it once
 (grouping of crate docs generated into the same output directory, like with `cargo doc`). Per-crate
 files like the search index will still load from the documentation root, but anything that gets
 renamed with `--resource-suffix` will load from the given path.
+
+### `--persist-doctests`: persist doctest executables after running
+
+Using this flag looks like this:
+
+```bash
+$ rustdoc src/lib.rs --test -Z unstable-options --persist-doctests target/rustdoctest
+```
+
+This flag allows you to keep doctest executables around after they're compiled or run.
+Usually, rustdoc will immediately discard a compiled doctest after it's been tested, but
+with this option, you can keep those binaries around for farther testing.

Diff for: src/librustdoc/config.rs

@@ -68,6 +68,9 @@ pub struct Options {
     pub should_test: bool,
     /// List of arguments to pass to the test harness, if running tests.
     pub test_args: Vec<String>,
+    /// Optional path to persist the doctest executables to, defaults to a
+    /// temporary directory if not set.
+    pub persist_doctests: Option<PathBuf>,
 
     // Options that affect the documentation process
 
@@ -121,6 +124,7 @@ impl fmt::Debug for Options {
             .field("lint_cap", &self.lint_cap)
             .field("should_test", &self.should_test)
             .field("test_args", &self.test_args)
+            .field("persist_doctests", &self.persist_doctests)
             .field("default_passes", &self.default_passes)
             .field("manual_passes", &self.manual_passes)
             .field("display_warnings", &self.display_warnings)
@@ -431,6 +435,7 @@ impl Options {
         let enable_index_page = matches.opt_present("enable-index-page") || index_page.is_some();
         let static_root_path = matches.opt_str("static-root-path");
         let generate_search_filter = !matches.opt_present("disable-per-crate-search");
+        let persist_doctests = matches.opt_str("persist-doctests").map(PathBuf::from);
 
         let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(matches, error_format);
 
@@ -456,6 +461,7 @@ impl Options {
             manual_passes,
             display_warnings,
             crate_version,
+            persist_doctests,
             render_options: RenderOptions {
                 output,
                 external_html,

Diff for: src/librustdoc/lib.rs

@@ -341,6 +341,12 @@ fn opts() -> Vec<RustcOptGroup> {
                       "disable-per-crate-search",
                       "disables generating the crate selector on the search box")
         }),
+        unstable("persist-doctests", |o| {
+             o.optopt("",
+                       "persist-doctests",
+                       "Directory to persist doctest executables into",
+                       "PATH")
+        }),
     ]
 }
 

Diff for: src/librustdoc/markdown.rs

@@ -142,7 +142,7 @@ pub fn test(mut options: Options, diag: &errors::Handler) -> isize {
                                        options.libs, options.codegen_options, options.externs,
                                        true, opts, options.maybe_sysroot, None,
                                        Some(options.input),
-                                       options.linker, options.edition);
+                                       options.linker, options.edition, options.persist_doctests);
     collector.set_position(DUMMY_SP);
     let codes = ErrorCodes::from(UnstableFeatures::from_environment().is_nightly_build());
     let res = find_testable_code(&input_str, &mut collector, codes);

Diff for: src/librustdoc/test.rs

@@ -120,7 +120,8 @@ pub fn run(mut options: Options) -> isize {
             Some(source_map),
             None,
             options.linker,
-            options.edition
+            options.edition,
+            options.persist_doctests,
         );
 
         {
@@ -184,7 +185,8 @@ fn run_test(test: &str, cratename: &str, filename: &FileName, line: usize,
             cg: CodegenOptions, externs: Externs,
             should_panic: bool, no_run: bool, as_test_harness: bool,
             compile_fail: bool, mut error_codes: Vec<String>, opts: &TestOptions,
-            maybe_sysroot: Option<PathBuf>, linker: Option<PathBuf>, edition: Edition) {
+            maybe_sysroot: Option<PathBuf>, linker: Option<PathBuf>, edition: Edition,
+            persist_doctests: Option<PathBuf>) {
     // The test harness wants its own `main` and top-level functions, so
     // never wrap the test in `fn main() { ... }`.
     let (test, line_offset) = make_test(test, Some(cratename), as_test_harness, opts);
@@ -249,6 +251,20 @@ fn run_test(test: &str, cratename: &str, filename: &FileName, line: usize,
     let old = io::set_panic(Some(box Sink(data.clone())));
     let _bomb = Bomb(data.clone(), old.unwrap_or(box io::stdout()));
 
+    enum DirState {
+        Temp(tempfile::TempDir),
+        Perm(PathBuf),
+    }
+
+    impl DirState {
+        fn path(&self) -> &std::path::Path {
+            match self {
+                DirState::Temp(t) => t.path(),
+                DirState::Perm(p) => p.as_path(),
+            }
+        }
+    }
+
     let (libdir, outdir, compile_result) = driver::spawn_thread_pool(sessopts, |sessopts| {
         let source_map = Lrc::new(SourceMap::new(sessopts.file_path_mapping()));
         let emitter = errors::emitter::EmitterWriter::new(box Sink(data.clone()),
@@ -267,7 +283,26 @@ fn run_test(test: &str, cratename: &str, filename: &FileName, line: usize,
         rustc_lint::register_builtins(&mut sess.lint_store.borrow_mut(), Some(&sess));
 
         let outdir = Mutex::new(
-            TempFileBuilder::new().prefix("rustdoctest").tempdir().expect("rustdoc needs a tempdir")
+            if let Some(mut path) = persist_doctests {
+                path.push(format!("{}_{}",
+                    filename
+                        .to_string()
+                        .rsplit('/')
+                        .next()
+                        .unwrap()
+                        .replace(".", "_"),
+                        line)
+                );
+                std::fs::create_dir_all(&path)
+                    .expect("Couldn't create directory for doctest executables");
+
+                DirState::Perm(path)
+            } else {
+                DirState::Temp(TempFileBuilder::new()
+                                .prefix("rustdoctest")
+                                .tempdir()
+                                .expect("rustdoc needs a tempdir"))
+            }
         );
         let libdir = sess.target_filesearch(PathKind::All).get_lib_path();
         let mut control = driver::CompileController::basic();
@@ -629,13 +664,15 @@ pub struct Collector {
     filename: Option<PathBuf>,
     linker: Option<PathBuf>,
     edition: Edition,
+    persist_doctests: Option<PathBuf>,
 }
 
 impl Collector {
     pub fn new(cratename: String, cfgs: Vec<String>, libs: Vec<SearchPath>, cg: CodegenOptions,
                externs: Externs, use_headers: bool, opts: TestOptions,
                maybe_sysroot: Option<PathBuf>, source_map: Option<Lrc<SourceMap>>,
-               filename: Option<PathBuf>, linker: Option<PathBuf>, edition: Edition) -> Collector {
+               filename: Option<PathBuf>, linker: Option<PathBuf>, edition: Edition,
+               persist_doctests: Option<PathBuf>) -> Collector {
         Collector {
             tests: Vec::new(),
             names: Vec::new(),
@@ -652,6 +689,7 @@ impl Collector {
             filename,
             linker,
             edition,
+            persist_doctests,
         }
     }
 
@@ -695,6 +733,8 @@ impl Tester for Collector {
         let maybe_sysroot = self.maybe_sysroot.clone();
         let linker = self.linker.clone();
         let edition = config.edition.unwrap_or(self.edition);
+        let persist_doctests = self.persist_doctests.clone();
+
         debug!("Creating test {}: {}", name, test);
         self.tests.push(testing::TestDescAndFn {
             desc: testing::TestDesc {
@@ -727,7 +767,8 @@ impl Tester for Collector {
                                  &opts,
                                  maybe_sysroot,
                                  linker,
-                                 edition)
+                                 edition,
+                                 persist_doctests)
                     }))
                 } {
                     Ok(()) => (),

Diff for: src/test/rustdoc-ui/failed-doctest-output.stdout

@@ -12,7 +12,7 @@ error[E0425]: cannot find value `no` in this scope
 3 | no
   | ^^ not found in this scope
 
-thread '$DIR/failed-doctest-output.rs - OtherStruct (line 17)' panicked at 'couldn't compile the test', src/librustdoc/test.rs:319:13
+thread '$DIR/failed-doctest-output.rs - OtherStruct (line 17)' panicked at 'couldn't compile the test', src/librustdoc/test.rs:354:13
 note: Run with `RUST_BACKTRACE=1` environment variable to display a backtrace.
 
 ---- $DIR/failed-doctest-output.rs - SomeStruct (line 11) stdout ----
@@ -21,7 +21,7 @@ thread '$DIR/failed-doctest-output.rs - SomeStruct (line 11)' panicked at 'test
 thread 'main' panicked at 'oh no', $DIR/failed-doctest-output.rs:3:1
 note: Run with `RUST_BACKTRACE=1` environment variable to display a backtrace.
 
-', src/librustdoc/test.rs:354:17
+', src/librustdoc/test.rs:389:17
 
 
 failures: