Scrape code examples from examples/ directory for Rustdoc

src/doc/rustdoc/src/unstable-features.md

@@ -455,3 +455,17 @@ Calculating code examples follows these rules:
   * static
   * typedef
 2. If one of the previously listed items has a code example, then it'll be counted.
+
+### `--with-examples`: include examples of uses of items as documentation
+
+This option, combined with `--scrape-examples-target-crate` and `--scrape-examples-output-path`, is used to implement the functionality in [RFC #3123](https://github.com/rust-lang/rfcs/pull/3123). Uses of an item (currently functions / call-sites) are found in a crate and its reverse-dependencies, and then the uses are included as documentation for that item. This feature is intended to be used via `cargo doc --scrape-examples`, but the rustdoc-only workflow looks like:
+
+```bash
+$ rustdoc examples/ex.rs -Z unstable-options \
+    --extern foobar=target/deps/libfoobar.rmeta \
+    --scrape-examples-target-crate foobar \
+    --scrape-examples-output-path output.calls
+$ rustdoc src/lib.rs -Z unstable-options --with-examples output.calls
+```
+
+First, the library must be checked to generate an `rmeta`. Then a reverse-dependency like `examples/ex.rs` is given to rustdoc with the target crate being documented (`foobar`) and a path to output the calls (`output.calls`). Then, the generated calls file can be passed via `--with-examples` to the subsequent documentation of `foobar`.

src/librustdoc/clean/inline.rs

@@ -235,6 +235,7 @@ fn build_external_function(cx: &mut DocContext<'_>, did: DefId) -> clean::Functi
         decl,
         generics,
         header: hir::FnHeader { unsafety: sig.unsafety(), abi: sig.abi(), constness, asyncness },
+        def_id: did,
     }
 }
 

src/librustdoc/clean/mod.rs

@@ -801,7 +801,8 @@ impl<'a> Clean<Function> for (&'a hir::FnSig<'a>, &'a hir::Generics<'a>, hir::Bo
     fn clean(&self, cx: &mut DocContext<'_>) -> Function {
         let (generics, decl) =
             enter_impl_trait(cx, |cx| (self.1.clean(cx), (&*self.0.decl, self.2).clean(cx)));
-        Function { decl, generics, header: self.0.header }
+        let def_id = self.2.hir_id.owner.to_def_id();
+        Function { decl, generics, header: self.0.header, def_id }
     }
 }
 
@@ -933,7 +934,8 @@ impl Clean<Item> for hir::TraitItem<'_> {
                     let (generics, decl) = enter_impl_trait(cx, |cx| {
                         (self.generics.clean(cx), (&*sig.decl, &names[..]).clean(cx))
                     });
-                    let mut t = Function { header: sig.header, decl, generics };
+                    let def_id = self.def_id.to_def_id();
+                    let mut t = Function { header: sig.header, decl, generics, def_id };
                     if t.header.constness == hir::Constness::Const
                         && is_unstable_const_fn(cx.tcx, local_did).is_some()
                     {
@@ -1067,6 +1069,7 @@ impl Clean<Item> for ty::AssocItem {
                                 constness,
                                 asyncness,
                             },
+                            def_id: self.def_id,
                         },
                         defaultness,
                     )
@@ -1080,6 +1083,7 @@ impl Clean<Item> for ty::AssocItem {
                             constness: hir::Constness::NotConst,
                             asyncness: hir::IsAsync::NotAsync,
                         },
+                        def_id: self.def_id,
                     })
                 }
             }
@@ -2078,7 +2082,8 @@ fn clean_use_statement(
 impl Clean<Item> for (&hir::ForeignItem<'_>, Option<Symbol>) {
     fn clean(&self, cx: &mut DocContext<'_>) -> Item {
         let (item, renamed) = self;
-        cx.with_param_env(item.def_id.to_def_id(), |cx| {
+        let def_id = item.def_id.to_def_id();
+        cx.with_param_env(def_id, |cx| {
             let kind = match item.kind {
                 hir::ForeignItemKind::Fn(ref decl, ref names, ref generics) => {
                     let abi = cx.tcx.hir().get_foreign_abi(item.hir_id());
@@ -2098,6 +2103,7 @@ impl Clean<Item> for (&hir::ForeignItem<'_>, Option<Symbol>) {
                             constness: hir::Constness::NotConst,
                             asyncness: hir::IsAsync::NotAsync,
                         },
+                        def_id,
                     })
                 }
                 hir::ForeignItemKind::Static(ref ty, mutability) => {

src/librustdoc/clean/types.rs

@@ -1254,6 +1254,7 @@ crate struct Function {
     crate decl: FnDecl,
     crate generics: Generics,
     crate header: hir::FnHeader,
+    crate def_id: DefId,
 }
 
 #[derive(Clone, PartialEq, Eq, Debug, Hash)]

src/librustdoc/config.rs

@@ -25,6 +25,7 @@ use crate::html::render::StylePath;
 use crate::html::static_files;
 use crate::opts;
 use crate::passes::{self, Condition, DefaultPassOption};
+use crate::scrape_examples::{AllCallLocations, ScrapeExamplesOptions};
 use crate::theme;
 
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
@@ -158,6 +159,10 @@ crate struct Options {
     crate json_unused_externs: bool,
     /// Whether to skip capturing stdout and stderr of tests.
     crate nocapture: bool,
+
+    /// Configuration for scraping examples from the current crate. If this option is Some(..) then
+    /// the compiler will scrape examples and not generate documentation.
+    crate scrape_examples_options: Option<ScrapeExamplesOptions>,
 }
 
 impl fmt::Debug for Options {
@@ -202,6 +207,7 @@ impl fmt::Debug for Options {
             .field("run_check", &self.run_check)
             .field("no_run", &self.no_run)
             .field("nocapture", &self.nocapture)
+            .field("scrape_examples_options", &self.scrape_examples_options)
             .finish()
     }
 }
@@ -280,6 +286,7 @@ crate struct RenderOptions {
     crate emit: Vec<EmitType>,
     /// If `true`, HTML source pages will generate links for items to their definition.
     crate generate_link_to_definition: bool,
+    crate call_locations: AllCallLocations,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -671,6 +678,10 @@ impl Options {
             return Err(1);
         }
 
+        let scrape_examples_options = ScrapeExamplesOptions::new(&matches, &diag)?;
+        let with_examples = matches.opt_strs("with-examples");
+        let call_locations = crate::scrape_examples::load_call_locations(with_examples, &diag)?;
+
         let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(matches, error_format);
 
         Ok(Options {
@@ -737,10 +748,12 @@ impl Options {
                 ),
                 emit,
                 generate_link_to_definition,
+                call_locations,
             },
             crate_name,
             output_format,
             json_unused_externs,
+            scrape_examples_options,
         })
     }
 

src/librustdoc/html/highlight.rs

@@ -12,6 +12,7 @@ use crate::html::render::Context;
 use std::collections::VecDeque;
 use std::fmt::{Display, Write};
 
+use rustc_data_structures::fx::FxHashMap;
 use rustc_lexer::{LiteralKind, TokenKind};
 use rustc_span::edition::Edition;
 use rustc_span::symbol::Symbol;
@@ -30,6 +31,10 @@ crate struct ContextInfo<'a, 'b, 'c> {
     crate root_path: &'c str,
 }
 
+/// Decorations are represented as a map from CSS class to vector of character ranges.
+/// Each range will be wrapped in a span with that class.
+crate struct DecorationInfo(crate FxHashMap<&'static str, Vec<(u32, u32)>>);
+
 /// Highlights `src`, returning the HTML output.
 crate fn render_with_highlighting(
     src: &str,
@@ -40,6 +45,7 @@ crate fn render_with_highlighting(
     edition: Edition,
     extra_content: Option<Buffer>,
     context_info: Option<ContextInfo<'_, '_, '_>>,
+    decoration_info: Option<DecorationInfo>,
 ) {
     debug!("highlighting: ================\n{}\n==============", src);
     if let Some((edition_info, class)) = tooltip {
@@ -56,7 +62,7 @@ crate fn render_with_highlighting(
     }
 
     write_header(out, class, extra_content);
-    write_code(out, &src, edition, context_info);
+    write_code(out, &src, edition, context_info, decoration_info);
     write_footer(out, playground_button);
 }
 
@@ -89,17 +95,23 @@ fn write_code(
     src: &str,
     edition: Edition,
     context_info: Option<ContextInfo<'_, '_, '_>>,
+    decoration_info: Option<DecorationInfo>,
 ) {
     // This replace allows to fix how the code source with DOS backline characters is displayed.
     let src = src.replace("\r\n", "\n");
-    Classifier::new(&src, edition, context_info.as_ref().map(|c| c.file_span).unwrap_or(DUMMY_SP))
-        .highlight(&mut |highlight| {
-            match highlight {
-                Highlight::Token { text, class } => string(out, Escape(text), class, &context_info),
-                Highlight::EnterSpan { class } => enter_span(out, class),
-                Highlight::ExitSpan => exit_span(out),
-            };
-        });
+    Classifier::new(
+        &src,
+        edition,
+        context_info.as_ref().map(|c| c.file_span).unwrap_or(DUMMY_SP),
+        decoration_info,
+    )
+    .highlight(&mut |highlight| {
+        match highlight {
+            Highlight::Token { text, class } => string(out, Escape(text), class, &context_info),
+            Highlight::EnterSpan { class } => enter_span(out, class),
+            Highlight::ExitSpan => exit_span(out),
+        };
+    });
 }
 
 fn write_footer(out: &mut Buffer, playground_button: Option<&str>) {
@@ -127,6 +139,7 @@ enum Class {
     PreludeTy,
     PreludeVal,
     QuestionMark,
+    Decoration(&'static str),
 }
 
 impl Class {
@@ -150,6 +163,7 @@ impl Class {
             Class::PreludeTy => "prelude-ty",
             Class::PreludeVal => "prelude-val",
             Class::QuestionMark => "question-mark",
+            Class::Decoration(kind) => kind,
         }
     }
 
@@ -248,6 +262,24 @@ impl Iterator for PeekIter<'a> {
     }
 }
 
+/// Custom spans inserted into the source. Eg --scrape-examples uses this to highlight function calls
+struct Decorations {
+    starts: Vec<(u32, &'static str)>,
+    ends: Vec<u32>,
+}
+
+impl Decorations {
+    fn new(info: DecorationInfo) -> Self {
+        let (starts, ends) = info
+            .0
+            .into_iter()
+            .map(|(kind, ranges)| ranges.into_iter().map(move |(lo, hi)| ((lo, kind), hi)))
+            .flatten()
+            .unzip();
+        Decorations { starts, ends }
+    }
+}
+
 /// Processes program tokens, classifying strings of text by highlighting
 /// category (`Class`).
 struct Classifier<'a> {
@@ -259,13 +291,20 @@ struct Classifier<'a> {
     byte_pos: u32,
     file_span: Span,
     src: &'a str,
+    decorations: Option<Decorations>,
 }
 
 impl<'a> Classifier<'a> {
     /// Takes as argument the source code to HTML-ify, the rust edition to use and the source code
     /// file span which will be used later on by the `span_correspondance_map`.
-    fn new(src: &str, edition: Edition, file_span: Span) -> Classifier<'_> {
+    fn new(
+        src: &str,
+        edition: Edition,
+        file_span: Span,
+        decoration_info: Option<DecorationInfo>,
+    ) -> Classifier<'_> {
         let tokens = PeekIter::new(TokenIter { src });
+        let decorations = decoration_info.map(Decorations::new);
         Classifier {
             tokens,
             in_attribute: false,
@@ -275,6 +314,7 @@ impl<'a> Classifier<'a> {
             byte_pos: 0,
             file_span,
             src,
+            decorations,
         }
     }
 
@@ -356,6 +396,19 @@ impl<'a> Classifier<'a> {
     /// token is used.
     fn highlight(mut self, sink: &mut dyn FnMut(Highlight<'a>)) {
         loop {
+            if let Some(decs) = self.decorations.as_mut() {
+                let byte_pos = self.byte_pos;
+                let n_starts = decs.starts.iter().filter(|(i, _)| byte_pos >= *i).count();
+                for (_, kind) in decs.starts.drain(0..n_starts) {
+                    sink(Highlight::EnterSpan { class: Class::Decoration(kind) });
+                }
+
+                let n_ends = decs.ends.iter().filter(|i| byte_pos >= **i).count();
+                for _ in decs.ends.drain(0..n_ends) {
+                    sink(Highlight::ExitSpan);
+                }
+            }
+
             if self
                 .tokens
                 .peek()
@@ -638,7 +691,7 @@ fn string<T: Display>(
                 // https://github.com/rust-lang/rust/blob/60f1a2fc4b535ead9c85ce085fdce49b1b097531/src/librustdoc/html/render/context.rs#L315-L338
                 match href {
                     LinkFromSrc::Local(span) => context
-                        .href_from_span(*span)
+                        .href_from_span(*span, true)
                         .map(|s| format!("{}{}", context_info.root_path, s)),
                     LinkFromSrc::External(def_id) => {
                         format::href_with_root_path(*def_id, context, Some(context_info.root_path))

src/librustdoc/html/highlight/fixtures/decorations.html

@@ -0,0 +1,2 @@
+<span class="example"><span class="kw">let</span> <span class="ident">x</span> <span class="op">=</span> <span class="number">1</span>;</span>
+<span class="kw">let</span> <span class="ident">y</span> <span class="op">=</span> <span class="number">2</span>;

src/librustdoc/html/highlight/tests.rs

@@ -1,6 +1,7 @@
-use super::write_code;
+use super::{write_code, DecorationInfo};
 use crate::html::format::Buffer;
 use expect_test::expect_file;
+use rustc_data_structures::fx::FxHashMap;
 use rustc_span::create_default_session_globals_then;
 use rustc_span::edition::Edition;
 
@@ -22,7 +23,7 @@ fn test_html_highlighting() {
         let src = include_str!("fixtures/sample.rs");
         let html = {
             let mut out = Buffer::new();
-            write_code(&mut out, src, Edition::Edition2018, None);
+            write_code(&mut out, src, Edition::Edition2018, None, None);
             format!("{}<pre><code>{}</code></pre>\n", STYLE, out.into_inner())
         };
         expect_file!["fixtures/sample.html"].assert_eq(&html);
@@ -36,7 +37,7 @@ fn test_dos_backline() {
     println!(\"foo\");\r\n\
 }\r\n";
         let mut html = Buffer::new();
-        write_code(&mut html, src, Edition::Edition2018, None);
+        write_code(&mut html, src, Edition::Edition2018, None, None);
         expect_file!["fixtures/dos_line.html"].assert_eq(&html.into_inner());
     });
 }
@@ -50,7 +51,7 @@ let x = super::b::foo;
 let y = Self::whatever;";
 
         let mut html = Buffer::new();
-        write_code(&mut html, src, Edition::Edition2018, None);
+        write_code(&mut html, src, Edition::Edition2018, None, None);
         expect_file!["fixtures/highlight.html"].assert_eq(&html.into_inner());
     });
 }
@@ -60,7 +61,21 @@ fn test_union_highlighting() {
     create_default_session_globals_then(|| {
         let src = include_str!("fixtures/union.rs");
         let mut html = Buffer::new();
-        write_code(&mut html, src, Edition::Edition2018, None);
+        write_code(&mut html, src, Edition::Edition2018, None, None);
         expect_file!["fixtures/union.html"].assert_eq(&html.into_inner());
     });
 }
+
+#[test]
+fn test_decorations() {
+    create_default_session_globals_then(|| {
+        let src = "let x = 1;
+let y = 2;";
+        let mut decorations = FxHashMap::default();
+        decorations.insert("example", vec![(0, 10)]);
+
+        let mut html = Buffer::new();
+        write_code(&mut html, src, Edition::Edition2018, None, Some(DecorationInfo(decorations)));
+        expect_file!["fixtures/decorations.html"].assert_eq(&html.into_inner());
+    });
+}

src/librustdoc/html/layout.rs

@@ -22,6 +22,8 @@ crate struct Layout {
     /// If false, the `select` element to have search filtering by crates on rendered docs
     /// won't be generated.
     crate generate_search_filter: bool,
+    /// If true, then scrape-examples.js will be included in the output HTML file
+    crate scrape_examples_extension: bool,
 }
 
 #[derive(Serialize)]

src/librustdoc/html/markdown.rs

@@ -360,6 +360,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
             edition,
             None,
             None,
+            None,
         );
         Some(Event::Html(s.into_inner().into()))
     }