• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //! Markdown formatting for rustdoc.
2 //!
3 //! This module implements markdown formatting through the pulldown-cmark library.
4 //!
5 //! ```
6 //! #![feature(rustc_private)]
7 //!
8 //! extern crate rustc_span;
9 //!
10 //! use rustc_span::edition::Edition;
11 //! use rustdoc::html::markdown::{HeadingOffset, IdMap, Markdown, ErrorCodes};
12 //!
13 //! let s = "My *markdown* _text_";
14 //! let mut id_map = IdMap::new();
15 //! let md = Markdown {
16 //!     content: s,
17 //!     links: &[],
18 //!     ids: &mut id_map,
19 //!     error_codes: ErrorCodes::Yes,
20 //!     edition: Edition::Edition2015,
21 //!     playground: &None,
22 //!     heading_offset: HeadingOffset::H2,
23 //! };
24 //! let html = md.into_string();
25 //! // ... something using html
26 //! ```
27 
28 use rustc_data_structures::fx::FxHashMap;
29 use rustc_hir::def_id::DefId;
30 use rustc_middle::ty::TyCtxt;
31 pub(crate) use rustc_resolve::rustdoc::main_body_opts;
32 use rustc_resolve::rustdoc::may_be_doc_link;
33 use rustc_span::edition::Edition;
34 use rustc_span::{Span, Symbol};
35 
36 use once_cell::sync::Lazy;
37 use std::borrow::Cow;
38 use std::collections::VecDeque;
39 use std::fmt::Write;
40 use std::ops::{ControlFlow, Range};
41 use std::str;
42 
43 use crate::clean::RenderedLink;
44 use crate::doctest;
45 use crate::html::escape::Escape;
46 use crate::html::format::Buffer;
47 use crate::html::highlight;
48 use crate::html::length_limit::HtmlWithLimit;
49 use crate::html::render::small_url_encode;
50 use crate::html::toc::TocBuilder;
51 
52 use pulldown_cmark::{
53     html, BrokenLink, CodeBlockKind, CowStr, Event, LinkType, Options, Parser, Tag,
54 };
55 
56 #[cfg(test)]
57 mod tests;
58 
59 const MAX_HEADER_LEVEL: u32 = 6;
60 
61 /// Options for rendering Markdown in summaries (e.g., in search results).
summary_opts() -> Options62 pub(crate) fn summary_opts() -> Options {
63     Options::ENABLE_TABLES
64         | Options::ENABLE_FOOTNOTES
65         | Options::ENABLE_STRIKETHROUGH
66         | Options::ENABLE_TASKLISTS
67         | Options::ENABLE_SMART_PUNCTUATION
68 }
69 
70 #[derive(Debug, Clone, Copy)]
71 pub enum HeadingOffset {
72     H1 = 0,
73     H2,
74     H3,
75     H4,
76     H5,
77     H6,
78 }
79 
80 /// When `to_string` is called, this struct will emit the HTML corresponding to
81 /// the rendered version of the contained markdown string.
82 pub struct Markdown<'a> {
83     pub content: &'a str,
84     /// A list of link replacements.
85     pub links: &'a [RenderedLink],
86     /// The current list of used header IDs.
87     pub ids: &'a mut IdMap,
88     /// Whether to allow the use of explicit error codes in doctest lang strings.
89     pub error_codes: ErrorCodes,
90     /// Default edition to use when parsing doctests (to add a `fn main`).
91     pub edition: Edition,
92     pub playground: &'a Option<Playground>,
93     /// Offset at which we render headings.
94     /// E.g. if `heading_offset: HeadingOffset::H2`, then `# something` renders an `<h2>`.
95     pub heading_offset: HeadingOffset,
96 }
97 /// A struct like `Markdown` that renders the markdown with a table of contents.
98 pub(crate) struct MarkdownWithToc<'a> {
99     pub(crate) content: &'a str,
100     pub(crate) ids: &'a mut IdMap,
101     pub(crate) error_codes: ErrorCodes,
102     pub(crate) edition: Edition,
103     pub(crate) playground: &'a Option<Playground>,
104 }
105 /// A tuple struct like `Markdown` that renders the markdown escaping HTML tags
106 /// and includes no paragraph tags.
107 pub(crate) struct MarkdownItemInfo<'a>(pub(crate) &'a str, pub(crate) &'a mut IdMap);
108 /// A tuple struct like `Markdown` that renders only the first paragraph.
109 pub(crate) struct MarkdownSummaryLine<'a>(pub &'a str, pub &'a [RenderedLink]);
110 
111 #[derive(Copy, Clone, PartialEq, Debug)]
112 pub enum ErrorCodes {
113     Yes,
114     No,
115 }
116 
117 impl ErrorCodes {
from(b: bool) -> Self118     pub(crate) fn from(b: bool) -> Self {
119         match b {
120             true => ErrorCodes::Yes,
121             false => ErrorCodes::No,
122         }
123     }
124 
as_bool(self) -> bool125     pub(crate) fn as_bool(self) -> bool {
126         match self {
127             ErrorCodes::Yes => true,
128             ErrorCodes::No => false,
129         }
130     }
131 }
132 
133 /// Controls whether a line will be hidden or shown in HTML output.
134 ///
135 /// All lines are used in documentation tests.
136 enum Line<'a> {
137     Hidden(&'a str),
138     Shown(Cow<'a, str>),
139 }
140 
141 impl<'a> Line<'a> {
for_html(self) -> Option<Cow<'a, str>>142     fn for_html(self) -> Option<Cow<'a, str>> {
143         match self {
144             Line::Shown(l) => Some(l),
145             Line::Hidden(_) => None,
146         }
147     }
148 
for_code(self) -> Cow<'a, str>149     fn for_code(self) -> Cow<'a, str> {
150         match self {
151             Line::Shown(l) => l,
152             Line::Hidden(l) => Cow::Borrowed(l),
153         }
154     }
155 }
156 
157 // FIXME: There is a minor inconsistency here. For lines that start with ##, we
158 // have no easy way of removing a potential single space after the hashes, which
159 // is done in the single # case. This inconsistency seems okay, if non-ideal. In
160 // order to fix it we'd have to iterate to find the first non-# character, and
161 // then reallocate to remove it; which would make us return a String.
map_line(s: &str) -> Line<'_>162 fn map_line(s: &str) -> Line<'_> {
163     let trimmed = s.trim();
164     if trimmed.starts_with("##") {
165         Line::Shown(Cow::Owned(s.replacen("##", "#", 1)))
166     } else if let Some(stripped) = trimmed.strip_prefix("# ") {
167         // # text
168         Line::Hidden(stripped)
169     } else if trimmed == "#" {
170         // We cannot handle '#text' because it could be #[attr].
171         Line::Hidden("")
172     } else {
173         Line::Shown(Cow::Borrowed(s))
174     }
175 }
176 
177 /// Convert chars from a title for an id.
178 ///
179 /// "Hello, world!" -> "hello-world"
slugify(c: char) -> Option<char>180 fn slugify(c: char) -> Option<char> {
181     if c.is_alphanumeric() || c == '-' || c == '_' {
182         if c.is_ascii() { Some(c.to_ascii_lowercase()) } else { Some(c) }
183     } else if c.is_whitespace() && c.is_ascii() {
184         Some('-')
185     } else {
186         None
187     }
188 }
189 
190 #[derive(Clone, Debug)]
191 pub struct Playground {
192     pub crate_name: Option<Symbol>,
193     pub url: String,
194 }
195 
196 /// Adds syntax highlighting and playground Run buttons to Rust code blocks.
197 struct CodeBlocks<'p, 'a, I: Iterator<Item = Event<'a>>> {
198     inner: I,
199     check_error_codes: ErrorCodes,
200     edition: Edition,
201     // Information about the playground if a URL has been specified, containing an
202     // optional crate name and the URL.
203     playground: &'p Option<Playground>,
204 }
205 
206 impl<'p, 'a, I: Iterator<Item = Event<'a>>> CodeBlocks<'p, 'a, I> {
new( iter: I, error_codes: ErrorCodes, edition: Edition, playground: &'p Option<Playground>, ) -> Self207     fn new(
208         iter: I,
209         error_codes: ErrorCodes,
210         edition: Edition,
211         playground: &'p Option<Playground>,
212     ) -> Self {
213         CodeBlocks { inner: iter, check_error_codes: error_codes, edition, playground }
214     }
215 }
216 
217 impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
218     type Item = Event<'a>;
219 
next(&mut self) -> Option<Self::Item>220     fn next(&mut self) -> Option<Self::Item> {
221         let event = self.inner.next();
222         let compile_fail;
223         let should_panic;
224         let ignore;
225         let edition;
226         let Some(Event::Start(Tag::CodeBlock(kind))) = event else {
227             return event;
228         };
229 
230         let mut original_text = String::new();
231         for event in &mut self.inner {
232             match event {
233                 Event::End(Tag::CodeBlock(..)) => break,
234                 Event::Text(ref s) => {
235                     original_text.push_str(s);
236                 }
237                 _ => {}
238             }
239         }
240 
241         let parse_result = match kind {
242             CodeBlockKind::Fenced(ref lang) => {
243                 let parse_result =
244                     LangString::parse_without_check(lang, self.check_error_codes, false);
245                 if !parse_result.rust {
246                     return Some(Event::Html(
247                         format!(
248                             "<div class=\"example-wrap\">\
249                                  <pre class=\"language-{}\"><code>{}</code></pre>\
250                              </div>",
251                             lang,
252                             Escape(&original_text),
253                         )
254                         .into(),
255                     ));
256                 }
257                 parse_result
258             }
259             CodeBlockKind::Indented => Default::default(),
260         };
261 
262         let lines = original_text.lines().filter_map(|l| map_line(l).for_html());
263         let text = lines.intersperse("\n".into()).collect::<String>();
264 
265         compile_fail = parse_result.compile_fail;
266         should_panic = parse_result.should_panic;
267         ignore = parse_result.ignore;
268         edition = parse_result.edition;
269 
270         let explicit_edition = edition.is_some();
271         let edition = edition.unwrap_or(self.edition);
272 
273         let playground_button = self.playground.as_ref().and_then(|playground| {
274             let krate = &playground.crate_name;
275             let url = &playground.url;
276             if url.is_empty() {
277                 return None;
278             }
279             let test = original_text
280                 .lines()
281                 .map(|l| map_line(l).for_code())
282                 .intersperse("\n".into())
283                 .collect::<String>();
284             let krate = krate.as_ref().map(|s| s.as_str());
285             let (test, _, _) =
286                 doctest::make_test(&test, krate, false, &Default::default(), edition, None);
287             let channel = if test.contains("#![feature(") { "&amp;version=nightly" } else { "" };
288 
289             let test_escaped = small_url_encode(test);
290             Some(format!(
291                 r#"<a class="test-arrow" target="_blank" href="{}?code={}{}&amp;edition={}">Run</a>"#,
292                 url, test_escaped, channel, edition,
293             ))
294         });
295 
296         let tooltip = if ignore != Ignore::None {
297             highlight::Tooltip::Ignore
298         } else if compile_fail {
299             highlight::Tooltip::CompileFail
300         } else if should_panic {
301             highlight::Tooltip::ShouldPanic
302         } else if explicit_edition {
303             highlight::Tooltip::Edition(edition)
304         } else {
305             highlight::Tooltip::None
306         };
307 
308         // insert newline to clearly separate it from the
309         // previous block so we can shorten the html output
310         let mut s = Buffer::new();
311         s.push_str("\n");
312 
313         highlight::render_example_with_highlighting(
314             &text,
315             &mut s,
316             tooltip,
317             playground_button.as_deref(),
318         );
319         Some(Event::Html(s.into_inner().into()))
320     }
321 }
322 
323 /// Make headings links with anchor IDs and build up TOC.
324 struct LinkReplacer<'a, I: Iterator<Item = Event<'a>>> {
325     inner: I,
326     links: &'a [RenderedLink],
327     shortcut_link: Option<&'a RenderedLink>,
328 }
329 
330 impl<'a, I: Iterator<Item = Event<'a>>> LinkReplacer<'a, I> {
new(iter: I, links: &'a [RenderedLink]) -> Self331     fn new(iter: I, links: &'a [RenderedLink]) -> Self {
332         LinkReplacer { inner: iter, links, shortcut_link: None }
333     }
334 }
335 
336 impl<'a, I: Iterator<Item = Event<'a>>> Iterator for LinkReplacer<'a, I> {
337     type Item = Event<'a>;
338 
next(&mut self) -> Option<Self::Item>339     fn next(&mut self) -> Option<Self::Item> {
340         let mut event = self.inner.next();
341 
342         // Replace intra-doc links and remove disambiguators from shortcut links (`[fn@f]`).
343         match &mut event {
344             // This is a shortcut link that was resolved by the broken_link_callback: `[fn@f]`
345             // Remove any disambiguator.
346             Some(Event::Start(Tag::Link(
347                 // [fn@f] or [fn@f][]
348                 LinkType::ShortcutUnknown | LinkType::CollapsedUnknown,
349                 dest,
350                 title,
351             ))) => {
352                 debug!("saw start of shortcut link to {} with title {}", dest, title);
353                 // If this is a shortcut link, it was resolved by the broken_link_callback.
354                 // So the URL will already be updated properly.
355                 let link = self.links.iter().find(|&link| *link.href == **dest);
356                 // Since this is an external iterator, we can't replace the inner text just yet.
357                 // Store that we saw a link so we know to replace it later.
358                 if let Some(link) = link {
359                     trace!("it matched");
360                     assert!(self.shortcut_link.is_none(), "shortcut links cannot be nested");
361                     self.shortcut_link = Some(link);
362                     if title.is_empty() && !link.tooltip.is_empty() {
363                         *title = CowStr::Borrowed(link.tooltip.as_ref());
364                     }
365                 }
366             }
367             // Now that we're done with the shortcut link, don't replace any more text.
368             Some(Event::End(Tag::Link(
369                 LinkType::ShortcutUnknown | LinkType::CollapsedUnknown,
370                 dest,
371                 _,
372             ))) => {
373                 debug!("saw end of shortcut link to {}", dest);
374                 if self.links.iter().any(|link| *link.href == **dest) {
375                     assert!(self.shortcut_link.is_some(), "saw closing link without opening tag");
376                     self.shortcut_link = None;
377                 }
378             }
379             // Handle backticks in inline code blocks, but only if we're in the middle of a shortcut link.
380             // [`fn@f`]
381             Some(Event::Code(text)) => {
382                 trace!("saw code {}", text);
383                 if let Some(link) = self.shortcut_link {
384                     // NOTE: this only replaces if the code block is the *entire* text.
385                     // If only part of the link has code highlighting, the disambiguator will not be removed.
386                     // e.g. [fn@`f`]
387                     // This is a limitation from `collect_intra_doc_links`: it passes a full link,
388                     // and does not distinguish at all between code blocks.
389                     // So we could never be sure we weren't replacing too much:
390                     // [fn@my_`f`unc] is treated the same as [my_func()] in that pass.
391                     //
392                     // NOTE: .get(1..len() - 1) is to strip the backticks
393                     if let Some(link) = self.links.iter().find(|l| {
394                         l.href == link.href
395                             && Some(&**text) == l.original_text.get(1..l.original_text.len() - 1)
396                     }) {
397                         debug!("replacing {} with {}", text, link.new_text);
398                         *text = CowStr::Borrowed(&link.new_text);
399                     }
400                 }
401             }
402             // Replace plain text in links, but only in the middle of a shortcut link.
403             // [fn@f]
404             Some(Event::Text(text)) => {
405                 trace!("saw text {}", text);
406                 if let Some(link) = self.shortcut_link {
407                     // NOTE: same limitations as `Event::Code`
408                     if let Some(link) = self
409                         .links
410                         .iter()
411                         .find(|l| l.href == link.href && **text == *l.original_text)
412                     {
413                         debug!("replacing {} with {}", text, link.new_text);
414                         *text = CowStr::Borrowed(&link.new_text);
415                     }
416                 }
417             }
418             // If this is a link, but not a shortcut link,
419             // replace the URL, since the broken_link_callback was not called.
420             Some(Event::Start(Tag::Link(_, dest, title))) => {
421                 if let Some(link) = self.links.iter().find(|&link| *link.original_text == **dest) {
422                     *dest = CowStr::Borrowed(link.href.as_ref());
423                     if title.is_empty() && !link.tooltip.is_empty() {
424                         *title = CowStr::Borrowed(link.tooltip.as_ref());
425                     }
426                 }
427             }
428             // Anything else couldn't have been a valid Rust path, so no need to replace the text.
429             _ => {}
430         }
431 
432         // Yield the modified event
433         event
434     }
435 }
436 
437 /// Wrap HTML tables into `<div>` to prevent having the doc blocks width being too big.
438 struct TableWrapper<'a, I: Iterator<Item = Event<'a>>> {
439     inner: I,
440     stored_events: VecDeque<Event<'a>>,
441 }
442 
443 impl<'a, I: Iterator<Item = Event<'a>>> TableWrapper<'a, I> {
new(iter: I) -> Self444     fn new(iter: I) -> Self {
445         Self { inner: iter, stored_events: VecDeque::new() }
446     }
447 }
448 
449 impl<'a, I: Iterator<Item = Event<'a>>> Iterator for TableWrapper<'a, I> {
450     type Item = Event<'a>;
451 
next(&mut self) -> Option<Self::Item>452     fn next(&mut self) -> Option<Self::Item> {
453         if let Some(first) = self.stored_events.pop_front() {
454             return Some(first);
455         }
456 
457         let event = self.inner.next()?;
458 
459         Some(match event {
460             Event::Start(Tag::Table(t)) => {
461                 self.stored_events.push_back(Event::Start(Tag::Table(t)));
462                 Event::Html(CowStr::Borrowed("<div>"))
463             }
464             Event::End(Tag::Table(t)) => {
465                 self.stored_events.push_back(Event::Html(CowStr::Borrowed("</div>")));
466                 Event::End(Tag::Table(t))
467             }
468             e => e,
469         })
470     }
471 }
472 
473 type SpannedEvent<'a> = (Event<'a>, Range<usize>);
474 
475 /// Make headings links with anchor IDs and build up TOC.
476 struct HeadingLinks<'a, 'b, 'ids, I> {
477     inner: I,
478     toc: Option<&'b mut TocBuilder>,
479     buf: VecDeque<SpannedEvent<'a>>,
480     id_map: &'ids mut IdMap,
481     heading_offset: HeadingOffset,
482 }
483 
484 impl<'a, 'b, 'ids, I> HeadingLinks<'a, 'b, 'ids, I> {
new( iter: I, toc: Option<&'b mut TocBuilder>, ids: &'ids mut IdMap, heading_offset: HeadingOffset, ) -> Self485     fn new(
486         iter: I,
487         toc: Option<&'b mut TocBuilder>,
488         ids: &'ids mut IdMap,
489         heading_offset: HeadingOffset,
490     ) -> Self {
491         HeadingLinks { inner: iter, toc, buf: VecDeque::new(), id_map: ids, heading_offset }
492     }
493 }
494 
495 impl<'a, 'b, 'ids, I: Iterator<Item = SpannedEvent<'a>>> Iterator
496     for HeadingLinks<'a, 'b, 'ids, I>
497 {
498     type Item = SpannedEvent<'a>;
499 
next(&mut self) -> Option<Self::Item>500     fn next(&mut self) -> Option<Self::Item> {
501         if let Some(e) = self.buf.pop_front() {
502             return Some(e);
503         }
504 
505         let event = self.inner.next();
506         if let Some((Event::Start(Tag::Heading(level, _, _)), _)) = event {
507             let mut id = String::new();
508             for event in &mut self.inner {
509                 match &event.0 {
510                     Event::End(Tag::Heading(..)) => break,
511                     Event::Start(Tag::Link(_, _, _)) | Event::End(Tag::Link(..)) => {}
512                     Event::Text(text) | Event::Code(text) => {
513                         id.extend(text.chars().filter_map(slugify));
514                         self.buf.push_back(event);
515                     }
516                     _ => self.buf.push_back(event),
517                 }
518             }
519             let id = self.id_map.derive(id);
520 
521             if let Some(ref mut builder) = self.toc {
522                 let mut html_header = String::new();
523                 html::push_html(&mut html_header, self.buf.iter().map(|(ev, _)| ev.clone()));
524                 let sec = builder.push(level as u32, html_header, id.clone());
525                 self.buf.push_front((Event::Html(format!("{} ", sec).into()), 0..0));
526             }
527 
528             let level =
529                 std::cmp::min(level as u32 + (self.heading_offset as u32), MAX_HEADER_LEVEL);
530             self.buf.push_back((Event::Html(format!("</a></h{}>", level).into()), 0..0));
531 
532             let start_tags = format!(
533                 "<h{level} id=\"{id}\">\
534                     <a href=\"#{id}\">",
535                 id = id,
536                 level = level
537             );
538             return Some((Event::Html(start_tags.into()), 0..0));
539         }
540         event
541     }
542 }
543 
544 /// Extracts just the first paragraph.
545 struct SummaryLine<'a, I: Iterator<Item = Event<'a>>> {
546     inner: I,
547     started: bool,
548     depth: u32,
549     skipped_tags: u32,
550 }
551 
552 impl<'a, I: Iterator<Item = Event<'a>>> SummaryLine<'a, I> {
new(iter: I) -> Self553     fn new(iter: I) -> Self {
554         SummaryLine { inner: iter, started: false, depth: 0, skipped_tags: 0 }
555     }
556 }
557 
check_if_allowed_tag(t: &Tag<'_>) -> bool558 fn check_if_allowed_tag(t: &Tag<'_>) -> bool {
559     matches!(
560         t,
561         Tag::Paragraph
562             | Tag::Emphasis
563             | Tag::Strong
564             | Tag::Strikethrough
565             | Tag::Link(..)
566             | Tag::BlockQuote
567     )
568 }
569 
is_forbidden_tag(t: &Tag<'_>) -> bool570 fn is_forbidden_tag(t: &Tag<'_>) -> bool {
571     matches!(
572         t,
573         Tag::CodeBlock(_)
574             | Tag::Table(_)
575             | Tag::TableHead
576             | Tag::TableRow
577             | Tag::TableCell
578             | Tag::FootnoteDefinition(_)
579     )
580 }
581 
582 impl<'a, I: Iterator<Item = Event<'a>>> Iterator for SummaryLine<'a, I> {
583     type Item = Event<'a>;
584 
next(&mut self) -> Option<Self::Item>585     fn next(&mut self) -> Option<Self::Item> {
586         if self.started && self.depth == 0 {
587             return None;
588         }
589         if !self.started {
590             self.started = true;
591         }
592         if let Some(event) = self.inner.next() {
593             let mut is_start = true;
594             let is_allowed_tag = match event {
595                 Event::Start(ref c) => {
596                     if is_forbidden_tag(c) {
597                         self.skipped_tags += 1;
598                         return None;
599                     }
600                     self.depth += 1;
601                     check_if_allowed_tag(c)
602                 }
603                 Event::End(ref c) => {
604                     if is_forbidden_tag(c) {
605                         self.skipped_tags += 1;
606                         return None;
607                     }
608                     self.depth -= 1;
609                     is_start = false;
610                     check_if_allowed_tag(c)
611                 }
612                 Event::FootnoteReference(_) => {
613                     self.skipped_tags += 1;
614                     false
615                 }
616                 _ => true,
617             };
618             if !is_allowed_tag {
619                 self.skipped_tags += 1;
620             }
621             return if !is_allowed_tag {
622                 if is_start {
623                     Some(Event::Start(Tag::Paragraph))
624                 } else {
625                     Some(Event::End(Tag::Paragraph))
626                 }
627             } else {
628                 Some(event)
629             };
630         }
631         None
632     }
633 }
634 
635 /// Moves all footnote definitions to the end and add back links to the
636 /// references.
637 struct Footnotes<'a, I> {
638     inner: I,
639     footnotes: FxHashMap<String, (Vec<Event<'a>>, u16)>,
640 }
641 
642 impl<'a, I> Footnotes<'a, I> {
new(iter: I) -> Self643     fn new(iter: I) -> Self {
644         Footnotes { inner: iter, footnotes: FxHashMap::default() }
645     }
646 
get_entry(&mut self, key: &str) -> &mut (Vec<Event<'a>>, u16)647     fn get_entry(&mut self, key: &str) -> &mut (Vec<Event<'a>>, u16) {
648         let new_id = self.footnotes.len() + 1;
649         let key = key.to_owned();
650         self.footnotes.entry(key).or_insert((Vec::new(), new_id as u16))
651     }
652 }
653 
654 impl<'a, I: Iterator<Item = SpannedEvent<'a>>> Iterator for Footnotes<'a, I> {
655     type Item = SpannedEvent<'a>;
656 
next(&mut self) -> Option<Self::Item>657     fn next(&mut self) -> Option<Self::Item> {
658         loop {
659             match self.inner.next() {
660                 Some((Event::FootnoteReference(ref reference), range)) => {
661                     let entry = self.get_entry(reference);
662                     let reference = format!(
663                         "<sup id=\"fnref{0}\"><a href=\"#fn{0}\">{0}</a></sup>",
664                         (*entry).1
665                     );
666                     return Some((Event::Html(reference.into()), range));
667                 }
668                 Some((Event::Start(Tag::FootnoteDefinition(def)), _)) => {
669                     let mut content = Vec::new();
670                     for (event, _) in &mut self.inner {
671                         if let Event::End(Tag::FootnoteDefinition(..)) = event {
672                             break;
673                         }
674                         content.push(event);
675                     }
676                     let entry = self.get_entry(&def);
677                     (*entry).0 = content;
678                 }
679                 Some(e) => return Some(e),
680                 None => {
681                     if !self.footnotes.is_empty() {
682                         let mut v: Vec<_> = self.footnotes.drain().map(|(_, x)| x).collect();
683                         v.sort_by(|a, b| a.1.cmp(&b.1));
684                         let mut ret = String::from("<div class=\"footnotes\"><hr><ol>");
685                         for (mut content, id) in v {
686                             write!(ret, "<li id=\"fn{}\">", id).unwrap();
687                             let mut is_paragraph = false;
688                             if let Some(&Event::End(Tag::Paragraph)) = content.last() {
689                                 content.pop();
690                                 is_paragraph = true;
691                             }
692                             html::push_html(&mut ret, content.into_iter());
693                             write!(ret, "&nbsp;<a href=\"#fnref{}\">↩</a>", id).unwrap();
694                             if is_paragraph {
695                                 ret.push_str("</p>");
696                             }
697                             ret.push_str("</li>");
698                         }
699                         ret.push_str("</ol></div>");
700                         return Some((Event::Html(ret.into()), 0..0));
701                     } else {
702                         return None;
703                     }
704                 }
705             }
706         }
707     }
708 }
709 
find_testable_code<T: doctest::Tester>( doc: &str, tests: &mut T, error_codes: ErrorCodes, enable_per_target_ignores: bool, extra_info: Option<&ExtraInfo<'_>>, )710 pub(crate) fn find_testable_code<T: doctest::Tester>(
711     doc: &str,
712     tests: &mut T,
713     error_codes: ErrorCodes,
714     enable_per_target_ignores: bool,
715     extra_info: Option<&ExtraInfo<'_>>,
716 ) {
717     let mut parser = Parser::new(doc).into_offset_iter();
718     let mut prev_offset = 0;
719     let mut nb_lines = 0;
720     let mut register_header = None;
721     while let Some((event, offset)) = parser.next() {
722         match event {
723             Event::Start(Tag::CodeBlock(kind)) => {
724                 let block_info = match kind {
725                     CodeBlockKind::Fenced(ref lang) => {
726                         if lang.is_empty() {
727                             Default::default()
728                         } else {
729                             LangString::parse(
730                                 lang,
731                                 error_codes,
732                                 enable_per_target_ignores,
733                                 extra_info,
734                             )
735                         }
736                     }
737                     CodeBlockKind::Indented => Default::default(),
738                 };
739                 if !block_info.rust {
740                     continue;
741                 }
742 
743                 let mut test_s = String::new();
744 
745                 while let Some((Event::Text(s), _)) = parser.next() {
746                     test_s.push_str(&s);
747                 }
748                 let text = test_s
749                     .lines()
750                     .map(|l| map_line(l).for_code())
751                     .collect::<Vec<Cow<'_, str>>>()
752                     .join("\n");
753 
754                 nb_lines += doc[prev_offset..offset.start].lines().count();
755                 // If there are characters between the preceding line ending and
756                 // this code block, `str::lines` will return an additional line,
757                 // which we subtract here.
758                 if nb_lines != 0 && !&doc[prev_offset..offset.start].ends_with('\n') {
759                     nb_lines -= 1;
760                 }
761                 let line = tests.get_line() + nb_lines + 1;
762                 tests.add_test(text, block_info, line);
763                 prev_offset = offset.start;
764             }
765             Event::Start(Tag::Heading(level, _, _)) => {
766                 register_header = Some(level as u32);
767             }
768             Event::Text(ref s) if register_header.is_some() => {
769                 let level = register_header.unwrap();
770                 tests.register_header(s, level);
771                 register_header = None;
772             }
773             _ => {}
774         }
775     }
776 }
777 
778 pub(crate) struct ExtraInfo<'tcx> {
779     def_id: DefId,
780     sp: Span,
781     tcx: TyCtxt<'tcx>,
782 }
783 
784 impl<'tcx> ExtraInfo<'tcx> {
new(tcx: TyCtxt<'tcx>, def_id: DefId, sp: Span) -> ExtraInfo<'tcx>785     pub(crate) fn new(tcx: TyCtxt<'tcx>, def_id: DefId, sp: Span) -> ExtraInfo<'tcx> {
786         ExtraInfo { def_id, sp, tcx }
787     }
788 
error_invalid_codeblock_attr(&self, msg: String, help: &'static str)789     fn error_invalid_codeblock_attr(&self, msg: String, help: &'static str) {
790         if let Some(def_id) = self.def_id.as_local() {
791             self.tcx.struct_span_lint_hir(
792                 crate::lint::INVALID_CODEBLOCK_ATTRIBUTES,
793                 self.tcx.hir().local_def_id_to_hir_id(def_id),
794                 self.sp,
795                 msg,
796                 |lint| lint.help(help),
797             );
798         }
799     }
800 }
801 
802 #[derive(Eq, PartialEq, Clone, Debug)]
803 pub(crate) struct LangString {
804     original: String,
805     pub(crate) should_panic: bool,
806     pub(crate) no_run: bool,
807     pub(crate) ignore: Ignore,
808     pub(crate) rust: bool,
809     pub(crate) test_harness: bool,
810     pub(crate) compile_fail: bool,
811     pub(crate) error_codes: Vec<String>,
812     pub(crate) edition: Option<Edition>,
813 }
814 
815 #[derive(Eq, PartialEq, Clone, Debug)]
816 pub(crate) enum Ignore {
817     All,
818     None,
819     Some(Vec<String>),
820 }
821 
822 impl Default for LangString {
default() -> Self823     fn default() -> Self {
824         Self {
825             original: String::new(),
826             should_panic: false,
827             no_run: false,
828             ignore: Ignore::None,
829             rust: true,
830             test_harness: false,
831             compile_fail: false,
832             error_codes: Vec::new(),
833             edition: None,
834         }
835     }
836 }
837 
838 impl LangString {
parse_without_check( string: &str, allow_error_code_check: ErrorCodes, enable_per_target_ignores: bool, ) -> LangString839     fn parse_without_check(
840         string: &str,
841         allow_error_code_check: ErrorCodes,
842         enable_per_target_ignores: bool,
843     ) -> LangString {
844         Self::parse(string, allow_error_code_check, enable_per_target_ignores, None)
845     }
846 
tokens(string: &str) -> impl Iterator<Item = &str>847     fn tokens(string: &str) -> impl Iterator<Item = &str> {
848         // Pandoc, which Rust once used for generating documentation,
849         // expects lang strings to be surrounded by `{}` and for each token
850         // to be proceeded by a `.`. Since some of these lang strings are still
851         // loose in the wild, we strip a pair of surrounding `{}` from the lang
852         // string and a leading `.` from each token.
853 
854         let string = string.trim();
855 
856         let first = string.chars().next();
857         let last = string.chars().last();
858 
859         let string = if first == Some('{') && last == Some('}') {
860             &string[1..string.len() - 1]
861         } else {
862             string
863         };
864 
865         string
866             .split(|c| c == ',' || c == ' ' || c == '\t')
867             .map(str::trim)
868             .map(|token| token.strip_prefix('.').unwrap_or(token))
869             .filter(|token| !token.is_empty())
870     }
871 
parse( string: &str, allow_error_code_check: ErrorCodes, enable_per_target_ignores: bool, extra: Option<&ExtraInfo<'_>>, ) -> LangString872     fn parse(
873         string: &str,
874         allow_error_code_check: ErrorCodes,
875         enable_per_target_ignores: bool,
876         extra: Option<&ExtraInfo<'_>>,
877     ) -> LangString {
878         let allow_error_code_check = allow_error_code_check.as_bool();
879         let mut seen_rust_tags = false;
880         let mut seen_other_tags = false;
881         let mut data = LangString::default();
882         let mut ignores = vec![];
883 
884         data.original = string.to_owned();
885 
886         for token in Self::tokens(string) {
887             match token {
888                 "should_panic" => {
889                     data.should_panic = true;
890                     seen_rust_tags = !seen_other_tags;
891                 }
892                 "no_run" => {
893                     data.no_run = true;
894                     seen_rust_tags = !seen_other_tags;
895                 }
896                 "ignore" => {
897                     data.ignore = Ignore::All;
898                     seen_rust_tags = !seen_other_tags;
899                 }
900                 x if x.starts_with("ignore-") => {
901                     if enable_per_target_ignores {
902                         ignores.push(x.trim_start_matches("ignore-").to_owned());
903                         seen_rust_tags = !seen_other_tags;
904                     }
905                 }
906                 "rust" => {
907                     data.rust = true;
908                     seen_rust_tags = true;
909                 }
910                 "test_harness" => {
911                     data.test_harness = true;
912                     seen_rust_tags = !seen_other_tags || seen_rust_tags;
913                 }
914                 "compile_fail" => {
915                     data.compile_fail = true;
916                     seen_rust_tags = !seen_other_tags || seen_rust_tags;
917                     data.no_run = true;
918                 }
919                 x if x.starts_with("edition") => {
920                     data.edition = x[7..].parse::<Edition>().ok();
921                 }
922                 x if allow_error_code_check && x.starts_with('E') && x.len() == 5 => {
923                     if x[1..].parse::<u32>().is_ok() {
924                         data.error_codes.push(x.to_owned());
925                         seen_rust_tags = !seen_other_tags || seen_rust_tags;
926                     } else {
927                         seen_other_tags = true;
928                     }
929                 }
930                 x if extra.is_some() => {
931                     let s = x.to_lowercase();
932                     if let Some((flag, help)) = if s == "compile-fail"
933                         || s == "compile_fail"
934                         || s == "compilefail"
935                     {
936                         Some((
937                             "compile_fail",
938                             "the code block will either not be tested if not marked as a rust one \
939                              or won't fail if it compiles successfully",
940                         ))
941                     } else if s == "should-panic" || s == "should_panic" || s == "shouldpanic" {
942                         Some((
943                             "should_panic",
944                             "the code block will either not be tested if not marked as a rust one \
945                              or won't fail if it doesn't panic when running",
946                         ))
947                     } else if s == "no-run" || s == "no_run" || s == "norun" {
948                         Some((
949                             "no_run",
950                             "the code block will either not be tested if not marked as a rust one \
951                              or will be run (which you might not want)",
952                         ))
953                     } else if s == "test-harness" || s == "test_harness" || s == "testharness" {
954                         Some((
955                             "test_harness",
956                             "the code block will either not be tested if not marked as a rust one \
957                              or the code will be wrapped inside a main function",
958                         ))
959                     } else {
960                         None
961                     } {
962                         if let Some(extra) = extra {
963                             extra.error_invalid_codeblock_attr(
964                                 format!("unknown attribute `{}`. Did you mean `{}`?", x, flag),
965                                 help,
966                             );
967                         }
968                     }
969                     seen_other_tags = true;
970                 }
971                 _ => seen_other_tags = true,
972             }
973         }
974 
975         // ignore-foo overrides ignore
976         if !ignores.is_empty() {
977             data.ignore = Ignore::Some(ignores);
978         }
979 
980         data.rust &= !seen_other_tags || seen_rust_tags;
981 
982         data
983     }
984 }
985 
986 impl Markdown<'_> {
into_string(self) -> String987     pub fn into_string(self) -> String {
988         let Markdown {
989             content: md,
990             links,
991             ids,
992             error_codes: codes,
993             edition,
994             playground,
995             heading_offset,
996         } = self;
997 
998         // This is actually common enough to special-case
999         if md.is_empty() {
1000             return String::new();
1001         }
1002         let mut replacer = |broken_link: BrokenLink<'_>| {
1003             links
1004                 .iter()
1005                 .find(|link| &*link.original_text == &*broken_link.reference)
1006                 .map(|link| (link.href.as_str().into(), link.tooltip.as_str().into()))
1007         };
1008 
1009         let p = Parser::new_with_broken_link_callback(md, main_body_opts(), Some(&mut replacer));
1010         let p = p.into_offset_iter();
1011 
1012         let mut s = String::with_capacity(md.len() * 3 / 2);
1013 
1014         let p = HeadingLinks::new(p, None, ids, heading_offset);
1015         let p = Footnotes::new(p);
1016         let p = LinkReplacer::new(p.map(|(ev, _)| ev), links);
1017         let p = TableWrapper::new(p);
1018         let p = CodeBlocks::new(p, codes, edition, playground);
1019         html::push_html(&mut s, p);
1020 
1021         s
1022     }
1023 }
1024 
1025 impl MarkdownWithToc<'_> {
into_string(self) -> String1026     pub(crate) fn into_string(self) -> String {
1027         let MarkdownWithToc { content: md, ids, error_codes: codes, edition, playground } = self;
1028 
1029         let p = Parser::new_ext(md, main_body_opts()).into_offset_iter();
1030 
1031         let mut s = String::with_capacity(md.len() * 3 / 2);
1032 
1033         let mut toc = TocBuilder::new();
1034 
1035         {
1036             let p = HeadingLinks::new(p, Some(&mut toc), ids, HeadingOffset::H1);
1037             let p = Footnotes::new(p);
1038             let p = TableWrapper::new(p.map(|(ev, _)| ev));
1039             let p = CodeBlocks::new(p, codes, edition, playground);
1040             html::push_html(&mut s, p);
1041         }
1042 
1043         format!("<nav id=\"TOC\">{}</nav>{}", toc.into_toc().print(), s)
1044     }
1045 }
1046 
1047 impl MarkdownItemInfo<'_> {
into_string(self) -> String1048     pub(crate) fn into_string(self) -> String {
1049         let MarkdownItemInfo(md, ids) = self;
1050 
1051         // This is actually common enough to special-case
1052         if md.is_empty() {
1053             return String::new();
1054         }
1055         let p = Parser::new_ext(md, main_body_opts()).into_offset_iter();
1056 
1057         // Treat inline HTML as plain text.
1058         let p = p.map(|event| match event.0 {
1059             Event::Html(text) => (Event::Text(text), event.1),
1060             _ => event,
1061         });
1062 
1063         let mut s = String::with_capacity(md.len() * 3 / 2);
1064 
1065         let p = HeadingLinks::new(p, None, ids, HeadingOffset::H1);
1066         let p = Footnotes::new(p);
1067         let p = TableWrapper::new(p.map(|(ev, _)| ev));
1068         let p = p.filter(|event| {
1069             !matches!(event, Event::Start(Tag::Paragraph) | Event::End(Tag::Paragraph))
1070         });
1071         html::push_html(&mut s, p);
1072 
1073         s
1074     }
1075 }
1076 
1077 impl MarkdownSummaryLine<'_> {
into_string_with_has_more_content(self) -> (String, bool)1078     pub(crate) fn into_string_with_has_more_content(self) -> (String, bool) {
1079         let MarkdownSummaryLine(md, links) = self;
1080         // This is actually common enough to special-case
1081         if md.is_empty() {
1082             return (String::new(), false);
1083         }
1084 
1085         let mut replacer = |broken_link: BrokenLink<'_>| {
1086             links
1087                 .iter()
1088                 .find(|link| &*link.original_text == &*broken_link.reference)
1089                 .map(|link| (link.href.as_str().into(), link.tooltip.as_str().into()))
1090         };
1091 
1092         let p = Parser::new_with_broken_link_callback(md, summary_opts(), Some(&mut replacer))
1093             .peekable();
1094         let mut summary = SummaryLine::new(p);
1095 
1096         let mut s = String::new();
1097 
1098         let without_paragraphs = LinkReplacer::new(&mut summary, links).filter(|event| {
1099             !matches!(event, Event::Start(Tag::Paragraph) | Event::End(Tag::Paragraph))
1100         });
1101 
1102         html::push_html(&mut s, without_paragraphs);
1103 
1104         let has_more_content =
1105             matches!(summary.inner.peek(), Some(Event::Start(_))) || summary.skipped_tags > 0;
1106 
1107         (s, has_more_content)
1108     }
1109 
into_string(self) -> String1110     pub(crate) fn into_string(self) -> String {
1111         self.into_string_with_has_more_content().0
1112     }
1113 }
1114 
1115 /// Renders a subset of Markdown in the first paragraph of the provided Markdown.
1116 ///
1117 /// - *Italics*, **bold**, and `inline code` styles **are** rendered.
1118 /// - Headings and links are stripped (though the text *is* rendered).
1119 /// - HTML, code blocks, and everything else are ignored.
1120 ///
1121 /// Returns a tuple of the rendered HTML string and whether the output was shortened
1122 /// due to the provided `length_limit`.
markdown_summary_with_limit( md: &str, link_names: &[RenderedLink], length_limit: usize, ) -> (String, bool)1123 fn markdown_summary_with_limit(
1124     md: &str,
1125     link_names: &[RenderedLink],
1126     length_limit: usize,
1127 ) -> (String, bool) {
1128     if md.is_empty() {
1129         return (String::new(), false);
1130     }
1131 
1132     let mut replacer = |broken_link: BrokenLink<'_>| {
1133         link_names
1134             .iter()
1135             .find(|link| &*link.original_text == &*broken_link.reference)
1136             .map(|link| (link.href.as_str().into(), link.tooltip.as_str().into()))
1137     };
1138 
1139     let p = Parser::new_with_broken_link_callback(md, summary_opts(), Some(&mut replacer));
1140     let mut p = LinkReplacer::new(p, link_names);
1141 
1142     let mut buf = HtmlWithLimit::new(length_limit);
1143     let mut stopped_early = false;
1144     p.try_for_each(|event| {
1145         match &event {
1146             Event::Text(text) => {
1147                 let r =
1148                     text.split_inclusive(char::is_whitespace).try_for_each(|word| buf.push(word));
1149                 if r.is_break() {
1150                     stopped_early = true;
1151                 }
1152                 return r;
1153             }
1154             Event::Code(code) => {
1155                 buf.open_tag("code");
1156                 let r = buf.push(code);
1157                 if r.is_break() {
1158                     stopped_early = true;
1159                 } else {
1160                     buf.close_tag();
1161                 }
1162                 return r;
1163             }
1164             Event::Start(tag) => match tag {
1165                 Tag::Emphasis => buf.open_tag("em"),
1166                 Tag::Strong => buf.open_tag("strong"),
1167                 Tag::CodeBlock(..) => return ControlFlow::Break(()),
1168                 _ => {}
1169             },
1170             Event::End(tag) => match tag {
1171                 Tag::Emphasis | Tag::Strong => buf.close_tag(),
1172                 Tag::Paragraph | Tag::Heading(..) => return ControlFlow::Break(()),
1173                 _ => {}
1174             },
1175             Event::HardBreak | Event::SoftBreak => buf.push(" ")?,
1176             _ => {}
1177         };
1178         ControlFlow::Continue(())
1179     });
1180 
1181     (buf.finish(), stopped_early)
1182 }
1183 
1184 /// Renders a shortened first paragraph of the given Markdown as a subset of Markdown,
1185 /// making it suitable for contexts like the search index.
1186 ///
1187 /// Will shorten to 59 or 60 characters, including an ellipsis (…) if it was shortened.
1188 ///
1189 /// See [`markdown_summary_with_limit`] for details about what is rendered and what is not.
short_markdown_summary(markdown: &str, link_names: &[RenderedLink]) -> String1190 pub(crate) fn short_markdown_summary(markdown: &str, link_names: &[RenderedLink]) -> String {
1191     let (mut s, was_shortened) = markdown_summary_with_limit(markdown, link_names, 59);
1192 
1193     if was_shortened {
1194         s.push('…');
1195     }
1196 
1197     s
1198 }
1199 
1200 /// Renders the first paragraph of the provided markdown as plain text.
1201 /// Useful for alt-text.
1202 ///
1203 /// - Headings, links, and formatting are stripped.
1204 /// - Inline code is rendered as-is, surrounded by backticks.
1205 /// - HTML and code blocks are ignored.
plain_text_summary(md: &str, link_names: &[RenderedLink]) -> String1206 pub(crate) fn plain_text_summary(md: &str, link_names: &[RenderedLink]) -> String {
1207     if md.is_empty() {
1208         return String::new();
1209     }
1210 
1211     let mut s = String::with_capacity(md.len() * 3 / 2);
1212 
1213     let mut replacer = |broken_link: BrokenLink<'_>| {
1214         link_names
1215             .iter()
1216             .find(|link| &*link.original_text == &*broken_link.reference)
1217             .map(|link| (link.href.as_str().into(), link.tooltip.as_str().into()))
1218     };
1219 
1220     let p = Parser::new_with_broken_link_callback(md, summary_opts(), Some(&mut replacer));
1221 
1222     for event in p {
1223         match &event {
1224             Event::Text(text) => s.push_str(text),
1225             Event::Code(code) => {
1226                 s.push('`');
1227                 s.push_str(code);
1228                 s.push('`');
1229             }
1230             Event::HardBreak | Event::SoftBreak => s.push(' '),
1231             Event::Start(Tag::CodeBlock(..)) => break,
1232             Event::End(Tag::Paragraph) => break,
1233             Event::End(Tag::Heading(..)) => break,
1234             _ => (),
1235         }
1236     }
1237 
1238     s
1239 }
1240 
1241 #[derive(Debug)]
1242 pub(crate) struct MarkdownLink {
1243     pub kind: LinkType,
1244     pub link: String,
1245     pub range: MarkdownLinkRange,
1246 }
1247 
1248 #[derive(Clone, Debug)]
1249 pub(crate) enum MarkdownLinkRange {
1250     /// Normally, markdown link warnings point only at the destination.
1251     Destination(Range<usize>),
1252     /// In some cases, it's not possible to point at the destination.
1253     /// Usually, this happens because backslashes `\\` are used.
1254     /// When that happens, point at the whole link, and don't provide structured suggestions.
1255     WholeLink(Range<usize>),
1256 }
1257 
1258 impl MarkdownLinkRange {
1259     /// Extracts the inner range.
inner_range(&self) -> &Range<usize>1260     pub fn inner_range(&self) -> &Range<usize> {
1261         match self {
1262             MarkdownLinkRange::Destination(range) => range,
1263             MarkdownLinkRange::WholeLink(range) => range,
1264         }
1265     }
1266 }
1267 
markdown_links<R>( md: &str, preprocess_link: impl Fn(MarkdownLink) -> Option<R>, ) -> Vec<R>1268 pub(crate) fn markdown_links<R>(
1269     md: &str,
1270     preprocess_link: impl Fn(MarkdownLink) -> Option<R>,
1271 ) -> Vec<R> {
1272     if md.is_empty() {
1273         return vec![];
1274     }
1275 
1276     // FIXME: remove this function once pulldown_cmark can provide spans for link definitions.
1277     let locate = |s: &str, fallback: Range<usize>| unsafe {
1278         let s_start = s.as_ptr();
1279         let s_end = s_start.add(s.len());
1280         let md_start = md.as_ptr();
1281         let md_end = md_start.add(md.len());
1282         if md_start <= s_start && s_end <= md_end {
1283             let start = s_start.offset_from(md_start) as usize;
1284             let end = s_end.offset_from(md_start) as usize;
1285             MarkdownLinkRange::Destination(start..end)
1286         } else {
1287             MarkdownLinkRange::WholeLink(fallback)
1288         }
1289     };
1290 
1291     let span_for_link = |link: &CowStr<'_>, span: Range<usize>| {
1292         // For diagnostics, we want to underline the link's definition but `span` will point at
1293         // where the link is used. This is a problem for reference-style links, where the definition
1294         // is separate from the usage.
1295 
1296         match link {
1297             // `Borrowed` variant means the string (the link's destination) may come directly from
1298             // the markdown text and we can locate the original link destination.
1299             // NOTE: LinkReplacer also provides `Borrowed` but possibly from other sources,
1300             // so `locate()` can fall back to use `span`.
1301             CowStr::Borrowed(s) => locate(s, span),
1302 
1303             // For anything else, we can only use the provided range.
1304             CowStr::Boxed(_) | CowStr::Inlined(_) => MarkdownLinkRange::WholeLink(span),
1305         }
1306     };
1307 
1308     let span_for_offset_backward = |span: Range<usize>, open: u8, close: u8| {
1309         let mut open_brace = !0;
1310         let mut close_brace = !0;
1311         for (i, b) in md.as_bytes()[span.clone()].iter().copied().enumerate().rev() {
1312             let i = i + span.start;
1313             if b == close {
1314                 close_brace = i;
1315                 break;
1316             }
1317         }
1318         if close_brace < span.start || close_brace >= span.end {
1319             return MarkdownLinkRange::WholeLink(span);
1320         }
1321         let mut nesting = 1;
1322         for (i, b) in md.as_bytes()[span.start..close_brace].iter().copied().enumerate().rev() {
1323             let i = i + span.start;
1324             if b == close {
1325                 nesting += 1;
1326             }
1327             if b == open {
1328                 nesting -= 1;
1329             }
1330             if nesting == 0 {
1331                 open_brace = i;
1332                 break;
1333             }
1334         }
1335         assert!(open_brace != close_brace);
1336         if open_brace < span.start || open_brace >= span.end {
1337             return MarkdownLinkRange::WholeLink(span);
1338         }
1339         // do not actually include braces in the span
1340         let range = (open_brace + 1)..close_brace;
1341         MarkdownLinkRange::Destination(range.clone())
1342     };
1343 
1344     let span_for_offset_forward = |span: Range<usize>, open: u8, close: u8| {
1345         let mut open_brace = !0;
1346         let mut close_brace = !0;
1347         for (i, b) in md.as_bytes()[span.clone()].iter().copied().enumerate() {
1348             let i = i + span.start;
1349             if b == open {
1350                 open_brace = i;
1351                 break;
1352             }
1353         }
1354         if open_brace < span.start || open_brace >= span.end {
1355             return MarkdownLinkRange::WholeLink(span);
1356         }
1357         let mut nesting = 0;
1358         for (i, b) in md.as_bytes()[open_brace..span.end].iter().copied().enumerate() {
1359             let i = i + open_brace;
1360             if b == close {
1361                 nesting -= 1;
1362             }
1363             if b == open {
1364                 nesting += 1;
1365             }
1366             if nesting == 0 {
1367                 close_brace = i;
1368                 break;
1369             }
1370         }
1371         assert!(open_brace != close_brace);
1372         if open_brace < span.start || open_brace >= span.end {
1373             return MarkdownLinkRange::WholeLink(span);
1374         }
1375         // do not actually include braces in the span
1376         let range = (open_brace + 1)..close_brace;
1377         MarkdownLinkRange::Destination(range.clone())
1378     };
1379 
1380     Parser::new_with_broken_link_callback(
1381         md,
1382         main_body_opts(),
1383         Some(&mut |link: BrokenLink<'_>| Some((link.reference, "".into()))),
1384     )
1385     .into_offset_iter()
1386     .filter_map(|(event, span)| match event {
1387         Event::Start(Tag::Link(link_type, dest, _)) if may_be_doc_link(link_type) => {
1388             let range = match link_type {
1389                 // Link is pulled from the link itself.
1390                 LinkType::ReferenceUnknown | LinkType::ShortcutUnknown => {
1391                     span_for_offset_backward(span, b'[', b']')
1392                 }
1393                 LinkType::CollapsedUnknown => span_for_offset_forward(span, b'[', b']'),
1394                 LinkType::Inline => span_for_offset_backward(span, b'(', b')'),
1395                 // Link is pulled from elsewhere in the document.
1396                 LinkType::Reference | LinkType::Collapsed | LinkType::Shortcut => {
1397                     span_for_link(&dest, span)
1398                 }
1399                 LinkType::Autolink | LinkType::Email => unreachable!(),
1400             };
1401             preprocess_link(MarkdownLink { kind: link_type, range, link: dest.into_string() })
1402         }
1403         _ => None,
1404     })
1405     .collect()
1406 }
1407 
1408 #[derive(Debug)]
1409 pub(crate) struct RustCodeBlock {
1410     /// The range in the markdown that the code block occupies. Note that this includes the fences
1411     /// for fenced code blocks.
1412     pub(crate) range: Range<usize>,
1413     /// The range in the markdown that the code within the code block occupies.
1414     pub(crate) code: Range<usize>,
1415     pub(crate) is_fenced: bool,
1416     pub(crate) lang_string: LangString,
1417 }
1418 
1419 /// Returns a range of bytes for each code block in the markdown that is tagged as `rust` or
1420 /// untagged (and assumed to be rust).
rust_code_blocks(md: &str, extra_info: &ExtraInfo<'_>) -> Vec<RustCodeBlock>1421 pub(crate) fn rust_code_blocks(md: &str, extra_info: &ExtraInfo<'_>) -> Vec<RustCodeBlock> {
1422     let mut code_blocks = vec![];
1423 
1424     if md.is_empty() {
1425         return code_blocks;
1426     }
1427 
1428     let mut p = Parser::new_ext(md, main_body_opts()).into_offset_iter();
1429 
1430     while let Some((event, offset)) = p.next() {
1431         if let Event::Start(Tag::CodeBlock(syntax)) = event {
1432             let (lang_string, code_start, code_end, range, is_fenced) = match syntax {
1433                 CodeBlockKind::Fenced(syntax) => {
1434                     let syntax = syntax.as_ref();
1435                     let lang_string = if syntax.is_empty() {
1436                         Default::default()
1437                     } else {
1438                         LangString::parse(&*syntax, ErrorCodes::Yes, false, Some(extra_info))
1439                     };
1440                     if !lang_string.rust {
1441                         continue;
1442                     }
1443                     let (code_start, mut code_end) = match p.next() {
1444                         Some((Event::Text(_), offset)) => (offset.start, offset.end),
1445                         Some((_, sub_offset)) => {
1446                             let code = Range { start: sub_offset.start, end: sub_offset.start };
1447                             code_blocks.push(RustCodeBlock {
1448                                 is_fenced: true,
1449                                 range: offset,
1450                                 code,
1451                                 lang_string,
1452                             });
1453                             continue;
1454                         }
1455                         None => {
1456                             let code = Range { start: offset.end, end: offset.end };
1457                             code_blocks.push(RustCodeBlock {
1458                                 is_fenced: true,
1459                                 range: offset,
1460                                 code,
1461                                 lang_string,
1462                             });
1463                             continue;
1464                         }
1465                     };
1466                     while let Some((Event::Text(_), offset)) = p.next() {
1467                         code_end = offset.end;
1468                     }
1469                     (lang_string, code_start, code_end, offset, true)
1470                 }
1471                 CodeBlockKind::Indented => {
1472                     // The ending of the offset goes too far sometime so we reduce it by one in
1473                     // these cases.
1474                     if offset.end > offset.start && md.get(offset.end..=offset.end) == Some("\n") {
1475                         (
1476                             LangString::default(),
1477                             offset.start,
1478                             offset.end,
1479                             Range { start: offset.start, end: offset.end - 1 },
1480                             false,
1481                         )
1482                     } else {
1483                         (LangString::default(), offset.start, offset.end, offset, false)
1484                     }
1485                 }
1486             };
1487 
1488             code_blocks.push(RustCodeBlock {
1489                 is_fenced,
1490                 range,
1491                 code: Range { start: code_start, end: code_end },
1492                 lang_string,
1493             });
1494         }
1495     }
1496 
1497     code_blocks
1498 }
1499 
1500 #[derive(Clone, Default, Debug)]
1501 pub struct IdMap {
1502     map: FxHashMap<Cow<'static, str>, usize>,
1503 }
1504 
1505 // The map is pre-initialized and cloned each time to avoid reinitializing it repeatedly.
1506 static DEFAULT_ID_MAP: Lazy<FxHashMap<Cow<'static, str>, usize>> = Lazy::new(|| init_id_map());
1507 
init_id_map() -> FxHashMap<Cow<'static, str>, usize>1508 fn init_id_map() -> FxHashMap<Cow<'static, str>, usize> {
1509     let mut map = FxHashMap::default();
1510     // This is the list of IDs used in JavaScript.
1511     map.insert("help".into(), 1);
1512     map.insert("settings".into(), 1);
1513     map.insert("not-displayed".into(), 1);
1514     map.insert("alternative-display".into(), 1);
1515     map.insert("search".into(), 1);
1516     map.insert("crate-search".into(), 1);
1517     map.insert("crate-search-div".into(), 1);
1518     // This is the list of IDs used in HTML generated in Rust (including the ones
1519     // used in tera template files).
1520     map.insert("mainThemeStyle".into(), 1);
1521     map.insert("themeStyle".into(), 1);
1522     map.insert("settings-menu".into(), 1);
1523     map.insert("help-button".into(), 1);
1524     map.insert("main-content".into(), 1);
1525     map.insert("toggle-all-docs".into(), 1);
1526     map.insert("all-types".into(), 1);
1527     map.insert("default-settings".into(), 1);
1528     map.insert("sidebar-vars".into(), 1);
1529     map.insert("copy-path".into(), 1);
1530     map.insert("TOC".into(), 1);
1531     // This is the list of IDs used by rustdoc sections (but still generated by
1532     // rustdoc).
1533     map.insert("fields".into(), 1);
1534     map.insert("variants".into(), 1);
1535     map.insert("implementors-list".into(), 1);
1536     map.insert("synthetic-implementors-list".into(), 1);
1537     map.insert("foreign-impls".into(), 1);
1538     map.insert("implementations".into(), 1);
1539     map.insert("trait-implementations".into(), 1);
1540     map.insert("synthetic-implementations".into(), 1);
1541     map.insert("blanket-implementations".into(), 1);
1542     map.insert("required-associated-types".into(), 1);
1543     map.insert("provided-associated-types".into(), 1);
1544     map.insert("provided-associated-consts".into(), 1);
1545     map.insert("required-associated-consts".into(), 1);
1546     map.insert("required-methods".into(), 1);
1547     map.insert("provided-methods".into(), 1);
1548     map.insert("implementors".into(), 1);
1549     map.insert("synthetic-implementors".into(), 1);
1550     map.insert("implementations-list".into(), 1);
1551     map.insert("trait-implementations-list".into(), 1);
1552     map.insert("synthetic-implementations-list".into(), 1);
1553     map.insert("blanket-implementations-list".into(), 1);
1554     map.insert("deref-methods".into(), 1);
1555     map.insert("layout".into(), 1);
1556     map
1557 }
1558 
1559 impl IdMap {
new() -> Self1560     pub fn new() -> Self {
1561         IdMap { map: DEFAULT_ID_MAP.clone() }
1562     }
1563 
derive<S: AsRef<str> + ToString>(&mut self, candidate: S) -> String1564     pub(crate) fn derive<S: AsRef<str> + ToString>(&mut self, candidate: S) -> String {
1565         let id = match self.map.get_mut(candidate.as_ref()) {
1566             None => candidate.to_string(),
1567             Some(a) => {
1568                 let id = format!("{}-{}", candidate.as_ref(), *a);
1569                 *a += 1;
1570                 id
1571             }
1572         };
1573 
1574         self.map.insert(id.clone().into(), 1);
1575         id
1576     }
1577 }
1578