• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2"""
3    pyspecific.py
4    ~~~~~~~~~~~~~
5
6    Sphinx extension with Python doc-specific markup.
7
8    :copyright: 2008-2014 by Georg Brandl.
9    :license: Python license.
10"""
11
12import re
13import io
14from os import getenv, path
15from time import asctime
16from pprint import pformat
17from docutils.io import StringOutput
18from docutils.parsers.rst import Directive
19from docutils.utils import new_document
20
21from docutils import nodes, utils
22
23from sphinx import addnodes
24from sphinx.builders import Builder
25try:
26    from sphinx.errors import NoUri
27except ImportError:
28    from sphinx.environment import NoUri
29from sphinx.locale import translators
30from sphinx.util import status_iterator, logging
31from sphinx.util.nodes import split_explicit_title
32from sphinx.writers.text import TextWriter, TextTranslator
33from sphinx.writers.latex import LaTeXTranslator
34
35try:
36    from sphinx.domains.python import PyFunction, PyMethod
37except ImportError:
38    from sphinx.domains.python import PyClassmember as PyMethod
39    from sphinx.domains.python import PyModulelevel as PyFunction
40
41# Support for checking for suspicious markup
42
43import suspicious
44
45
46ISSUE_URI = 'https://bugs.python.org/issue%s'
47SOURCE_URI = 'https://github.com/python/cpython/tree/3.10/%s'
48
49# monkey-patch reST parser to disable alphabetic and roman enumerated lists
50from docutils.parsers.rst.states import Body
51Body.enum.converters['loweralpha'] = \
52    Body.enum.converters['upperalpha'] = \
53    Body.enum.converters['lowerroman'] = \
54    Body.enum.converters['upperroman'] = lambda x: None
55
56
57# Support for marking up and linking to bugs.python.org issues
58
59def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
60    issue = utils.unescape(text)
61    text = 'bpo-' + issue
62    refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue)
63    return [refnode], []
64
65
66# Support for linking to Python source files easily
67
68def source_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
69    has_t, title, target = split_explicit_title(text)
70    title = utils.unescape(title)
71    target = utils.unescape(target)
72    refnode = nodes.reference(title, title, refuri=SOURCE_URI % target)
73    return [refnode], []
74
75
76# Support for marking up implementation details
77
78class ImplementationDetail(Directive):
79
80    has_content = True
81    required_arguments = 0
82    optional_arguments = 1
83    final_argument_whitespace = True
84
85    # This text is copied to templates/dummy.html
86    label_text = 'CPython implementation detail:'
87
88    def run(self):
89        pnode = nodes.compound(classes=['impl-detail'])
90        label = translators['sphinx'].gettext(self.label_text)
91        content = self.content
92        add_text = nodes.strong(label, label)
93        if self.arguments:
94            n, m = self.state.inline_text(self.arguments[0], self.lineno)
95            pnode.append(nodes.paragraph('', '', *(n + m)))
96        self.state.nested_parse(content, self.content_offset, pnode)
97        if pnode.children and isinstance(pnode[0], nodes.paragraph):
98            content = nodes.inline(pnode[0].rawsource, translatable=True)
99            content.source = pnode[0].source
100            content.line = pnode[0].line
101            content += pnode[0].children
102            pnode[0].replace_self(nodes.paragraph('', '', content,
103                                                  translatable=False))
104            pnode[0].insert(0, add_text)
105            pnode[0].insert(1, nodes.Text(' '))
106        else:
107            pnode.insert(0, nodes.paragraph('', '', add_text))
108        return [pnode]
109
110
111# Support for documenting platform availability
112
113class Availability(Directive):
114
115    has_content = False
116    required_arguments = 1
117    optional_arguments = 0
118    final_argument_whitespace = True
119
120    def run(self):
121        availability_ref = ':ref:`Availability <availability>`: '
122        pnode = nodes.paragraph(availability_ref + self.arguments[0],
123                                classes=["availability"],)
124        n, m = self.state.inline_text(availability_ref, self.lineno)
125        pnode.extend(n + m)
126        n, m = self.state.inline_text(self.arguments[0], self.lineno)
127        pnode.extend(n + m)
128        return [pnode]
129
130
131# Support for documenting audit event
132
133def audit_events_purge(app, env, docname):
134    """This is to remove from env.all_audit_events old traces of removed
135    documents.
136    """
137    if not hasattr(env, 'all_audit_events'):
138        return
139    fresh_all_audit_events = {}
140    for name, event in env.all_audit_events.items():
141        event["source"] = [(d, t) for d, t in event["source"] if d != docname]
142        if event["source"]:
143            # Only keep audit_events that have at least one source.
144            fresh_all_audit_events[name] = event
145    env.all_audit_events = fresh_all_audit_events
146
147
148def audit_events_merge(app, env, docnames, other):
149    """In Sphinx parallel builds, this merges env.all_audit_events from
150    subprocesses.
151
152    all_audit_events is a dict of names, with values like:
153    {'source': [(docname, target), ...], 'args': args}
154    """
155    if not hasattr(other, 'all_audit_events'):
156        return
157    if not hasattr(env, 'all_audit_events'):
158        env.all_audit_events = {}
159    for name, value in other.all_audit_events.items():
160        if name in env.all_audit_events:
161            env.all_audit_events[name]["source"].extend(value["source"])
162        else:
163            env.all_audit_events[name] = value
164
165
166class AuditEvent(Directive):
167
168    has_content = True
169    required_arguments = 1
170    optional_arguments = 2
171    final_argument_whitespace = True
172
173    _label = [
174        "Raises an :ref:`auditing event <auditing>` {name} with no arguments.",
175        "Raises an :ref:`auditing event <auditing>` {name} with argument {args}.",
176        "Raises an :ref:`auditing event <auditing>` {name} with arguments {args}.",
177    ]
178
179    @property
180    def logger(self):
181        cls = type(self)
182        return logging.getLogger(cls.__module__ + "." + cls.__name__)
183
184    def run(self):
185        name = self.arguments[0]
186        if len(self.arguments) >= 2 and self.arguments[1]:
187            args = (a.strip() for a in self.arguments[1].strip("'\"").split(","))
188            args = [a for a in args if a]
189        else:
190            args = []
191
192        label = translators['sphinx'].gettext(self._label[min(2, len(args))])
193        text = label.format(name="``{}``".format(name),
194                            args=", ".join("``{}``".format(a) for a in args if a))
195
196        env = self.state.document.settings.env
197        if not hasattr(env, 'all_audit_events'):
198            env.all_audit_events = {}
199
200        new_info = {
201            'source': [],
202            'args': args
203        }
204        info = env.all_audit_events.setdefault(name, new_info)
205        if info is not new_info:
206            if not self._do_args_match(info['args'], new_info['args']):
207                self.logger.warn(
208                    "Mismatched arguments for audit-event {}: {!r} != {!r}"
209                    .format(name, info['args'], new_info['args'])
210                )
211
212        ids = []
213        try:
214            target = self.arguments[2].strip("\"'")
215        except (IndexError, TypeError):
216            target = None
217        if not target:
218            target = "audit_event_{}_{}".format(
219                re.sub(r'\W', '_', name),
220                len(info['source']),
221            )
222            ids.append(target)
223
224        info['source'].append((env.docname, target))
225
226        pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids)
227        pnode.line = self.lineno
228        if self.content:
229            self.state.nested_parse(self.content, self.content_offset, pnode)
230        else:
231            n, m = self.state.inline_text(text, self.lineno)
232            pnode.extend(n + m)
233
234        return [pnode]
235
236    # This list of sets are allowable synonyms for event argument names.
237    # If two names are in the same set, they are treated as equal for the
238    # purposes of warning. This won't help if number of arguments is
239    # different!
240    _SYNONYMS = [
241        {"file", "path", "fd"},
242    ]
243
244    def _do_args_match(self, args1, args2):
245        if args1 == args2:
246            return True
247        if len(args1) != len(args2):
248            return False
249        for a1, a2 in zip(args1, args2):
250            if a1 == a2:
251                continue
252            if any(a1 in s and a2 in s for s in self._SYNONYMS):
253                continue
254            return False
255        return True
256
257
258class audit_event_list(nodes.General, nodes.Element):
259    pass
260
261
262class AuditEventListDirective(Directive):
263
264    def run(self):
265        return [audit_event_list('')]
266
267
268# Support for documenting decorators
269
270class PyDecoratorMixin(object):
271    def handle_signature(self, sig, signode):
272        ret = super(PyDecoratorMixin, self).handle_signature(sig, signode)
273        signode.insert(0, addnodes.desc_addname('@', '@'))
274        return ret
275
276    def needs_arglist(self):
277        return False
278
279
280class PyDecoratorFunction(PyDecoratorMixin, PyFunction):
281    def run(self):
282        # a decorator function is a function after all
283        self.name = 'py:function'
284        return PyFunction.run(self)
285
286
287# TODO: Use sphinx.domains.python.PyDecoratorMethod when possible
288class PyDecoratorMethod(PyDecoratorMixin, PyMethod):
289    def run(self):
290        self.name = 'py:method'
291        return PyMethod.run(self)
292
293
294class PyCoroutineMixin(object):
295    def handle_signature(self, sig, signode):
296        ret = super(PyCoroutineMixin, self).handle_signature(sig, signode)
297        signode.insert(0, addnodes.desc_annotation('coroutine ', 'coroutine '))
298        return ret
299
300
301class PyAwaitableMixin(object):
302    def handle_signature(self, sig, signode):
303        ret = super(PyAwaitableMixin, self).handle_signature(sig, signode)
304        signode.insert(0, addnodes.desc_annotation('awaitable ', 'awaitable '))
305        return ret
306
307
308class PyCoroutineFunction(PyCoroutineMixin, PyFunction):
309    def run(self):
310        self.name = 'py:function'
311        return PyFunction.run(self)
312
313
314class PyCoroutineMethod(PyCoroutineMixin, PyMethod):
315    def run(self):
316        self.name = 'py:method'
317        return PyMethod.run(self)
318
319
320class PyAwaitableFunction(PyAwaitableMixin, PyFunction):
321    def run(self):
322        self.name = 'py:function'
323        return PyFunction.run(self)
324
325
326class PyAwaitableMethod(PyAwaitableMixin, PyMethod):
327    def run(self):
328        self.name = 'py:method'
329        return PyMethod.run(self)
330
331
332class PyAbstractMethod(PyMethod):
333
334    def handle_signature(self, sig, signode):
335        ret = super(PyAbstractMethod, self).handle_signature(sig, signode)
336        signode.insert(0, addnodes.desc_annotation('abstractmethod ',
337                                                   'abstractmethod '))
338        return ret
339
340    def run(self):
341        self.name = 'py:method'
342        return PyMethod.run(self)
343
344
345# Support for documenting version of removal in deprecations
346
347class DeprecatedRemoved(Directive):
348    has_content = True
349    required_arguments = 2
350    optional_arguments = 1
351    final_argument_whitespace = True
352    option_spec = {}
353
354    _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}'
355    _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}'
356
357    def run(self):
358        node = addnodes.versionmodified()
359        node.document = self.state.document
360        node['type'] = 'deprecated-removed'
361        version = (self.arguments[0], self.arguments[1])
362        node['version'] = version
363        env = self.state.document.settings.env
364        current_version = tuple(int(e) for e in env.config.version.split('.'))
365        removed_version = tuple(int(e) for e in self.arguments[1].split('.'))
366        if current_version < removed_version:
367            label = self._deprecated_label
368        else:
369            label = self._removed_label
370
371        label = translators['sphinx'].gettext(label)
372        text = label.format(deprecated=self.arguments[0], removed=self.arguments[1])
373        if len(self.arguments) == 3:
374            inodes, messages = self.state.inline_text(self.arguments[2],
375                                                      self.lineno+1)
376            para = nodes.paragraph(self.arguments[2], '', *inodes, translatable=False)
377            node.append(para)
378        else:
379            messages = []
380        if self.content:
381            self.state.nested_parse(self.content, self.content_offset, node)
382        if len(node):
383            if isinstance(node[0], nodes.paragraph) and node[0].rawsource:
384                content = nodes.inline(node[0].rawsource, translatable=True)
385                content.source = node[0].source
386                content.line = node[0].line
387                content += node[0].children
388                node[0].replace_self(nodes.paragraph('', '', content, translatable=False))
389            node[0].insert(0, nodes.inline('', '%s: ' % text,
390                                           classes=['versionmodified']))
391        else:
392            para = nodes.paragraph('', '',
393                                   nodes.inline('', '%s.' % text,
394                                                classes=['versionmodified']),
395                                   translatable=False)
396            node.append(para)
397        env = self.state.document.settings.env
398        # deprecated pre-Sphinx-2 method
399        if hasattr(env, 'note_versionchange'):
400            env.note_versionchange('deprecated', version[0], node, self.lineno)
401        # new method
402        else:
403            env.get_domain('changeset').note_changeset(node)
404        return [node] + messages
405
406
407# Support for including Misc/NEWS
408
409issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)')
410whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$")
411
412
413class MiscNews(Directive):
414    has_content = False
415    required_arguments = 1
416    optional_arguments = 0
417    final_argument_whitespace = False
418    option_spec = {}
419
420    def run(self):
421        fname = self.arguments[0]
422        source = self.state_machine.input_lines.source(
423            self.lineno - self.state_machine.input_offset - 1)
424        source_dir = getenv('PY_MISC_NEWS_DIR')
425        if not source_dir:
426            source_dir = path.dirname(path.abspath(source))
427        fpath = path.join(source_dir, fname)
428        self.state.document.settings.record_dependencies.add(fpath)
429        try:
430            with io.open(fpath, encoding='utf-8') as fp:
431                content = fp.read()
432        except Exception:
433            text = 'The NEWS file is not available.'
434            node = nodes.strong(text, text)
435            return [node]
436        content = issue_re.sub(r'`bpo-\1 <https://bugs.python.org/issue\1>`__',
437                               content)
438        content = whatsnew_re.sub(r'\1', content)
439        # remove first 3 lines as they are the main heading
440        lines = ['.. default-role:: obj', ''] + content.splitlines()[3:]
441        self.state_machine.insert_input(lines, fname)
442        return []
443
444
445# Support for building "topic help" for pydoc
446
447pydoc_topic_labels = [
448    'assert', 'assignment', 'async', 'atom-identifiers', 'atom-literals',
449    'attribute-access', 'attribute-references', 'augassign', 'await',
450    'binary', 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object',
451    'bltin-null-object', 'bltin-type-objects', 'booleans',
452    'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound',
453    'context-managers', 'continue', 'conversions', 'customization', 'debugger',
454    'del', 'dict', 'dynamic-features', 'else', 'exceptions', 'execmodel',
455    'exprlists', 'floating', 'for', 'formatstrings', 'function', 'global',
456    'id-classes', 'identifiers', 'if', 'imaginary', 'import', 'in', 'integers',
457    'lambda', 'lists', 'naming', 'nonlocal', 'numbers', 'numeric-types',
458    'objects', 'operator-summary', 'pass', 'power', 'raise', 'return',
459    'sequence-types', 'shifting', 'slicings', 'specialattrs', 'specialnames',
460    'string-methods', 'strings', 'subscriptions', 'truth', 'try', 'types',
461    'typesfunctions', 'typesmapping', 'typesmethods', 'typesmodules',
462    'typesseq', 'typesseq-mutable', 'unary', 'while', 'with', 'yield'
463]
464
465
466class PydocTopicsBuilder(Builder):
467    name = 'pydoc-topics'
468
469    default_translator_class = TextTranslator
470
471    def init(self):
472        self.topics = {}
473        self.secnumbers = {}
474
475    def get_outdated_docs(self):
476        return 'all pydoc topics'
477
478    def get_target_uri(self, docname, typ=None):
479        return ''  # no URIs
480
481    def write(self, *ignored):
482        writer = TextWriter(self)
483        for label in status_iterator(pydoc_topic_labels,
484                                     'building topics... ',
485                                     length=len(pydoc_topic_labels)):
486            if label not in self.env.domaindata['std']['labels']:
487                self.env.logger.warn('label %r not in documentation' % label)
488                continue
489            docname, labelid, sectname = self.env.domaindata['std']['labels'][label]
490            doctree = self.env.get_and_resolve_doctree(docname, self)
491            document = new_document('<section node>')
492            document.append(doctree.ids[labelid])
493            destination = StringOutput(encoding='utf-8')
494            writer.write(document, destination)
495            self.topics[label] = writer.output
496
497    def finish(self):
498        f = open(path.join(self.outdir, 'topics.py'), 'wb')
499        try:
500            f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8'))
501            f.write(('# Autogenerated by Sphinx on %s\n' % asctime()).encode('utf-8'))
502            f.write(('topics = ' + pformat(self.topics) + '\n').encode('utf-8'))
503        finally:
504            f.close()
505
506
507# Support for documenting Opcodes
508
509opcode_sig_re = re.compile(r'(\w+(?:\+\d)?)(?:\s*\((.*)\))?')
510
511
512def parse_opcode_signature(env, sig, signode):
513    """Transform an opcode signature into RST nodes."""
514    m = opcode_sig_re.match(sig)
515    if m is None:
516        raise ValueError
517    opname, arglist = m.groups()
518    signode += addnodes.desc_name(opname, opname)
519    if arglist is not None:
520        paramlist = addnodes.desc_parameterlist()
521        signode += paramlist
522        paramlist += addnodes.desc_parameter(arglist, arglist)
523    return opname.strip()
524
525
526# Support for documenting pdb commands
527
528pdbcmd_sig_re = re.compile(r'([a-z()!]+)\s*(.*)')
529
530# later...
531# pdbargs_tokens_re = re.compile(r'''[a-zA-Z]+  |  # identifiers
532#                                   [.,:]+     |  # punctuation
533#                                   [\[\]()]   |  # parens
534#                                   \s+           # whitespace
535#                                   ''', re.X)
536
537
538def parse_pdb_command(env, sig, signode):
539    """Transform a pdb command signature into RST nodes."""
540    m = pdbcmd_sig_re.match(sig)
541    if m is None:
542        raise ValueError
543    name, args = m.groups()
544    fullname = name.replace('(', '').replace(')', '')
545    signode += addnodes.desc_name(name, name)
546    if args:
547        signode += addnodes.desc_addname(' '+args, ' '+args)
548    return fullname
549
550
551def process_audit_events(app, doctree, fromdocname):
552    for node in doctree.traverse(audit_event_list):
553        break
554    else:
555        return
556
557    env = app.builder.env
558
559    table = nodes.table(cols=3)
560    group = nodes.tgroup(
561        '',
562        nodes.colspec(colwidth=30),
563        nodes.colspec(colwidth=55),
564        nodes.colspec(colwidth=15),
565        cols=3,
566    )
567    head = nodes.thead()
568    body = nodes.tbody()
569
570    table += group
571    group += head
572    group += body
573
574    row = nodes.row()
575    row += nodes.entry('', nodes.paragraph('', nodes.Text('Audit event')))
576    row += nodes.entry('', nodes.paragraph('', nodes.Text('Arguments')))
577    row += nodes.entry('', nodes.paragraph('', nodes.Text('References')))
578    head += row
579
580    for name in sorted(getattr(env, "all_audit_events", ())):
581        audit_event = env.all_audit_events[name]
582
583        row = nodes.row()
584        node = nodes.paragraph('', nodes.Text(name))
585        row += nodes.entry('', node)
586
587        node = nodes.paragraph()
588        for i, a in enumerate(audit_event['args']):
589            if i:
590                node += nodes.Text(", ")
591            node += nodes.literal(a, nodes.Text(a))
592        row += nodes.entry('', node)
593
594        node = nodes.paragraph()
595        backlinks = enumerate(sorted(set(audit_event['source'])), start=1)
596        for i, (doc, label) in backlinks:
597            if isinstance(label, str):
598                ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True)
599                try:
600                    ref['refuri'] = "{}#{}".format(
601                        app.builder.get_relative_uri(fromdocname, doc),
602                        label,
603                    )
604                except NoUri:
605                    continue
606                node += ref
607        row += nodes.entry('', node)
608
609        body += row
610
611    for node in doctree.traverse(audit_event_list):
612        node.replace_self(table)
613
614
615def setup(app):
616    app.add_role('issue', issue_role)
617    app.add_role('source', source_role)
618    app.add_directive('impl-detail', ImplementationDetail)
619    app.add_directive('availability', Availability)
620    app.add_directive('audit-event', AuditEvent)
621    app.add_directive('audit-event-table', AuditEventListDirective)
622    app.add_directive('deprecated-removed', DeprecatedRemoved)
623    app.add_builder(PydocTopicsBuilder)
624    app.add_builder(suspicious.CheckSuspiciousMarkupBuilder)
625    app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature)
626    app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command)
627    app.add_object_type('2to3fixer', '2to3fixer', '%s (2to3 fixer)')
628    app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction)
629    app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod)
630    app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction)
631    app.add_directive_to_domain('py', 'coroutinemethod', PyCoroutineMethod)
632    app.add_directive_to_domain('py', 'awaitablefunction', PyAwaitableFunction)
633    app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod)
634    app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod)
635    app.add_directive('miscnews', MiscNews)
636    app.connect('doctree-resolved', process_audit_events)
637    app.connect('env-merge-info', audit_events_merge)
638    app.connect('env-purge-doc', audit_events_purge)
639    return {'version': '1.0', 'parallel_read_safe': True}
640