• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# -*- coding: utf-8 -*-
2"""
3    pyspecific.py
4    ~~~~~~~~~~~~~
5
6    Sphinx extension with Python doc-specific markup.
7
8    :copyright: 2008-2014 by Georg Brandl.
9    :license: Python license.
10"""
11
12import re
13import io
14from os import getenv, path
15from time import asctime
16from pprint import pformat
17from docutils.io import StringOutput
18from docutils.parsers.rst import Directive
19from docutils.utils import new_document
20
21from docutils import nodes, utils
22
23from sphinx import addnodes
24from sphinx.builders import Builder
25try:
26    from sphinx.errors import NoUri
27except ImportError:
28    from sphinx.environment import NoUri
29from sphinx.locale import translators
30from sphinx.util import status_iterator, logging
31from sphinx.util.nodes import split_explicit_title
32from sphinx.writers.text import TextWriter, TextTranslator
33from sphinx.writers.latex import LaTeXTranslator
34
35try:
36    from sphinx.domains.python import PyFunction, PyMethod
37except ImportError:
38    from sphinx.domains.python import PyClassmember as PyMethod
39    from sphinx.domains.python import PyModulelevel as PyFunction
40
41# Support for checking for suspicious markup
42
43import suspicious
44
45
46ISSUE_URI = 'https://bugs.python.org/issue%s'
47SOURCE_URI = 'https://github.com/python/cpython/tree/3.9/%s'
48
49# monkey-patch reST parser to disable alphabetic and roman enumerated lists
50from docutils.parsers.rst.states import Body
51Body.enum.converters['loweralpha'] = \
52    Body.enum.converters['upperalpha'] = \
53    Body.enum.converters['lowerroman'] = \
54    Body.enum.converters['upperroman'] = lambda x: None
55
56
57# Support for marking up and linking to bugs.python.org issues
58
59def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
60    issue = utils.unescape(text)
61    text = 'bpo-' + issue
62    refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue)
63    return [refnode], []
64
65
66# Support for linking to Python source files easily
67
68def source_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
69    has_t, title, target = split_explicit_title(text)
70    title = utils.unescape(title)
71    target = utils.unescape(target)
72    refnode = nodes.reference(title, title, refuri=SOURCE_URI % target)
73    return [refnode], []
74
75
76# Support for marking up implementation details
77
78class ImplementationDetail(Directive):
79
80    has_content = True
81    required_arguments = 0
82    optional_arguments = 1
83    final_argument_whitespace = True
84
85    # This text is copied to templates/dummy.html
86    label_text = 'CPython implementation detail:'
87
88    def run(self):
89        pnode = nodes.compound(classes=['impl-detail'])
90        label = translators['sphinx'].gettext(self.label_text)
91        content = self.content
92        add_text = nodes.strong(label, label)
93        if self.arguments:
94            n, m = self.state.inline_text(self.arguments[0], self.lineno)
95            pnode.append(nodes.paragraph('', '', *(n + m)))
96        self.state.nested_parse(content, self.content_offset, pnode)
97        if pnode.children and isinstance(pnode[0], nodes.paragraph):
98            content = nodes.inline(pnode[0].rawsource, translatable=True)
99            content.source = pnode[0].source
100            content.line = pnode[0].line
101            content += pnode[0].children
102            pnode[0].replace_self(nodes.paragraph('', '', content,
103                                                  translatable=False))
104            pnode[0].insert(0, add_text)
105            pnode[0].insert(1, nodes.Text(' '))
106        else:
107            pnode.insert(0, nodes.paragraph('', '', add_text))
108        return [pnode]
109
110
111# Support for documenting platform availability
112
113class Availability(Directive):
114
115    has_content = False
116    required_arguments = 1
117    optional_arguments = 0
118    final_argument_whitespace = True
119
120    def run(self):
121        availability_ref = ':ref:`Availability <availability>`: '
122        pnode = nodes.paragraph(availability_ref + self.arguments[0],
123                                classes=["availability"],)
124        n, m = self.state.inline_text(availability_ref, self.lineno)
125        pnode.extend(n + m)
126        n, m = self.state.inline_text(self.arguments[0], self.lineno)
127        pnode.extend(n + m)
128        return [pnode]
129
130
131# Support for documenting audit event
132
133class AuditEvent(Directive):
134
135    has_content = True
136    required_arguments = 1
137    optional_arguments = 2
138    final_argument_whitespace = True
139
140    _label = [
141        "Raises an :ref:`auditing event <auditing>` {name} with no arguments.",
142        "Raises an :ref:`auditing event <auditing>` {name} with argument {args}.",
143        "Raises an :ref:`auditing event <auditing>` {name} with arguments {args}.",
144    ]
145
146    @property
147    def logger(self):
148        cls = type(self)
149        return logging.getLogger(cls.__module__ + "." + cls.__name__)
150
151    def run(self):
152        name = self.arguments[0]
153        if len(self.arguments) >= 2 and self.arguments[1]:
154            args = (a.strip() for a in self.arguments[1].strip("'\"").split(","))
155            args = [a for a in args if a]
156        else:
157            args = []
158
159        label = translators['sphinx'].gettext(self._label[min(2, len(args))])
160        text = label.format(name="``{}``".format(name),
161                            args=", ".join("``{}``".format(a) for a in args if a))
162
163        env = self.state.document.settings.env
164        if not hasattr(env, 'all_audit_events'):
165            env.all_audit_events = {}
166
167        new_info = {
168            'source': [],
169            'args': args
170        }
171        info = env.all_audit_events.setdefault(name, new_info)
172        if info is not new_info:
173            if not self._do_args_match(info['args'], new_info['args']):
174                self.logger.warn(
175                    "Mismatched arguments for audit-event {}: {!r} != {!r}"
176                    .format(name, info['args'], new_info['args'])
177                )
178
179        ids = []
180        try:
181            target = self.arguments[2].strip("\"'")
182        except (IndexError, TypeError):
183            target = None
184        if not target:
185            target = "audit_event_{}_{}".format(
186                re.sub(r'\W', '_', name),
187                len(info['source']),
188            )
189            ids.append(target)
190
191        info['source'].append((env.docname, target))
192
193        pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids)
194        if self.content:
195            self.state.nested_parse(self.content, self.content_offset, pnode)
196        else:
197            n, m = self.state.inline_text(text, self.lineno)
198            pnode.extend(n + m)
199
200        return [pnode]
201
202    # This list of sets are allowable synonyms for event argument names.
203    # If two names are in the same set, they are treated as equal for the
204    # purposes of warning. This won't help if number of arguments is
205    # different!
206    _SYNONYMS = [
207        {"file", "path", "fd"},
208    ]
209
210    def _do_args_match(self, args1, args2):
211        if args1 == args2:
212            return True
213        if len(args1) != len(args2):
214            return False
215        for a1, a2 in zip(args1, args2):
216            if a1 == a2:
217                continue
218            if any(a1 in s and a2 in s for s in self._SYNONYMS):
219                continue
220            return False
221        return True
222
223
224class audit_event_list(nodes.General, nodes.Element):
225    pass
226
227
228class AuditEventListDirective(Directive):
229
230    def run(self):
231        return [audit_event_list('')]
232
233
234# Support for documenting decorators
235
236class PyDecoratorMixin(object):
237    def handle_signature(self, sig, signode):
238        ret = super(PyDecoratorMixin, self).handle_signature(sig, signode)
239        signode.insert(0, addnodes.desc_addname('@', '@'))
240        return ret
241
242    def needs_arglist(self):
243        return False
244
245
246class PyDecoratorFunction(PyDecoratorMixin, PyFunction):
247    def run(self):
248        # a decorator function is a function after all
249        self.name = 'py:function'
250        return PyFunction.run(self)
251
252
253# TODO: Use sphinx.domains.python.PyDecoratorMethod when possible
254class PyDecoratorMethod(PyDecoratorMixin, PyMethod):
255    def run(self):
256        self.name = 'py:method'
257        return PyMethod.run(self)
258
259
260class PyCoroutineMixin(object):
261    def handle_signature(self, sig, signode):
262        ret = super(PyCoroutineMixin, self).handle_signature(sig, signode)
263        signode.insert(0, addnodes.desc_annotation('coroutine ', 'coroutine '))
264        return ret
265
266
267class PyAwaitableMixin(object):
268    def handle_signature(self, sig, signode):
269        ret = super(PyAwaitableMixin, self).handle_signature(sig, signode)
270        signode.insert(0, addnodes.desc_annotation('awaitable ', 'awaitable '))
271        return ret
272
273
274class PyCoroutineFunction(PyCoroutineMixin, PyFunction):
275    def run(self):
276        self.name = 'py:function'
277        return PyFunction.run(self)
278
279
280class PyCoroutineMethod(PyCoroutineMixin, PyMethod):
281    def run(self):
282        self.name = 'py:method'
283        return PyMethod.run(self)
284
285
286class PyAwaitableFunction(PyAwaitableMixin, PyFunction):
287    def run(self):
288        self.name = 'py:function'
289        return PyFunction.run(self)
290
291
292class PyAwaitableMethod(PyAwaitableMixin, PyMethod):
293    def run(self):
294        self.name = 'py:method'
295        return PyMethod.run(self)
296
297
298class PyAbstractMethod(PyMethod):
299
300    def handle_signature(self, sig, signode):
301        ret = super(PyAbstractMethod, self).handle_signature(sig, signode)
302        signode.insert(0, addnodes.desc_annotation('abstractmethod ',
303                                                   'abstractmethod '))
304        return ret
305
306    def run(self):
307        self.name = 'py:method'
308        return PyMethod.run(self)
309
310
311# Support for documenting version of removal in deprecations
312
313class DeprecatedRemoved(Directive):
314    has_content = True
315    required_arguments = 2
316    optional_arguments = 1
317    final_argument_whitespace = True
318    option_spec = {}
319
320    _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}'
321    _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}'
322
323    def run(self):
324        node = addnodes.versionmodified()
325        node.document = self.state.document
326        node['type'] = 'deprecated-removed'
327        version = (self.arguments[0], self.arguments[1])
328        node['version'] = version
329        env = self.state.document.settings.env
330        current_version = tuple(int(e) for e in env.config.version.split('.'))
331        removed_version = tuple(int(e) for e in self.arguments[1].split('.'))
332        if current_version < removed_version:
333            label = self._deprecated_label
334        else:
335            label = self._removed_label
336
337        label = translators['sphinx'].gettext(label)
338        text = label.format(deprecated=self.arguments[0], removed=self.arguments[1])
339        if len(self.arguments) == 3:
340            inodes, messages = self.state.inline_text(self.arguments[2],
341                                                      self.lineno+1)
342            para = nodes.paragraph(self.arguments[2], '', *inodes, translatable=False)
343            node.append(para)
344        else:
345            messages = []
346        if self.content:
347            self.state.nested_parse(self.content, self.content_offset, node)
348        if len(node):
349            if isinstance(node[0], nodes.paragraph) and node[0].rawsource:
350                content = nodes.inline(node[0].rawsource, translatable=True)
351                content.source = node[0].source
352                content.line = node[0].line
353                content += node[0].children
354                node[0].replace_self(nodes.paragraph('', '', content, translatable=False))
355            node[0].insert(0, nodes.inline('', '%s: ' % text,
356                                           classes=['versionmodified']))
357        else:
358            para = nodes.paragraph('', '',
359                                   nodes.inline('', '%s.' % text,
360                                                classes=['versionmodified']),
361                                   translatable=False)
362            node.append(para)
363        env = self.state.document.settings.env
364        env.get_domain('changeset').note_changeset(node)
365        return [node] + messages
366
367
368# Support for including Misc/NEWS
369
370issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)')
371whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$")
372
373
374class MiscNews(Directive):
375    has_content = False
376    required_arguments = 1
377    optional_arguments = 0
378    final_argument_whitespace = False
379    option_spec = {}
380
381    def run(self):
382        fname = self.arguments[0]
383        source = self.state_machine.input_lines.source(
384            self.lineno - self.state_machine.input_offset - 1)
385        source_dir = getenv('PY_MISC_NEWS_DIR')
386        if not source_dir:
387            source_dir = path.dirname(path.abspath(source))
388        fpath = path.join(source_dir, fname)
389        self.state.document.settings.record_dependencies.add(fpath)
390        try:
391            with io.open(fpath, encoding='utf-8') as fp:
392                content = fp.read()
393        except Exception:
394            text = 'The NEWS file is not available.'
395            node = nodes.strong(text, text)
396            return [node]
397        content = issue_re.sub(r'`bpo-\1 <https://bugs.python.org/issue\1>`__',
398                               content)
399        content = whatsnew_re.sub(r'\1', content)
400        # remove first 3 lines as they are the main heading
401        lines = ['.. default-role:: obj', ''] + content.splitlines()[3:]
402        self.state_machine.insert_input(lines, fname)
403        return []
404
405
406# Support for building "topic help" for pydoc
407
408pydoc_topic_labels = [
409    'assert', 'assignment', 'async', 'atom-identifiers', 'atom-literals',
410    'attribute-access', 'attribute-references', 'augassign', 'await',
411    'binary', 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object',
412    'bltin-null-object', 'bltin-type-objects', 'booleans',
413    'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound',
414    'context-managers', 'continue', 'conversions', 'customization', 'debugger',
415    'del', 'dict', 'dynamic-features', 'else', 'exceptions', 'execmodel',
416    'exprlists', 'floating', 'for', 'formatstrings', 'function', 'global',
417    'id-classes', 'identifiers', 'if', 'imaginary', 'import', 'in', 'integers',
418    'lambda', 'lists', 'naming', 'nonlocal', 'numbers', 'numeric-types',
419    'objects', 'operator-summary', 'pass', 'power', 'raise', 'return',
420    'sequence-types', 'shifting', 'slicings', 'specialattrs', 'specialnames',
421    'string-methods', 'strings', 'subscriptions', 'truth', 'try', 'types',
422    'typesfunctions', 'typesmapping', 'typesmethods', 'typesmodules',
423    'typesseq', 'typesseq-mutable', 'unary', 'while', 'with', 'yield'
424]
425
426
427class PydocTopicsBuilder(Builder):
428    name = 'pydoc-topics'
429
430    default_translator_class = TextTranslator
431
432    def init(self):
433        self.topics = {}
434        self.secnumbers = {}
435
436    def get_outdated_docs(self):
437        return 'all pydoc topics'
438
439    def get_target_uri(self, docname, typ=None):
440        return ''  # no URIs
441
442    def write(self, *ignored):
443        writer = TextWriter(self)
444        for label in status_iterator(pydoc_topic_labels,
445                                     'building topics... ',
446                                     length=len(pydoc_topic_labels)):
447            if label not in self.env.domaindata['std']['labels']:
448                self.env.logger.warn('label %r not in documentation' % label)
449                continue
450            docname, labelid, sectname = self.env.domaindata['std']['labels'][label]
451            doctree = self.env.get_and_resolve_doctree(docname, self)
452            document = new_document('<section node>')
453            document.append(doctree.ids[labelid])
454            destination = StringOutput(encoding='utf-8')
455            writer.write(document, destination)
456            self.topics[label] = writer.output
457
458    def finish(self):
459        f = open(path.join(self.outdir, 'topics.py'), 'wb')
460        try:
461            f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8'))
462            f.write(('# Autogenerated by Sphinx on %s\n' % asctime()).encode('utf-8'))
463            f.write(('topics = ' + pformat(self.topics) + '\n').encode('utf-8'))
464        finally:
465            f.close()
466
467
468# Support for documenting Opcodes
469
470opcode_sig_re = re.compile(r'(\w+(?:\+\d)?)(?:\s*\((.*)\))?')
471
472
473def parse_opcode_signature(env, sig, signode):
474    """Transform an opcode signature into RST nodes."""
475    m = opcode_sig_re.match(sig)
476    if m is None:
477        raise ValueError
478    opname, arglist = m.groups()
479    signode += addnodes.desc_name(opname, opname)
480    if arglist is not None:
481        paramlist = addnodes.desc_parameterlist()
482        signode += paramlist
483        paramlist += addnodes.desc_parameter(arglist, arglist)
484    return opname.strip()
485
486
487# Support for documenting pdb commands
488
489pdbcmd_sig_re = re.compile(r'([a-z()!]+)\s*(.*)')
490
491# later...
492# pdbargs_tokens_re = re.compile(r'''[a-zA-Z]+  |  # identifiers
493#                                   [.,:]+     |  # punctuation
494#                                   [\[\]()]   |  # parens
495#                                   \s+           # whitespace
496#                                   ''', re.X)
497
498
499def parse_pdb_command(env, sig, signode):
500    """Transform a pdb command signature into RST nodes."""
501    m = pdbcmd_sig_re.match(sig)
502    if m is None:
503        raise ValueError
504    name, args = m.groups()
505    fullname = name.replace('(', '').replace(')', '')
506    signode += addnodes.desc_name(name, name)
507    if args:
508        signode += addnodes.desc_addname(' '+args, ' '+args)
509    return fullname
510
511
512def process_audit_events(app, doctree, fromdocname):
513    for node in doctree.traverse(audit_event_list):
514        break
515    else:
516        return
517
518    env = app.builder.env
519
520    table = nodes.table(cols=3)
521    group = nodes.tgroup(
522        '',
523        nodes.colspec(colwidth=30),
524        nodes.colspec(colwidth=55),
525        nodes.colspec(colwidth=15),
526        cols=3,
527    )
528    head = nodes.thead()
529    body = nodes.tbody()
530
531    table += group
532    group += head
533    group += body
534
535    row = nodes.row()
536    row += nodes.entry('', nodes.paragraph('', nodes.Text('Audit event')))
537    row += nodes.entry('', nodes.paragraph('', nodes.Text('Arguments')))
538    row += nodes.entry('', nodes.paragraph('', nodes.Text('References')))
539    head += row
540
541    for name in sorted(getattr(env, "all_audit_events", ())):
542        audit_event = env.all_audit_events[name]
543
544        row = nodes.row()
545        node = nodes.paragraph('', nodes.Text(name))
546        row += nodes.entry('', node)
547
548        node = nodes.paragraph()
549        for i, a in enumerate(audit_event['args']):
550            if i:
551                node += nodes.Text(", ")
552            node += nodes.literal(a, nodes.Text(a))
553        row += nodes.entry('', node)
554
555        node = nodes.paragraph()
556        backlinks = enumerate(sorted(set(audit_event['source'])), start=1)
557        for i, (doc, label) in backlinks:
558            if isinstance(label, str):
559                ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True)
560                try:
561                    ref['refuri'] = "{}#{}".format(
562                        app.builder.get_relative_uri(fromdocname, doc),
563                        label,
564                    )
565                except NoUri:
566                    continue
567                node += ref
568        row += nodes.entry('', node)
569
570        body += row
571
572    for node in doctree.traverse(audit_event_list):
573        node.replace_self(table)
574
575
576def setup(app):
577    app.add_role('issue', issue_role)
578    app.add_role('source', source_role)
579    app.add_directive('impl-detail', ImplementationDetail)
580    app.add_directive('availability', Availability)
581    app.add_directive('audit-event', AuditEvent)
582    app.add_directive('audit-event-table', AuditEventListDirective)
583    app.add_directive('deprecated-removed', DeprecatedRemoved)
584    app.add_builder(PydocTopicsBuilder)
585    app.add_builder(suspicious.CheckSuspiciousMarkupBuilder)
586    app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature)
587    app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command)
588    app.add_object_type('2to3fixer', '2to3fixer', '%s (2to3 fixer)')
589    app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction)
590    app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod)
591    app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction)
592    app.add_directive_to_domain('py', 'coroutinemethod', PyCoroutineMethod)
593    app.add_directive_to_domain('py', 'awaitablefunction', PyAwaitableFunction)
594    app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod)
595    app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod)
596    app.add_directive('miscnews', MiscNews)
597    app.connect('doctree-resolved', process_audit_events)
598    return {'version': '1.0', 'parallel_read_safe': True}
599