• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Turn Python docstrings into Markdown for TensorFlow documentation."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21import ast
22import collections
23import functools
24import itertools
25import json
26import os
27import re
28
29import astor
30import six
31
32from google.protobuf.message import Message as ProtoMessage
33from tensorflow.python.platform import tf_logging as logging
34from tensorflow.python.util import tf_inspect
35from tensorflow.tools.docs import doc_controls
36
37
38def is_free_function(py_object, full_name, index):
39  """Check if input is a free function (and not a class- or static method).
40
41  Args:
42    py_object: The the object in question.
43    full_name: The full name of the object, like `tf.module.symbol`.
44    index: The {full_name:py_object} dictionary for the public API.
45
46  Returns:
47    True if the obeject is a stand-alone function, and not part of a class
48    definition.
49  """
50  if not tf_inspect.isfunction(py_object):
51    return False
52
53  parent_name = full_name.rsplit('.', 1)[0]
54  if tf_inspect.isclass(index[parent_name]):
55    return False
56
57  return True
58
59
60# A regular expression capturing a python identifier.
61IDENTIFIER_RE = r'[a-zA-Z_]\w*'
62
63
64class TFDocsError(Exception):
65  pass
66
67
68class _Errors(object):
69  """A collection of errors."""
70
71  def __init__(self):
72    self._errors = []
73
74  def log_all(self):
75    """Log all the collected errors to the standard error."""
76    template = 'ERROR:\n    output file name: %s\n    %s\n\n'
77
78    for full_name, message in self._errors:
79      logging.warn(template, full_name, message)
80
81  def append(self, full_name, message):
82    """Add an error to the collection.
83
84    Args:
85      full_name: The path to the file in which the error occurred.
86      message: The message to display with the error.
87    """
88    self._errors.append((full_name, message))
89
90  def __len__(self):
91    return len(self._errors)
92
93  def __eq__(self, other):
94    if not isinstance(other, _Errors):
95      return False
96    return self._errors == other._errors  # pylint: disable=protected-access
97
98
99def documentation_path(full_name, is_fragment=False):
100  """Returns the file path for the documentation for the given API symbol.
101
102  Given the fully qualified name of a library symbol, compute the path to which
103  to write the documentation for that symbol (relative to a base directory).
104  Documentation files are organized into directories that mirror the python
105  module/class structure.
106
107  Args:
108    full_name: Fully qualified name of a library symbol.
109    is_fragment: If `False` produce a direct markdown link (`tf.a.b.c` -->
110      `tf/a/b/c.md`). If `True` produce fragment link, `tf.a.b.c` -->
111      `tf/a/b.md#c`
112  Returns:
113    The file path to which to write the documentation for `full_name`.
114  """
115  parts = full_name.split('.')
116  if is_fragment:
117    parts, fragment = parts[:-1], parts[-1]
118
119  result = os.path.join(*parts) + '.md'
120
121  if is_fragment:
122    result = result + '#' + fragment
123
124  return result
125
126
127def _get_raw_docstring(py_object):
128  """Get the docs for a given python object.
129
130  Args:
131    py_object: A python object to retrieve the docs for (class, function/method,
132      or module).
133
134  Returns:
135    The docstring, or the empty string if no docstring was found.
136  """
137  # For object instances, tf_inspect.getdoc does give us the docstring of their
138  # type, which is not what we want. Only return the docstring if it is useful.
139  if (tf_inspect.isclass(py_object) or tf_inspect.ismethod(py_object) or
140      tf_inspect.isfunction(py_object) or tf_inspect.ismodule(py_object) or
141      isinstance(py_object, property)):
142    return tf_inspect.getdoc(py_object) or ''
143  else:
144    return ''
145
146
147# A regular expression for capturing a @{symbol} reference.
148SYMBOL_REFERENCE_RE = re.compile(
149    r"""
150    # Start with a literal "@{".
151    @\{
152      # Group at least 1 symbol, not "}".
153      ([^}]+)
154    # Followed by a closing "}"
155    \}
156    """,
157    flags=re.VERBOSE)
158
159AUTO_REFERENCE_RE = re.compile(r'`([a-zA-Z0-9_.]+?)`')
160
161
162class ReferenceResolver(object):
163  """Class for replacing @{...} references with Markdown links.
164
165  Attributes:
166    current_doc_full_name: A string (or None) indicating the name of the
167      document currently being processed, so errors can reference the broken
168      doc.
169  """
170
171  def __init__(self, duplicate_of, doc_index, is_fragment, py_module_names):
172    """Initializes a Reference Resolver.
173
174    Args:
175      duplicate_of: A map from duplicate names to preferred names of API
176        symbols.
177      doc_index: A `dict` mapping symbol name strings to objects with `url`
178        and `title` fields. Used to resolve @{$doc} references in docstrings.
179      is_fragment: A map from full names to bool for each symbol. If True the
180        object lives at a page fragment `tf.a.b.c` --> `tf/a/b#c`. If False
181        object has a page to itself: `tf.a.b.c` --> `tf/a/b/c`.
182      py_module_names: A list of string names of Python modules.
183    """
184    self._duplicate_of = duplicate_of
185    self._doc_index = doc_index
186    self._is_fragment = is_fragment
187    self._all_names = set(is_fragment.keys())
188    self._py_module_names = py_module_names
189
190    self.current_doc_full_name = None
191    self._errors = _Errors()
192
193  def add_error(self, message):
194    self._errors.append(self.current_doc_full_name, message)
195
196  def log_errors(self):
197    self._errors.log_all()
198
199  def num_errors(self):
200    return len(self._errors)
201
202  @classmethod
203  def from_visitor(cls, visitor, doc_index, **kwargs):
204    """A factory function for building a ReferenceResolver from a visitor.
205
206    Args:
207      visitor: an instance of `DocGeneratorVisitor`
208      doc_index: a dictionary mapping document names to references objects with
209        "title" and "url" fields
210      **kwargs: all remaining args are passed to the constructor
211    Returns:
212      an instance of `ReferenceResolver` ()
213    """
214    is_fragment = {}
215    for name, obj in visitor.index.items():
216      has_page = (
217          tf_inspect.isclass(obj) or tf_inspect.ismodule(obj) or
218          is_free_function(obj, name, visitor.index))
219
220      is_fragment[name] = not has_page
221
222    return cls(
223        duplicate_of=visitor.duplicate_of,
224        doc_index=doc_index,
225        is_fragment=is_fragment,
226        **kwargs)
227
228  @classmethod
229  def from_json_file(cls, filepath, doc_index):
230    with open(filepath) as f:
231      json_dict = json.load(f)
232
233    return cls(doc_index=doc_index, **json_dict)
234
235  def to_json_file(self, filepath):
236    """Converts the RefenceResolver to json and writes it to the specified file.
237
238    Args:
239      filepath: The file path to write the json to.
240    """
241    try:
242      os.makedirs(os.path.dirname(filepath))
243    except OSError:
244      pass
245    json_dict = {}
246    for key, value in self.__dict__.items():
247      # Drop these two fields. `_doc_index` is not serializable. `_all_names` is
248      # generated by the constructor.
249      if key in ('_doc_index', '_all_names',
250                 '_errors', 'current_doc_full_name'):
251        continue
252
253      # Strip off any leading underscores on field names as these are not
254      # recognized by the constructor.
255      json_dict[key.lstrip('_')] = value
256
257    with open(filepath, 'w') as f:
258      json.dump(json_dict, f, indent=2, sort_keys=True)
259
260  def replace_references(self, string, relative_path_to_root):
261    """Replace "@{symbol}" references with links to symbol's documentation page.
262
263    This functions finds all occurrences of "@{symbol}" in `string`
264    and replaces them with markdown links to the documentation page
265    for "symbol".
266
267    `relative_path_to_root` is the relative path from the document
268    that contains the "@{symbol}" reference to the root of the API
269    documentation that is linked to. If the containing page is part of
270    the same API docset, `relative_path_to_root` can be set to
271    `os.path.dirname(documentation_path(name))`, where `name` is the
272    python name of the object whose documentation page the reference
273    lives on.
274
275    Args:
276      string: A string in which "@{symbol}" references should be replaced.
277      relative_path_to_root: The relative path from the containing document to
278        the root of the API documentation that is being linked to.
279
280    Returns:
281      `string`, with "@{symbol}" references replaced by Markdown links.
282    """
283
284    def strict_one_ref(match):
285      try:
286        return self._one_ref(match, relative_path_to_root)
287      except TFDocsError as e:
288        self.add_error(e.message)
289        return 'BAD_LINK'
290
291    string = re.sub(SYMBOL_REFERENCE_RE, strict_one_ref, string)
292
293    def sloppy_one_ref(match):
294      try:
295        return self._one_ref(match, relative_path_to_root)
296      except TFDocsError:
297        return match.group(0)
298
299    string = re.sub(AUTO_REFERENCE_RE, sloppy_one_ref, string)
300
301    return string
302
303  def python_link(self, link_text, ref_full_name, relative_path_to_root,
304                  code_ref=True):
305    """Resolve a "@{python symbol}" reference to a Markdown link.
306
307    This will pick the canonical location for duplicate symbols.  The
308    input to this function should already be stripped of the '@' and
309    '{}'.  This function returns a Markdown link. If `code_ref` is
310    true, it is assumed that this is a code reference, so the link
311    text will be rendered as code (using backticks).
312    `link_text` should refer to a library symbol, starting with 'tf.'.
313
314    Args:
315      link_text: The text of the Markdown link.
316      ref_full_name: The fully qualified name of the symbol to link to.
317      relative_path_to_root: The relative path from the location of the current
318        document to the root of the API documentation.
319      code_ref: If true (the default), put `link_text` in `...`.
320
321    Returns:
322      A markdown link to the documentation page of `ref_full_name`.
323    """
324    url = self.reference_to_url(ref_full_name, relative_path_to_root)
325
326    if code_ref:
327      link_text = link_text.join(['<code>', '</code>'])
328    else:
329      link_text = self._link_text_to_html(link_text)
330
331    return '<a href="{}">{}</a>'.format(url, link_text)
332
333  @staticmethod
334  def _link_text_to_html(link_text):
335    code_re = '`(.*?)`'
336    return re.sub(code_re, r'<code>\1</code>', link_text)
337
338  def py_master_name(self, full_name):
339    """Return the master name for a Python symbol name."""
340    return self._duplicate_of.get(full_name, full_name)
341
342  def reference_to_url(self, ref_full_name, relative_path_to_root):
343    """Resolve a "@{python symbol}" reference to a relative path.
344
345    The input to this function should already be stripped of the '@'
346    and '{}', and its output is only the link, not the full Markdown.
347
348    If `ref_full_name` is the name of a class member, method, or property, the
349    link will point to the page of the containing class, and it will include the
350    method name as an anchor. For example, `tf.module.MyClass.my_method` will be
351    translated into a link to
352    `os.join.path(relative_path_to_root, 'tf/module/MyClass.md#my_method')`.
353
354    Args:
355      ref_full_name: The fully qualified name of the symbol to link to.
356      relative_path_to_root: The relative path from the location of the current
357        document to the root of the API documentation.
358
359    Returns:
360      A relative path that links from the documentation page of `from_full_name`
361      to the documentation page of `ref_full_name`.
362
363    Raises:
364      RuntimeError: If `ref_full_name` is not documented.
365      TFDocsError: If the @{} syntax cannot be decoded.
366    """
367    master_name = self._duplicate_of.get(ref_full_name, ref_full_name)
368
369    # Check whether this link exists
370    if master_name not in self._all_names:
371      raise TFDocsError(
372          'Cannot make link to "%s": Not in index.' % master_name)
373
374    ref_path = documentation_path(master_name, self._is_fragment[master_name])
375    return os.path.join(relative_path_to_root, ref_path)
376
377  def _one_ref(self, match, relative_path_to_root):
378    """Return a link for a single "@{symbol}" reference."""
379    string = match.group(1)
380
381    # Look for link text after $.
382    dollar = string.rfind('$')
383    if dollar > 0:  # Ignore $ in first character
384      link_text = string[dollar + 1:]
385      string = string[:dollar]
386      manual_link_text = True
387    else:
388      link_text = string
389      manual_link_text = False
390
391    # Handle different types of references.
392    if string.startswith('$'):  # Doc reference
393      return self._doc_link(string, link_text, manual_link_text,
394                            relative_path_to_root)
395
396    elif string.startswith('tensorflow::'):
397      # C++ symbol
398      return self._cc_link(string, link_text, manual_link_text,
399                           relative_path_to_root)
400
401    else:
402      is_python = False
403      for py_module_name in self._py_module_names:
404        if string == py_module_name or string.startswith(py_module_name + '.'):
405          is_python = True
406          break
407      if is_python:  # Python symbol
408        return self.python_link(
409            link_text,
410            string,
411            relative_path_to_root,
412            code_ref=not manual_link_text)
413
414    # Error!
415    raise TFDocsError('Did not understand "%s"' % match.group(0),
416                      'BROKEN_LINK')
417
418  def _doc_link(self, string, link_text, manual_link_text,
419                relative_path_to_root):
420    """Generate a link for a @{$...} reference."""
421    string = string[1:]  # remove leading $
422
423    # If string has a #, split that part into `hash_tag`
424    hash_pos = string.find('#')
425    if hash_pos > -1:
426      hash_tag = string[hash_pos:]
427      string = string[:hash_pos]
428    else:
429      hash_tag = ''
430
431    if string in self._doc_index:
432      if not manual_link_text: link_text = self._doc_index[string].title
433      url = os.path.normpath(os.path.join(
434          relative_path_to_root, '../..', self._doc_index[string].url))
435      link_text = self._link_text_to_html(link_text)
436      return '<a href="{}{}">{}</a>'.format(url, hash_tag, link_text)
437
438    return self._doc_missing(string, hash_tag, link_text, manual_link_text,
439                             relative_path_to_root)
440
441  def _doc_missing(self, string, unused_hash_tag, unused_link_text,
442                   unused_manual_link_text, unused_relative_path_to_root):
443    """Generate an error for unrecognized @{$...} references."""
444    raise TFDocsError('Unknown Document "%s"' % string)
445
446  def _cc_link(self, string, link_text, unused_manual_link_text,
447               relative_path_to_root):
448    """Generate a link for a @{tensorflow::...} reference."""
449    # TODO(josh11b): Fix this hard-coding of paths.
450    if string == 'tensorflow::ClientSession':
451      ret = 'class/tensorflow/client-session.md'
452    elif string == 'tensorflow::Scope':
453      ret = 'class/tensorflow/scope.md'
454    elif string == 'tensorflow::Status':
455      ret = 'class/tensorflow/status.md'
456    elif string == 'tensorflow::Tensor':
457      ret = 'class/tensorflow/tensor.md'
458    elif string == 'tensorflow::ops::Const':
459      ret = 'namespace/tensorflow/ops.md#const'
460    else:
461      raise TFDocsError('C++ reference not understood: "%s"' % string)
462
463    # relative_path_to_root gets you to api_docs/python, we go from there
464    # to api_docs/cc, and then add ret.
465    cc_relative_path = os.path.normpath(os.path.join(
466        relative_path_to_root, '../cc', ret))
467
468    return '<a href="{}"><code>{}</code></a>'.format(cc_relative_path,
469                                                     link_text)
470
471
472# TODO(aselle): Collect these into a big list for all modules and functions
473# and make a rosetta stone page.
474def _handle_compatibility(doc):
475  """Parse and remove compatibility blocks from the main docstring.
476
477  Args:
478    doc: The docstring that contains compatibility notes"
479
480  Returns:
481    a tuple of the modified doc string and a hash that maps from compatibility
482    note type to the text of the note.
483  """
484  compatibility_notes = {}
485  match_compatibility = re.compile(r'[ \t]*@compatibility\((\w+)\)\s*\n'
486                                   r'((?:[^@\n]*\n)+)'
487                                   r'\s*@end_compatibility')
488  for f in match_compatibility.finditer(doc):
489    compatibility_notes[f.group(1)] = f.group(2)
490  return match_compatibility.subn(r'', doc)[0], compatibility_notes
491
492
493def _gen_pairs(items):
494  """Given an list of items [a,b,a,b...], generate pairs [(a,b),(a,b)...].
495
496  Args:
497    items: A list of items (length must be even)
498
499  Yields:
500    The original items, in pairs
501  """
502  assert len(items) % 2 == 0
503  items = iter(items)
504  while True:
505    try:
506      yield next(items), next(items)
507    except StopIteration:
508      return
509
510
511class _FunctionDetail(
512    collections.namedtuple('_FunctionDetail', ['keyword', 'header', 'items'])):
513  """A simple class to contain function details.
514
515  Composed of a "keyword", a possibly empty "header" string, and a possibly
516  empty
517  list of key-value pair "items".
518  """
519  __slots__ = []
520
521  def __str__(self):
522    """Return the original string that represents the function detail."""
523    parts = [self.keyword + ':\n']
524    parts.append(self.header)
525    for key, value in self.items:
526      parts.append('  ' + key + ': ')
527      parts.append(value)
528
529    return ''.join(parts)
530
531
532def _parse_function_details(docstring):
533  r"""Given a docstring, split off the header and parse the function details.
534
535  For example the docstring of tf.nn.relu:
536
537  '''Computes rectified linear: `max(features, 0)`.
538
539  Args:
540    features: A `Tensor`. Must be one of the following types: `float32`,
541      `float64`, `int32`, `int64`, `uint8`, `int16`, `int8`, `uint16`,
542      `half`.
543    name: A name for the operation (optional).
544
545  Returns:
546    A `Tensor`. Has the same type as `features`.
547  '''
548
549  This is parsed, and returned as:
550
551  ```
552  ('Computes rectified linear: `max(features, 0)`.\n\n', [
553      _FunctionDetail(
554          keyword='Args',
555          header='',
556          items=[
557              ('features', ' A `Tensor`. Must be ...'),
558              ('name', ' A name for the operation (optional).\n\n')]),
559      _FunctionDetail(
560          keyword='Returns',
561          header='  A `Tensor`. Has the same type as `features`.',
562          items=[])
563  ])
564  ```
565
566  Args:
567    docstring: The docstring to parse
568
569  Returns:
570    A (header, function_details) pair, where header is a string and
571    function_details is a (possibly empty) list of `_FunctionDetail` objects.
572  """
573
574  detail_keywords = '|'.join([
575      'Args', 'Arguments', 'Fields', 'Returns', 'Yields', 'Raises', 'Attributes'
576  ])
577  tag_re = re.compile('(?<=\n)(' + detail_keywords + '):\n', re.MULTILINE)
578  parts = tag_re.split(docstring)
579
580  # The first part is the main docstring
581  docstring = parts[0]
582
583  # Everything else alternates keyword-content
584  pairs = list(_gen_pairs(parts[1:]))
585
586  function_details = []
587  item_re = re.compile(r'^   ? ?(\*?\*?\w[\w.]*?\s*):\s', re.MULTILINE)
588
589  for keyword, content in pairs:
590    content = item_re.split(content)
591    header = content[0]
592    items = list(_gen_pairs(content[1:]))
593
594    function_details.append(_FunctionDetail(keyword, header, items))
595
596  return docstring, function_details
597
598
599_DocstringInfo = collections.namedtuple('_DocstringInfo', [
600    'brief', 'docstring', 'function_details', 'compatibility'
601])
602
603
604def _parse_md_docstring(py_object, relative_path_to_root, reference_resolver):
605  """Parse the object's docstring and return a `_DocstringInfo`.
606
607  This function clears @@'s from the docstring, and replaces @{} references
608  with markdown links.
609
610  For links within the same set of docs, the `relative_path_to_root` for a
611  docstring on the page for `full_name` can be set to:
612
613  ```python
614  relative_path_to_root = os.path.relpath(
615    path='.', start=os.path.dirname(documentation_path(full_name)) or '.')
616  ```
617
618  Args:
619    py_object: A python object to retrieve the docs for (class, function/method,
620      or module).
621    relative_path_to_root: The relative path from the location of the current
622      document to the root of the Python API documentation. This is used to
623      compute links for "@{symbol}" references.
624    reference_resolver: An instance of ReferenceResolver.
625
626  Returns:
627    A _DocstringInfo object, all fields will be empty if no docstring was found.
628  """
629  # TODO(wicke): If this is a partial, use the .func docstring and add a note.
630  raw_docstring = _get_raw_docstring(py_object)
631
632  raw_docstring = reference_resolver.replace_references(
633      raw_docstring, relative_path_to_root)
634
635  atat_re = re.compile(r' *@@[a-zA-Z_.0-9]+ *$')
636  raw_docstring = '\n'.join(
637      line for line in raw_docstring.split('\n') if not atat_re.match(line))
638
639  docstring, compatibility = _handle_compatibility(raw_docstring)
640  docstring, function_details = _parse_function_details(docstring)
641
642  if 'Generated by: tensorflow/tools/api/generator' in docstring:
643    docstring = ''
644
645  return _DocstringInfo(
646      docstring.split('\n')[0], docstring, function_details, compatibility)
647
648
649def _get_arg_spec(func):
650  """Extracts signature information from a function or functools.partial object.
651
652  For functions, uses `tf_inspect.getfullargspec`. For `functools.partial`
653  objects, corrects the signature of the underlying function to take into
654  account the removed arguments.
655
656  Args:
657    func: A function whose signature to extract.
658
659  Returns:
660    An `FullArgSpec` namedtuple `(args, varargs, varkw, defaults, etc.)`,
661    as returned by `tf_inspect.getfullargspec`.
662  """
663  # getfullargspec does not work for functools.partial objects directly.
664  if isinstance(func, functools.partial):
665    argspec = tf_inspect.getfullargspec(func.func)
666    # Remove the args from the original function that have been used up.
667    first_default_arg = (
668        len(argspec.args or []) - len(argspec.defaults or []))
669    partial_args = len(func.args)
670    argspec_args = []
671
672    if argspec.args:
673      argspec_args = list(argspec.args[partial_args:])
674
675    argspec_defaults = list(argspec.defaults or ())
676    if argspec.defaults and partial_args > first_default_arg:
677      argspec_defaults = list(argspec.defaults[partial_args-first_default_arg:])
678
679    first_default_arg = max(0, first_default_arg - partial_args)
680    for kwarg in (func.keywords or []):
681      if kwarg in (argspec.args or []):
682        i = argspec_args.index(kwarg)
683        argspec_args.pop(i)
684        if i >= first_default_arg:
685          argspec_defaults.pop(i-first_default_arg)
686        else:
687          first_default_arg -= 1
688    return tf_inspect.FullArgSpec(
689        args=argspec_args,
690        varargs=argspec.varargs,
691        varkw=argspec.varkw,
692        defaults=tuple(argspec_defaults),
693        kwonlyargs=[],
694        kwonlydefaults=None,
695        annotations={})
696  else:  # Regular function or method, getargspec will work fine.
697    return tf_inspect.getfullargspec(func)
698
699
700def _remove_first_line_indent(string):
701  indent = len(re.match(r'^\s*', string).group(0))
702  return '\n'.join([line[indent:] for line in string.split('\n')])
703
704
705PAREN_NUMBER_RE = re.compile(r'^\(([0-9.e-]+)\)')
706
707
708def _generate_signature(func, reverse_index):
709  """Given a function, returns a list of strings representing its args.
710
711  This function produces a list of strings representing the arguments to a
712  python function. It uses tf_inspect.getfullargspec, which
713  does not generalize well to Python 3.x, which is more flexible in how *args
714  and **kwargs are handled. This is not a problem in TF, since we have to remain
715  compatible to Python 2.7 anyway.
716
717  This function uses `__name__` for callables if it is available. This can lead
718  to poor results for functools.partial and other callable objects.
719
720  The returned string is Python code, so if it is included in a Markdown
721  document, it should be typeset as code (using backticks), or escaped.
722
723  Args:
724    func: A function, method, or functools.partial to extract the signature for.
725    reverse_index: A map from object ids to canonical full names to use.
726
727  Returns:
728    A list of strings representing the argument signature of `func` as python
729    code.
730  """
731
732  args_list = []
733
734  argspec = _get_arg_spec(func)
735  first_arg_with_default = (
736      len(argspec.args or []) - len(argspec.defaults or []))
737
738  # Python documentation skips `self` when printing method signatures.
739  # Note we cannot test for ismethod here since unbound methods do not register
740  # as methods (in Python 3).
741  first_arg = 1 if 'self' in argspec.args[:1] else 0
742
743  # Add all args without defaults.
744  for arg in argspec.args[first_arg:first_arg_with_default]:
745    args_list.append(arg)
746
747  # Add all args with defaults.
748  if argspec.defaults:
749    try:
750      source = _remove_first_line_indent(tf_inspect.getsource(func))
751      func_ast = ast.parse(source)
752      ast_defaults = func_ast.body[0].args.defaults
753    except IOError:  # If this is a builtin, getsource fails with IOError
754      # If we cannot get the source, assume the AST would be equal to the repr
755      # of the defaults.
756      ast_defaults = [None] * len(argspec.defaults)
757
758    for arg, default, ast_default in zip(
759        argspec.args[first_arg_with_default:], argspec.defaults, ast_defaults):
760      if id(default) in reverse_index:
761        default_text = reverse_index[id(default)]
762      elif ast_default is not None:
763        default_text = (
764            astor.to_source(ast_default).rstrip('\n').replace('\t', '\\t')
765            .replace('\n', '\\n').replace('"""', "'"))
766        default_text = PAREN_NUMBER_RE.sub('\\1', default_text)
767
768        if default_text != repr(default):
769          # This may be an internal name. If so, handle the ones we know about.
770          # TODO(wicke): This should be replaced with a lookup in the index.
771          # TODO(wicke): (replace first ident with tf., check if in index)
772          internal_names = {
773              'ops.GraphKeys': 'tf.GraphKeys',
774              '_ops.GraphKeys': 'tf.GraphKeys',
775              'init_ops.zeros_initializer': 'tf.zeros_initializer',
776              'init_ops.ones_initializer': 'tf.ones_initializer',
777              'saver_pb2.SaverDef': 'tf.train.SaverDef',
778          }
779          full_name_re = '^%s(.%s)+' % (IDENTIFIER_RE, IDENTIFIER_RE)
780          match = re.match(full_name_re, default_text)
781          if match:
782            lookup_text = default_text
783            for internal_name, public_name in six.iteritems(internal_names):
784              if match.group(0).startswith(internal_name):
785                lookup_text = public_name + default_text[len(internal_name):]
786                break
787            if default_text is lookup_text:
788              logging.warn(
789                  'WARNING: Using default arg, failed lookup: %s, repr: %r',
790                  default_text, default)
791            else:
792              default_text = lookup_text
793      else:
794        default_text = repr(default)
795
796      args_list.append('%s=%s' % (arg, default_text))
797
798  # Add *args and *kwargs.
799  if argspec.varargs:
800    args_list.append('*' + argspec.varargs)
801  if argspec.varkw:
802    args_list.append('**' + argspec.varkw)
803
804  return args_list
805
806
807def _get_guides_markdown(duplicate_names, guide_index, relative_path):
808  all_guides = []
809  for name in duplicate_names:
810    all_guides.extend(guide_index.get(name, []))
811  if not all_guides: return ''
812  prefix = '../' * (relative_path.count('/') + 3)
813  links = sorted(set([guide_ref.make_md_link(prefix)
814                      for guide_ref in all_guides]))
815  return 'See the guide%s: %s\n\n' % (
816      's' if len(links) > 1 else '', ', '.join(links))
817
818
819def _get_defining_class(py_class, name):
820  for cls in tf_inspect.getmro(py_class):
821    if name in cls.__dict__:
822      return cls
823  return None
824
825
826class _LinkInfo(
827    collections.namedtuple(
828        '_LinkInfo', ['short_name', 'full_name', 'obj', 'doc', 'url'])):
829
830  __slots__ = []
831
832  def is_link(self):
833    return True
834
835
836class _OtherMemberInfo(
837    collections.namedtuple('_OtherMemberInfo',
838                           ['short_name', 'full_name', 'obj', 'doc'])):
839
840  __slots__ = []
841
842  def is_link(self):
843    return False
844
845
846_PropertyInfo = collections.namedtuple(
847    '_PropertyInfo', ['short_name', 'full_name', 'obj', 'doc'])
848
849_MethodInfo = collections.namedtuple('_MethodInfo', [
850    'short_name', 'full_name', 'obj', 'doc', 'signature', 'decorators'
851])
852
853
854class _FunctionPageInfo(object):
855  """Collects docs For a function Page."""
856
857  def __init__(self, full_name):
858    self._full_name = full_name
859    self._defined_in = None
860    self._aliases = None
861    self._doc = None
862    self._guides = None
863
864    self._signature = None
865    self._decorators = []
866
867  def for_function(self):
868    return True
869
870  def for_class(self):
871    return False
872
873  def for_module(self):
874    return False
875
876  @property
877  def full_name(self):
878    return self._full_name
879
880  @property
881  def short_name(self):
882    return self._full_name.split('.')[-1]
883
884  @property
885  def defined_in(self):
886    return self._defined_in
887
888  def set_defined_in(self, defined_in):
889    assert self.defined_in is None
890    self._defined_in = defined_in
891
892  @property
893  def aliases(self):
894    return self._aliases
895
896  def set_aliases(self, aliases):
897    assert self.aliases is None
898    self._aliases = aliases
899
900  @property
901  def doc(self):
902    return self._doc
903
904  def set_doc(self, doc):
905    assert self.doc is None
906    self._doc = doc
907
908  @property
909  def guides(self):
910    return self._guides
911
912  def set_guides(self, guides):
913    assert self.guides is None
914    self._guides = guides
915
916  @property
917  def signature(self):
918    return self._signature
919
920  def set_signature(self, function, reverse_index):
921    """Attach the function's signature.
922
923    Args:
924      function: The python function being documented.
925      reverse_index: A map from object ids in the index to full names.
926    """
927
928    assert self.signature is None
929    self._signature = _generate_signature(function, reverse_index)
930
931  @property
932  def decorators(self):
933    return list(self._decorators)
934
935  def add_decorator(self, dec):
936    self._decorators.append(dec)
937
938  def get_metadata_html(self):
939    return _Metadata(self.full_name).build_html()
940
941
942class _ClassPageInfo(object):
943  """Collects docs for a class page.
944
945  Attributes:
946    full_name: The fully qualified name of the object at the master
947      location. Aka `master_name`. For example: `tf.nn.sigmoid`.
948    short_name: The last component of the `full_name`. For example: `sigmoid`.
949    defined_in: The path to the file where this object is defined.
950    aliases: The list of all fully qualified names for the locations where the
951      object is visible in the public api. This includes the master location.
952    doc: A `_DocstringInfo` object representing the object's docstring (can be
953      created with `_parse_md_docstring`).
954    guides: A markdown string, of back links pointing to the api_guides that
955      reference this object.
956    bases: A list of `_LinkInfo` objects pointing to the docs for the parent
957      classes.
958    properties: A list of `_PropertyInfo` objects documenting the class'
959      properties (attributes that use `@property`).
960    methods: A list of `_MethodInfo` objects documenting the class' methods.
961    classes: A list of `_LinkInfo` objects pointing to docs for any nested
962      classes.
963    other_members: A list of `_OtherMemberInfo` objects documenting any other
964      object's defined inside the class object (mostly enum style fields).
965  """
966
967  def __init__(self, full_name):
968    self._full_name = full_name
969    self._defined_in = None
970    self._aliases = None
971    self._doc = None
972    self._guides = None
973    self._namedtuplefields = None
974
975    self._bases = None
976    self._properties = []
977    self._methods = []
978    self._classes = []
979    self._other_members = []
980
981  def for_function(self):
982    """Returns true if this object documents a function."""
983    return False
984
985  def for_class(self):
986    """Returns true if this object documents a class."""
987    return True
988
989  def for_module(self):
990    """Returns true if this object documents a module."""
991    return False
992
993  @property
994  def full_name(self):
995    """Returns the documented object's fully qualified name."""
996    return self._full_name
997
998  @property
999  def short_name(self):
1000    """Returns the documented object's short name."""
1001    return self._full_name.split('.')[-1]
1002
1003  @property
1004  def defined_in(self):
1005    """Returns the path to the file where the documented object is defined."""
1006    return self._defined_in
1007
1008  def set_defined_in(self, defined_in):
1009    """Sets the `defined_in` path."""
1010    assert self.defined_in is None
1011    self._defined_in = defined_in
1012
1013  @property
1014  def aliases(self):
1015    """Returns a list of all full names for the documented object."""
1016    return self._aliases
1017
1018  def set_aliases(self, aliases):
1019    """Sets the `aliases` list.
1020
1021    Args:
1022      aliases: A list of strings. Containing all the object's full names.
1023    """
1024    assert self.aliases is None
1025    self._aliases = aliases
1026
1027  @property
1028  def doc(self):
1029    """Returns a `_DocstringInfo` created from the object's docstring."""
1030    return self._doc
1031
1032  def set_doc(self, doc):
1033    """Sets the `doc` field.
1034
1035    Args:
1036      doc: An instance of `_DocstringInfo`.
1037    """
1038    assert self.doc is None
1039    self._doc = doc
1040
1041  @property
1042  def guides(self):
1043    """Returns a markdown string containing backlinks to relevant api_guides."""
1044    return self._guides
1045
1046  def set_guides(self, guides):
1047    """Sets the `guides` field.
1048
1049    Args:
1050      guides: A markdown string containing backlinks to all the api_guides that
1051        link to the documented object.
1052    """
1053    assert self.guides is None
1054    self._guides = guides
1055
1056  @property
1057  def namedtuplefields(self):
1058    return self._namedtuplefields
1059
1060  def set_namedtuplefields(self, py_class):
1061    if issubclass(py_class, tuple):
1062      if all(
1063          hasattr(py_class, attr)
1064          for attr in ('_asdict', '_fields', '_make', '_replace')):
1065        self._namedtuplefields = py_class._fields
1066
1067  @property
1068  def bases(self):
1069    """Returns a list of `_LinkInfo` objects pointing to the class' parents."""
1070    return self._bases
1071
1072  def _set_bases(self, relative_path, parser_config):
1073    """Builds the `bases` attribute, to document this class' parent-classes.
1074
1075    This method sets the `bases` to a list of `_LinkInfo` objects point to the
1076    doc pages for the class' parents.
1077
1078    Args:
1079      relative_path: The relative path from the doc this object describes to
1080        the documentation root.
1081      parser_config: An instance of `ParserConfig`.
1082    """
1083    bases = []
1084    obj = parser_config.py_name_to_object(self.full_name)
1085    for base in obj.__bases__:
1086      base_full_name = parser_config.reverse_index.get(id(base), None)
1087      if base_full_name is None:
1088        continue
1089      base_doc = _parse_md_docstring(base, relative_path,
1090                                     parser_config.reference_resolver)
1091      base_url = parser_config.reference_resolver.reference_to_url(
1092          base_full_name, relative_path)
1093
1094      link_info = _LinkInfo(short_name=base_full_name.split('.')[-1],
1095                            full_name=base_full_name, obj=base,
1096                            doc=base_doc, url=base_url)
1097      bases.append(link_info)
1098
1099    self._bases = bases
1100
1101  @property
1102  def properties(self):
1103    """Returns a list of `_PropertyInfo` describing the class' properties."""
1104    props_dict = {prop.short_name: prop for prop in self._properties}
1105    props = []
1106    if self.namedtuplefields:
1107      for field in self.namedtuplefields:
1108        props.append(props_dict.pop(field))
1109
1110    props.extend(sorted(props_dict.values()))
1111
1112    return props
1113
1114  def _add_property(self, short_name, full_name, obj, doc):
1115    """Adds a `_PropertyInfo` entry to the `properties` list.
1116
1117    Args:
1118      short_name: The property's short name.
1119      full_name: The property's fully qualified name.
1120      obj: The property object itself
1121      doc: The property's parsed docstring, a `_DocstringInfo`.
1122    """
1123    # Hide useless namedtuple docs-trings
1124    if re.match('Alias for field number [0-9]+', doc.docstring):
1125      doc = doc._replace(docstring='', brief='')
1126    property_info = _PropertyInfo(short_name, full_name, obj, doc)
1127    self._properties.append(property_info)
1128
1129  @property
1130  def methods(self):
1131    """Returns a list of `_MethodInfo` describing the class' methods."""
1132    return self._methods
1133
1134  def _add_method(self, short_name, full_name, obj, doc, signature, decorators):
1135    """Adds a `_MethodInfo` entry to the `methods` list.
1136
1137    Args:
1138      short_name: The method's short name.
1139      full_name: The method's fully qualified name.
1140      obj: The method object itself
1141      doc: The method's parsed docstring, a `_DocstringInfo`
1142      signature: The method's parsed signature (see: `_generate_signature`)
1143      decorators: A list of strings describing the decorators that should be
1144        mentioned on the object's docs page.
1145    """
1146
1147    method_info = _MethodInfo(short_name, full_name, obj, doc, signature,
1148                              decorators)
1149
1150    self._methods.append(method_info)
1151
1152  @property
1153  def classes(self):
1154    """Returns a list of `_LinkInfo` pointing to any nested classes."""
1155    return self._classes
1156
1157  def get_metadata_html(self):
1158    meta_data = _Metadata(self.full_name)
1159    for item in itertools.chain(self.classes, self.properties, self.methods,
1160                                self.other_members):
1161      meta_data.append(item)
1162
1163    return meta_data.build_html()
1164
1165  def _add_class(self, short_name, full_name, obj, doc, url):
1166    """Adds a `_LinkInfo` for a nested class to `classes` list.
1167
1168    Args:
1169      short_name: The class' short name.
1170      full_name: The class' fully qualified name.
1171      obj: The class object itself
1172      doc: The class' parsed docstring, a `_DocstringInfo`
1173      url: A url pointing to where the nested class is documented.
1174    """
1175    page_info = _LinkInfo(short_name, full_name, obj, doc, url)
1176
1177    self._classes.append(page_info)
1178
1179  @property
1180  def other_members(self):
1181    """Returns a list of `_OtherMemberInfo` describing any other contents."""
1182    return self._other_members
1183
1184  def _add_other_member(self, short_name, full_name, obj, doc):
1185    """Adds an `_OtherMemberInfo` entry to the `other_members` list.
1186
1187    Args:
1188      short_name: The class' short name.
1189      full_name: The class' fully qualified name.
1190      obj: The class object itself
1191      doc: The class' parsed docstring, a `_DocstringInfo`
1192    """
1193    other_member_info = _OtherMemberInfo(short_name, full_name, obj, doc)
1194    self._other_members.append(other_member_info)
1195
1196  def collect_docs_for_class(self, py_class, parser_config):
1197    """Collects information necessary specifically for a class's doc page.
1198
1199    Mainly, this is details about the class's members.
1200
1201    Args:
1202      py_class: The class object being documented
1203      parser_config: An instance of ParserConfig.
1204    """
1205    self.set_namedtuplefields(py_class)
1206    doc_path = documentation_path(self.full_name)
1207    relative_path = os.path.relpath(
1208        path='.', start=os.path.dirname(doc_path) or '.')
1209
1210    self._set_bases(relative_path, parser_config)
1211
1212    for short_name in parser_config.tree[self.full_name]:
1213      # Remove builtin members that we never want to document.
1214      if short_name in [
1215          '__class__', '__base__', '__weakref__', '__doc__', '__module__',
1216          '__dict__', '__abstractmethods__', '__slots__', '__getnewargs__',
1217          '__str__', '__repr__', '__hash__', '__reduce__'
1218      ]:
1219        continue
1220
1221      child_name = '.'.join([self.full_name, short_name])
1222      child = parser_config.py_name_to_object(child_name)
1223
1224      # Don't document anything that is defined in object or by protobuf.
1225      defining_class = _get_defining_class(py_class, short_name)
1226      if defining_class in [object, type, tuple, BaseException, Exception]:
1227        continue
1228
1229      # The following condition excludes most protobuf-defined symbols.
1230      if (defining_class and
1231          defining_class.__name__ in ['CMessage', 'Message', 'MessageMeta']):
1232        continue
1233      # TODO(markdaoust): Add a note in child docs showing the defining class.
1234
1235      if doc_controls.should_skip_class_attr(py_class, short_name):
1236        continue
1237
1238      child_doc = _parse_md_docstring(child, relative_path,
1239                                      parser_config.reference_resolver)
1240
1241      if isinstance(child, property):
1242        self._add_property(short_name, child_name, child, child_doc)
1243
1244      elif tf_inspect.isclass(child):
1245        if defining_class is None:
1246          continue
1247        url = parser_config.reference_resolver.reference_to_url(
1248            child_name, relative_path)
1249        self._add_class(short_name, child_name, child, child_doc, url)
1250
1251      elif (tf_inspect.ismethod(child) or tf_inspect.isfunction(child) or
1252            tf_inspect.isroutine(child)):
1253        if defining_class is None:
1254          continue
1255
1256        # Omit methods defined by namedtuple.
1257        original_method = defining_class.__dict__[short_name]
1258        if (hasattr(original_method, '__module__') and
1259            (original_method.__module__ or '').startswith('namedtuple')):
1260          continue
1261
1262        # Some methods are often overridden without documentation. Because it's
1263        # obvious what they do, don't include them in the docs if there's no
1264        # docstring.
1265        if not child_doc.brief.strip() and short_name in [
1266            '__del__', '__copy__'
1267        ]:
1268          continue
1269
1270        try:
1271          child_signature = _generate_signature(child,
1272                                                parser_config.reverse_index)
1273        except TypeError:
1274          # If this is a (dynamically created) slot wrapper, tf_inspect will
1275          # raise typeerror when trying to get to the code. Ignore such
1276          # functions.
1277          continue
1278
1279        child_decorators = []
1280        try:
1281          if isinstance(py_class.__dict__[short_name], classmethod):
1282            child_decorators.append('classmethod')
1283        except KeyError:
1284          pass
1285
1286        try:
1287          if isinstance(py_class.__dict__[short_name], staticmethod):
1288            child_decorators.append('staticmethod')
1289        except KeyError:
1290          pass
1291
1292        self._add_method(short_name, child_name, child, child_doc,
1293                         child_signature, child_decorators)
1294      else:
1295        # Exclude members defined by protobuf that are useless
1296        if issubclass(py_class, ProtoMessage):
1297          if (short_name.endswith('_FIELD_NUMBER') or
1298              short_name in ['__slots__', 'DESCRIPTOR']):
1299            continue
1300
1301        # TODO(wicke): We may want to also remember the object itself.
1302        self._add_other_member(short_name, child_name, child, child_doc)
1303
1304
1305class _ModulePageInfo(object):
1306  """Collects docs for a module page."""
1307
1308  def __init__(self, full_name):
1309    self._full_name = full_name
1310    self._defined_in = None
1311    self._aliases = None
1312    self._doc = None
1313    self._guides = None
1314
1315    self._modules = []
1316    self._classes = []
1317    self._functions = []
1318    self._other_members = []
1319
1320  def for_function(self):
1321    return False
1322
1323  def for_class(self):
1324    return False
1325
1326  def for_module(self):
1327    return True
1328
1329  @property
1330  def full_name(self):
1331    return self._full_name
1332
1333  @property
1334  def short_name(self):
1335    return self._full_name.split('.')[-1]
1336
1337  @property
1338  def defined_in(self):
1339    return self._defined_in
1340
1341  def set_defined_in(self, defined_in):
1342    assert self.defined_in is None
1343    self._defined_in = defined_in
1344
1345  @property
1346  def aliases(self):
1347    return self._aliases
1348
1349  def set_aliases(self, aliases):
1350    assert self.aliases is None
1351    self._aliases = aliases
1352
1353  @property
1354  def doc(self):
1355    return self._doc
1356
1357  def set_doc(self, doc):
1358    assert self.doc is None
1359    self._doc = doc
1360
1361  @property
1362  def guides(self):
1363    return self._guides
1364
1365  def set_guides(self, guides):
1366    assert self.guides is None
1367    self._guides = guides
1368
1369  @property
1370  def modules(self):
1371    return self._modules
1372
1373  def _add_module(self, short_name, full_name, obj, doc, url):
1374    self._modules.append(_LinkInfo(short_name, full_name, obj, doc, url))
1375
1376  @property
1377  def classes(self):
1378    return self._classes
1379
1380  def _add_class(self, short_name, full_name, obj, doc, url):
1381    self._classes.append(_LinkInfo(short_name, full_name, obj, doc, url))
1382
1383  @property
1384  def functions(self):
1385    return self._functions
1386
1387  def _add_function(self, short_name, full_name, obj, doc, url):
1388    self._functions.append(_LinkInfo(short_name, full_name, obj, doc, url))
1389
1390  @property
1391  def other_members(self):
1392    return self._other_members
1393
1394  def _add_other_member(self, short_name, full_name, obj, doc):
1395    self._other_members.append(
1396        _OtherMemberInfo(short_name, full_name, obj, doc))
1397
1398  def get_metadata_html(self):
1399    meta_data = _Metadata(self.full_name)
1400
1401    # Objects with their own pages are not added to the matadata list for the
1402    # module, the module only has a link to the object page. No docs.
1403    for item in self.other_members:
1404      meta_data.append(item)
1405
1406    return meta_data.build_html()
1407
1408  def collect_docs_for_module(self, parser_config):
1409    """Collect information necessary specifically for a module's doc page.
1410
1411    Mainly this is information about the members of the module.
1412
1413    Args:
1414      parser_config: An instance of ParserConfig.
1415    """
1416    relative_path = os.path.relpath(
1417        path='.',
1418        start=os.path.dirname(documentation_path(self.full_name)) or '.')
1419
1420    member_names = parser_config.tree.get(self.full_name, [])
1421    for name in member_names:
1422
1423      if name in ['__builtins__', '__doc__', '__file__',
1424                  '__name__', '__path__', '__package__',
1425                  '__cached__', '__loader__', '__spec__']:
1426        continue
1427
1428      member_full_name = self.full_name + '.' + name if self.full_name else name
1429      member = parser_config.py_name_to_object(member_full_name)
1430
1431      member_doc = _parse_md_docstring(member, relative_path,
1432                                       parser_config.reference_resolver)
1433
1434      url = parser_config.reference_resolver.reference_to_url(
1435          member_full_name, relative_path)
1436
1437      if tf_inspect.ismodule(member):
1438        self._add_module(name, member_full_name, member, member_doc, url)
1439
1440      elif tf_inspect.isclass(member):
1441        self._add_class(name, member_full_name, member, member_doc, url)
1442
1443      elif tf_inspect.isfunction(member):
1444        self._add_function(name, member_full_name, member, member_doc, url)
1445
1446      else:
1447        self._add_other_member(name, member_full_name, member, member_doc)
1448
1449
1450class ParserConfig(object):
1451  """Stores all indexes required to parse the docs."""
1452
1453  def __init__(self, reference_resolver, duplicates, duplicate_of, tree, index,
1454               reverse_index, guide_index, base_dir):
1455    """Object with the common config for docs_for_object() calls.
1456
1457    Args:
1458      reference_resolver: An instance of ReferenceResolver.
1459      duplicates: A `dict` mapping fully qualified names to a set of all
1460        aliases of this name. This is used to automatically generate a list of
1461        all aliases for each name.
1462      duplicate_of: A map from duplicate names to preferred names of API
1463        symbols.
1464      tree: A `dict` mapping a fully qualified name to the names of all its
1465        members. Used to populate the members section of a class or module page.
1466      index: A `dict` mapping full names to objects.
1467      reverse_index: A `dict` mapping object ids to full names.
1468
1469      guide_index: A `dict` mapping symbol name strings to objects with a
1470        `make_md_link()` method.
1471
1472      base_dir: A base path that is stripped from file locations written to the
1473        docs.
1474    """
1475    self.reference_resolver = reference_resolver
1476    self.duplicates = duplicates
1477    self.duplicate_of = duplicate_of
1478    self.tree = tree
1479    self.reverse_index = reverse_index
1480    self.index = index
1481    self.guide_index = guide_index
1482    self.base_dir = base_dir
1483    self.defined_in_prefix = 'tensorflow/'
1484    self.code_url_prefix = (
1485        '/code/stable/tensorflow/')  # pylint: disable=line-too-long
1486
1487  def py_name_to_object(self, full_name):
1488    """Return the Python object for a Python symbol name."""
1489    return self.index[full_name]
1490
1491
1492def docs_for_object(full_name, py_object, parser_config):
1493  """Return a PageInfo object describing a given object from the TF API.
1494
1495  This function uses _parse_md_docstring to parse the docs pertaining to
1496  `object`.
1497
1498  This function resolves '@{symbol}' references in the docstrings into links to
1499  the appropriate location. It also adds a list of alternative names for the
1500  symbol automatically.
1501
1502  It assumes that the docs for each object live in a file given by
1503  `documentation_path`, and that relative links to files within the
1504  documentation are resolvable.
1505
1506  Args:
1507    full_name: The fully qualified name of the symbol to be
1508      documented.
1509    py_object: The Python object to be documented. Its documentation is sourced
1510      from `py_object`'s docstring.
1511    parser_config: A ParserConfig object.
1512
1513  Returns:
1514    Either a `_FunctionPageInfo`, `_ClassPageInfo`, or a `_ModulePageInfo`
1515    depending on the type of the python object being documented.
1516
1517  Raises:
1518    RuntimeError: If an object is encountered for which we don't know how
1519      to make docs.
1520  """
1521
1522  # Which other aliases exist for the object referenced by full_name?
1523  master_name = parser_config.reference_resolver.py_master_name(full_name)
1524  duplicate_names = parser_config.duplicates.get(master_name, [full_name])
1525
1526  # TODO(wicke): Once other pieces are ready, enable this also for partials.
1527  if (tf_inspect.ismethod(py_object) or tf_inspect.isfunction(py_object) or
1528      # Some methods in classes from extensions come in as routines.
1529      tf_inspect.isroutine(py_object)):
1530    page_info = _FunctionPageInfo(master_name)
1531    page_info.set_signature(py_object, parser_config.reverse_index)
1532
1533  elif tf_inspect.isclass(py_object):
1534    page_info = _ClassPageInfo(master_name)
1535    page_info.collect_docs_for_class(py_object, parser_config)
1536
1537  elif tf_inspect.ismodule(py_object):
1538    page_info = _ModulePageInfo(master_name)
1539    page_info.collect_docs_for_module(parser_config)
1540
1541  else:
1542    raise RuntimeError('Cannot make docs for object %s: %r' % (full_name,
1543                                                               py_object))
1544
1545  relative_path = os.path.relpath(
1546      path='.', start=os.path.dirname(documentation_path(full_name)) or '.')
1547
1548  page_info.set_doc(_parse_md_docstring(
1549      py_object, relative_path, parser_config.reference_resolver))
1550
1551  page_info.set_aliases(duplicate_names)
1552
1553  page_info.set_guides(_get_guides_markdown(
1554      duplicate_names, parser_config.guide_index, relative_path))
1555
1556  page_info.set_defined_in(_get_defined_in(py_object, parser_config))
1557
1558  return page_info
1559
1560
1561class _PythonBuiltin(object):
1562  """This class indicated that the object in question is a python builtin.
1563
1564  This can be used for the `defined_in` slot of the `PageInfo` objects.
1565  """
1566
1567  def is_builtin(self):
1568    return True
1569
1570  def is_python_file(self):
1571    return False
1572
1573  def is_generated_file(self):
1574    return False
1575
1576  def __str__(self):
1577    return 'This is an alias for a Python built-in.\n\n'
1578
1579
1580class _PythonFile(object):
1581  """This class indicates that the object is defined in a regular python file.
1582
1583  This can be used for the `defined_in` slot of the `PageInfo` objects.
1584  """
1585
1586  def __init__(self, path, parser_config):
1587    self.path = path
1588    self.path_prefix = parser_config.defined_in_prefix
1589    self.code_url_prefix = parser_config.code_url_prefix
1590
1591  def is_builtin(self):
1592    return False
1593
1594  def is_python_file(self):
1595    return True
1596
1597  def is_generated_file(self):
1598    return False
1599
1600  def __str__(self):
1601    return 'Defined in [`{prefix}{path}`]({code_prefix}{path}).\n\n'.format(
1602        path=self.path, prefix=self.path_prefix,
1603        code_prefix=self.code_url_prefix)
1604
1605
1606class _ProtoFile(object):
1607  """This class indicates that the object is defined in a .proto file.
1608
1609  This can be used for the `defined_in` slot of the `PageInfo` objects.
1610  """
1611
1612  def __init__(self, path, parser_config):
1613    self.path = path
1614    self.path_prefix = parser_config.defined_in_prefix
1615    self.code_url_prefix = parser_config.code_url_prefix
1616
1617  def is_builtin(self):
1618    return False
1619
1620  def is_python_file(self):
1621    return False
1622
1623  def is_generated_file(self):
1624    return False
1625
1626  def __str__(self):
1627    return 'Defined in [`{prefix}{path}`]({code_prefix}{path}).\n\n'.format(
1628        path=self.path, prefix=self.path_prefix,
1629        code_prefix=self.code_url_prefix)
1630
1631
1632class _GeneratedFile(object):
1633  """This class indicates that the object is defined in a generated python file.
1634
1635  Generated files should not be linked to directly.
1636
1637  This can be used for the `defined_in` slot of the `PageInfo` objects.
1638  """
1639
1640  def __init__(self, path, parser_config):
1641    self.path = path
1642    self.path_prefix = parser_config.defined_in_prefix
1643
1644  def is_builtin(self):
1645    return False
1646
1647  def is_python_file(self):
1648    return False
1649
1650  def is_generated_file(self):
1651    return True
1652
1653  def __str__(self):
1654    return 'Defined in generated file: `%s%s`.\n\n' % (self.path_prefix,
1655                                                       self.path)
1656
1657
1658def _get_defined_in(py_object, parser_config):
1659  """Returns a description of where the passed in python object was defined.
1660
1661  Args:
1662    py_object: The Python object.
1663    parser_config: A ParserConfig object.
1664
1665  Returns:
1666    Either a `_PythonBuiltin`, `_PythonFile`, or a `_GeneratedFile`
1667  """
1668  # Every page gets a note about where this object is defined
1669  # TODO(wicke): If py_object is decorated, get the decorated object instead.
1670  # TODO(wicke): Only use decorators that support this in TF.
1671
1672  try:
1673    path = os.path.relpath(path=tf_inspect.getfile(py_object),
1674                           start=parser_config.base_dir)
1675  except TypeError:  # getfile throws TypeError if py_object is a builtin.
1676    return _PythonBuiltin()
1677
1678  # TODO(wicke): If this is a generated file, link to the source instead.
1679  # TODO(wicke): Move all generated files to a generated/ directory.
1680  # TODO(wicke): And make their source file predictable from the file name.
1681
1682  # In case this is compiled, point to the original
1683  if path.endswith('.pyc'):
1684    path = path[:-1]
1685
1686  # Never include links outside this code base.
1687  if path.startswith('..') or re.search(r'\b_api\b', path):
1688    return None
1689
1690  if re.match(r'.*/gen_[^/]*\.py$', path):
1691    return _GeneratedFile(path, parser_config)
1692  if 'genfiles' in path or 'tools/api/generator' in path:
1693    return _GeneratedFile(path, parser_config)
1694  elif re.match(r'.*_pb2\.py$', path):
1695    # The _pb2.py files all appear right next to their defining .proto file.
1696    return _ProtoFile(path[:-7] + '.proto', parser_config)
1697  else:
1698    return _PythonFile(path, parser_config)
1699
1700
1701# TODO(markdaoust): This should just parse, pretty_docs should generate the md.
1702def generate_global_index(library_name, index, reference_resolver):
1703  """Given a dict of full names to python objects, generate an index page.
1704
1705  The index page generated contains a list of links for all symbols in `index`
1706  that have their own documentation page.
1707
1708  Args:
1709    library_name: The name for the documented library to use in the title.
1710    index: A dict mapping full names to python objects.
1711    reference_resolver: An instance of ReferenceResolver.
1712
1713  Returns:
1714    A string containing an index page as Markdown.
1715  """
1716  symbol_links = []
1717  for full_name, py_object in six.iteritems(index):
1718    if (tf_inspect.ismodule(py_object) or tf_inspect.isfunction(py_object) or
1719        tf_inspect.isclass(py_object)):
1720      # In Python 3, unbound methods are functions, so eliminate those.
1721      if tf_inspect.isfunction(py_object):
1722        if full_name.count('.') == 0:
1723          parent_name = ''
1724        else:
1725          parent_name = full_name[:full_name.rfind('.')]
1726        if parent_name in index and tf_inspect.isclass(index[parent_name]):
1727          # Skip methods (=functions with class parents).
1728          continue
1729      symbol_links.append((
1730          full_name, reference_resolver.python_link(full_name, full_name, '.')))
1731
1732  lines = ['# All symbols in %s' % library_name, '']
1733  for _, link in sorted(symbol_links, key=lambda x: x[0]):
1734    lines.append('*  %s' % link)
1735
1736  # TODO(markdaoust): use a _ModulePageInfo -> prety_docs.build_md_page()
1737  return '\n'.join(lines)
1738
1739
1740class _Metadata(object):
1741  """A class for building a page's Metadata block.
1742
1743  Attributes:
1744    name: The name of the page being described by the Metadata block.
1745    version: The source version.
1746  """
1747
1748  def __init__(self, name, version='Stable'):
1749    """Creates a Metadata builder.
1750
1751    Args:
1752      name: The name of the page being described by the Metadata block.
1753      version: The source version.
1754    """
1755    self.name = name
1756    self.version = version
1757    self._content = []
1758
1759  def append(self, item):
1760    """Adds an item from the page to the Metadata block.
1761
1762    Args:
1763      item: The parsed page section to add.
1764    """
1765    self._content.append(item.short_name)
1766
1767  def build_html(self):
1768    """Returns the Metadata block as an Html string."""
1769    schema = 'http://developers.google.com/ReferenceObject'
1770    parts = ['<div itemscope itemtype="%s">' % schema]
1771
1772    parts.append('<meta itemprop="name" content="%s" />' % self.name)
1773    parts.append('<meta itemprop="path" content="%s" />' % self.version)
1774    for item in self._content:
1775      parts.append('<meta itemprop="property" content="%s"/>' % item)
1776
1777    parts.extend(['</div>', ''])
1778
1779    return '\n'.join(parts)
1780