• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Converting code to AST.
16
17Adapted from Tangent.
18"""
19
20from __future__ import absolute_import
21from __future__ import division
22from __future__ import print_function
23
24import re
25import textwrap
26import threading
27
28import gast
29import six
30
31from tensorflow.python.util import tf_inspect
32
33
34_parse_lock = threading.Lock()  # Prevents linecache concurrency errors.
35
36
37def parse_entity(entity):
38  """Returns the AST and source code of given entity.
39
40  Args:
41    entity: A python function/method/class
42
43  Returns:
44    gast.AST, str, gast.ModuleNode: a tuple of the AST node corresponding
45    exactly to the entity; the string that was parsed to generate the AST; and
46    the containing module AST node, which might contain extras like future
47    import nodes.
48  """
49  try:
50    with _parse_lock:
51      source = tf_inspect.getsource_no_unwrap(entity)
52  except (IOError, OSError) as e:
53    raise ValueError(
54        'Unable to locate the source code of {}. Note that functions defined'
55        ' in certain environments, like the interactive Python shell do not'
56        ' expose their source code. If that is the case, you should to define'
57        ' them in a .py source file. If you are certain the code is'
58        ' graph-compatible, wrap the call using'
59        ' @tf.autograph.do_not_convert. Original error: {}'.format(entity, e))
60
61  def raise_parse_failure(comment):
62    raise ValueError(
63        'Failed to parse source code of {}, which Python reported as:\n{}\n'
64        '{}'.format(entity, source, comment))
65
66  # Comments and multiline strings can appear at arbitrary indentation levels,
67  # causing textwrap.dedent to not correctly dedent source code.
68  # TODO(b/115884650): Automatic handling of comments/multiline strings.
69  source = textwrap.dedent(source)
70
71  try:
72    module_node = parse_str(source)
73    assert len(module_node.body) == 1
74    return module_node.body[0], source, module_node
75
76  except IndentationError:
77    # The text below lists the causes of this error known to us. There may
78    # be more.
79    raise_parse_failure(
80        'This may be caused by multiline strings or comments not indented at'
81        ' the same level as the code.')
82
83  except SyntaxError as e:
84    if not tf_inspect.isfunction(entity) or entity.__name__ != '<lambda>':
85      raise
86
87    # Certain entities, like lambdas, only hold the raw code lines which defined
88    # them, which may include surrounding tokens and may be syntactically
89    # invalid out of context. For example:
90    #
91    #     l = (
92    #         lambda x: x,)[0]
93    #
94    # will have the dedented source "lambda x: x,)[0]"
95    # Here we make an attempt to stip away the garbage by looking at the
96    # information in the syntax error.
97    lines = source.split('\n')
98    lineno, offset = e.lineno, e.offset  # 1-based
99
100    # Give up if there's nothing we can chip away.
101    if len(lines) == lineno and len(lines[-1]) == offset:
102      raise_parse_failure(
103          'If this is a lambda function, the error may be avoided by creating'
104          ' the lambda in a standalone statement.')
105
106    # Drop all lines following the error location
107    # TODO(mdan): What's with the pylint errors?
108    lines = lines[:lineno]  # pylint:disable=invalid-slice-index
109    # Drop all characters following the error location
110    lines[-1] = lines[-1][:offset - 1]  # pylint:disable=invalid-slice-index
111    new_source = '\n'.join(lines)
112
113    try:
114      module_node = parse_str(new_source)
115      return module_node.body[0], new_source, module_node
116    except SyntaxError as e:
117      raise_parse_failure(
118          'If this is a lambda function, the error may be avoided by creating'
119          ' the lambda in a standalone statement. Tried to strip down the'
120          ' source to:\n{}\nBut that did not work.'.format(new_source))
121
122
123def parse_str(src):
124  """Returns the AST of given piece of code."""
125  # TODO(mdan): This should exclude the module things are autowrapped in.
126
127  if six.PY2 and re.search('\\Wprint\\s*\\(', src):
128    # This special treatment is required because gast.parse is not aware of
129    # whether print_function was present in the original context.
130    src = 'from __future__ import print_function\n' + src
131    parsed_module = gast.parse(src)
132    parsed_module.body = parsed_module.body[1:]
133  else:
134    parsed_module = gast.parse(src)
135
136  return parsed_module
137
138
139def parse_expression(src):
140  """Returns the AST of given identifier.
141
142  Args:
143    src: A piece of code that represents a single Python expression
144  Returns:
145    A gast.AST object.
146  Raises:
147    ValueError: if src does not consist of a single Expression.
148  """
149  node = parse_str(src)
150  assert isinstance(node, gast.Module)
151  if len(node.body) != 1 or not isinstance(node.body[0], gast.Expr):
152    raise ValueError(
153        'Expected a single expression, found instead %s' % node.body)
154  return node.body[0].value
155