• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Wrapper script to help run clang tools across Chromium code.
7
8How to use this tool:
9If you want to run the tool across all Chromium code:
10run_tool.py <tool> <path/to/compiledb>
11
12If you only want to run the tool across just chrome/browser and content/browser:
13run_tool.py <tool> <path/to/compiledb> chrome/browser content/browser
14
15Please see https://code.google.com/p/chromium/wiki/ClangToolRefactoring for more
16information, which documents the entire automated refactoring flow in Chromium.
17
18Why use this tool:
19The clang tool implementation doesn't take advantage of multiple cores, and if
20it fails mysteriously in the middle, all the generated replacements will be
21lost.
22
23Unfortunately, if the work is simply sharded across multiple cores by running
24multiple RefactoringTools, problems arise when they attempt to rewrite a file at
25the same time. To work around that, clang tools that are run using this tool
26should output edits to stdout in the following format:
27
28==== BEGIN EDITS ====
29r:<file path>:<offset>:<length>:<replacement text>
30r:<file path>:<offset>:<length>:<replacement text>
31...etc...
32==== END EDITS ====
33
34Any generated edits are applied once the clang tool has finished running
35across Chromium, regardless of whether some instances failed or not.
36"""
37
38import collections
39import functools
40import multiprocessing
41import os.path
42import subprocess
43import sys
44
45
46Edit = collections.namedtuple(
47    'Edit', ('edit_type', 'offset', 'length', 'replacement'))
48
49
50def _GetFilesFromGit(paths = None):
51  """Gets the list of files in the git repository.
52
53  Args:
54    paths: Prefix filter for the returned paths. May contain multiple entries.
55  """
56  args = ['git', 'ls-files']
57  if paths:
58    args.extend(paths)
59  command = subprocess.Popen(args, stdout=subprocess.PIPE)
60  output, _ = command.communicate()
61  return output.splitlines()
62
63
64def _ExtractEditsFromStdout(build_directory, stdout):
65  """Extracts generated list of edits from the tool's stdout.
66
67  The expected format is documented at the top of this file.
68
69  Args:
70    build_directory: Directory that contains the compile database. Used to
71      normalize the filenames.
72    stdout: The stdout from running the clang tool.
73
74  Returns:
75    A dictionary mapping filenames to the associated edits.
76  """
77  lines = stdout.splitlines()
78  start_index = lines.index('==== BEGIN EDITS ====')
79  end_index = lines.index('==== END EDITS ====')
80  edits = collections.defaultdict(list)
81  for line in lines[start_index + 1:end_index]:
82    try:
83      edit_type, path, offset, length, replacement = line.split(':', 4)
84      # Normalize the file path emitted by the clang tool to be relative to the
85      # current working directory.
86      path = os.path.relpath(os.path.join(build_directory, path))
87      edits[path].append(Edit(edit_type, int(offset), int(length), replacement))
88    except ValueError:
89      print 'Unable to parse edit: %s' % line
90  return edits
91
92
93def _ExecuteTool(toolname, build_directory, filename):
94  """Executes the tool.
95
96  This is defined outside the class so it can be pickled for the multiprocessing
97  module.
98
99  Args:
100    toolname: Path to the tool to execute.
101    build_directory: Directory that contains the compile database.
102    filename: The file to run the tool over.
103
104  Returns:
105    A dictionary that must contain the key "status" and a boolean value
106    associated with it.
107
108    If status is True, then the generated edits are stored with the key "edits"
109    in the dictionary.
110
111    Otherwise, the filename and the output from stderr are associated with the
112    keys "filename" and "stderr" respectively.
113  """
114  command = subprocess.Popen((toolname, '-p', build_directory, filename),
115                             stdout=subprocess.PIPE,
116                             stderr=subprocess.PIPE)
117  stdout, stderr = command.communicate()
118  if command.returncode != 0:
119    return {'status': False, 'filename': filename, 'stderr': stderr}
120  else:
121    return {'status': True,
122            'edits': _ExtractEditsFromStdout(build_directory, stdout)}
123
124
125class _CompilerDispatcher(object):
126  """Multiprocessing controller for running clang tools in parallel."""
127
128  def __init__(self, toolname, build_directory, filenames):
129    """Initializer method.
130
131    Args:
132      toolname: Path to the tool to execute.
133      build_directory: Directory that contains the compile database.
134      filenames: The files to run the tool over.
135    """
136    self.__toolname = toolname
137    self.__build_directory = build_directory
138    self.__filenames = filenames
139    self.__success_count = 0
140    self.__failed_count = 0
141    self.__edits = collections.defaultdict(list)
142
143  @property
144  def edits(self):
145    return self.__edits
146
147  @property
148  def failed_count(self):
149    return self.__failed_count
150
151  def Run(self):
152    """Does the grunt work."""
153    pool = multiprocessing.Pool()
154    result_iterator = pool.imap_unordered(
155        functools.partial(_ExecuteTool, self.__toolname,
156                          self.__build_directory),
157        self.__filenames)
158    for result in result_iterator:
159      self.__ProcessResult(result)
160    sys.stdout.write('\n')
161    sys.stdout.flush()
162
163  def __ProcessResult(self, result):
164    """Handles result processing.
165
166    Args:
167      result: The result dictionary returned by _ExecuteTool.
168    """
169    if result['status']:
170      self.__success_count += 1
171      for k, v in result['edits'].iteritems():
172        self.__edits[k].extend(v)
173    else:
174      self.__failed_count += 1
175      sys.stdout.write('\nFailed to process %s\n' % result['filename'])
176      sys.stdout.write(result['stderr'])
177      sys.stdout.write('\n')
178    percentage = (
179        float(self.__success_count + self.__failed_count) /
180        len(self.__filenames)) * 100
181    sys.stdout.write('Succeeded: %d, Failed: %d [%.2f%%]\r' % (
182        self.__success_count, self.__failed_count, percentage))
183    sys.stdout.flush()
184
185
186def _ApplyEdits(edits, clang_format_diff_path):
187  """Apply the generated edits.
188
189  Args:
190    edits: A dict mapping filenames to Edit instances that apply to that file.
191    clang_format_diff_path: Path to the clang-format-diff.py helper to help
192      automatically reformat diffs to avoid style violations. Pass None if the
193      clang-format step should be skipped.
194  """
195  edit_count = 0
196  for k, v in edits.iteritems():
197    # Sort the edits and iterate through them in reverse order. Sorting allows
198    # duplicate edits to be quickly skipped, while reversing means that
199    # subsequent edits don't need to have their offsets updated with each edit
200    # applied.
201    v.sort()
202    last_edit = None
203    with open(k, 'rb+') as f:
204      contents = bytearray(f.read())
205      for edit in reversed(v):
206        if edit == last_edit:
207          continue
208        last_edit = edit
209        contents[edit.offset:edit.offset + edit.length] = edit.replacement
210        if not edit.replacement:
211          _ExtendDeletionIfElementIsInList(contents, edit.offset)
212        edit_count += 1
213      f.seek(0)
214      f.truncate()
215      f.write(contents)
216    if clang_format_diff_path:
217      if subprocess.call('git diff -U0 %s | python %s -style=Chromium' % (
218          k, clang_format_diff_path), shell=True) != 0:
219        print 'clang-format failed for %s' % k
220  print 'Applied %d edits to %d files' % (edit_count, len(edits))
221
222
223_WHITESPACE_BYTES = frozenset((ord('\t'), ord('\n'), ord('\r'), ord(' ')))
224
225
226def _ExtendDeletionIfElementIsInList(contents, offset):
227  """Extends the range of a deletion if the deleted element was part of a list.
228
229  This rewriter helper makes it easy for refactoring tools to remove elements
230  from a list. Even if a matcher callback knows that it is removing an element
231  from a list, it may not have enough information to accurately remove the list
232  element; for example, another matcher callback may end up removing an adjacent
233  list element, or all the list elements may end up being removed.
234
235  With this helper, refactoring tools can simply remove the list element and not
236  worry about having to include the comma in the replacement.
237
238  Args:
239    contents: A bytearray with the deletion already applied.
240    offset: The offset in the bytearray where the deleted range used to be.
241  """
242  char_before = char_after = None
243  left_trim_count = 0
244  for byte in reversed(contents[:offset]):
245    left_trim_count += 1
246    if byte in _WHITESPACE_BYTES:
247      continue
248    if byte in (ord(','), ord(':'), ord('('), ord('{')):
249      char_before = chr(byte)
250    break
251
252  right_trim_count = 0
253  for byte in contents[offset:]:
254    right_trim_count += 1
255    if byte in _WHITESPACE_BYTES:
256      continue
257    if byte == ord(','):
258      char_after = chr(byte)
259    break
260
261  if char_before:
262    if char_after:
263      del contents[offset:offset + right_trim_count]
264    elif char_before in (',', ':'):
265      del contents[offset - left_trim_count:offset]
266
267
268def main(argv):
269  if len(argv) < 2:
270    print 'Usage: run_tool.py <clang tool> <compile DB> <path 1> <path 2> ...'
271    print '  <clang tool> is the clang tool that should be run.'
272    print '  <compile db> is the directory that contains the compile database'
273    print '  <path 1> <path2> ... can be used to filter what files are edited'
274    return 1
275
276  clang_format_diff_path = os.path.join(
277      os.path.dirname(os.path.realpath(__file__)),
278      '../../../third_party/llvm/tools/clang/tools/clang-format',
279      'clang-format-diff.py')
280  # TODO(dcheng): Allow this to be controlled with a flag as well.
281  if not os.path.isfile(clang_format_diff_path):
282    clang_format_diff_path = None
283
284  filenames = frozenset(_GetFilesFromGit(argv[2:]))
285  # Filter out files that aren't C/C++/Obj-C/Obj-C++.
286  extensions = frozenset(('.c', '.cc', '.m', '.mm'))
287  dispatcher = _CompilerDispatcher(argv[0], argv[1],
288                                   [f for f in filenames
289                                    if os.path.splitext(f)[1] in extensions])
290  dispatcher.Run()
291  # Filter out edits to files that aren't in the git repository, since it's not
292  # useful to modify files that aren't under source control--typically, these
293  # are generated files or files in a git submodule that's not part of Chromium.
294  _ApplyEdits({k : v for k, v in dispatcher.edits.iteritems()
295                    if k in filenames},
296              clang_format_diff_path)
297  if dispatcher.failed_count != 0:
298    return 2
299  return 0
300
301
302if __name__ == '__main__':
303  sys.exit(main(sys.argv[1:]))
304