1#!/usr/bin/env python 2# Copyright (c) 2013 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Wrapper script to help run clang tools across Chromium code. 6 7How to use run_tool.py: 8If you want to run a clang tool across all Chromium code: 9run_tool.py <tool> <path/to/compiledb> 10 11If you want to include all files mentioned in the compilation database 12(this will also include generated files, unlike the previous command): 13run_tool.py <tool> <path/to/compiledb> --all 14 15If you want to run the clang tool across only chrome/browser and 16content/browser: 17run_tool.py <tool> <path/to/compiledb> chrome/browser content/browser 18 19Please see docs/clang_tool_refactoring.md for more information, which documents 20the entire automated refactoring flow in Chromium. 21 22Why use run_tool.py (instead of running a clang tool directly): 23The clang tool implementation doesn't take advantage of multiple cores, and if 24it fails mysteriously in the middle, all the generated replacements will be 25lost. Additionally, if the work is simply sharded across multiple cores by 26running multiple RefactoringTools, problems arise when they attempt to rewrite a 27file at the same time. 28 29run_tool.py will 301) run multiple instances of clang tool in parallel 312) gather stdout from clang tool invocations 323) "atomically" forward #2 to stdout 33 34Output of run_tool.py can be piped into extract_edits.py and then into 35apply_edits.py. These tools will extract individual edits and apply them to the 36source files. These tools assume the clang tool emits the edits in the 37following format: 38 ... 39 ==== BEGIN EDITS ==== 40 r:::<file path>:::<offset>:::<length>:::<replacement text> 41 r:::<file path>:::<offset>:::<length>:::<replacement text> 42 ...etc... 43 ==== END EDITS ==== 44 ... 45 46extract_edits.py extracts only lines between BEGIN/END EDITS markers 47apply_edits.py reads edit lines from stdin and applies the edits 48""" 49 50import argparse 51import functools 52import multiprocessing 53import os 54import os.path 55import re 56import subprocess 57import sys 58 59script_dir = os.path.dirname(os.path.realpath(__file__)) 60tool_dir = os.path.abspath(os.path.join(script_dir, '../pylib')) 61sys.path.insert(0, tool_dir) 62 63from clang import compile_db 64 65 66def _GetFilesFromGit(paths=None): 67 """Gets the list of files in the git repository. 68 69 Args: 70 paths: Prefix filter for the returned paths. May contain multiple entries. 71 """ 72 args = [] 73 if sys.platform == 'win32': 74 args.append('git.bat') 75 else: 76 args.append('git') 77 args.append('ls-files') 78 if paths: 79 args.extend(paths) 80 command = subprocess.Popen(args, stdout=subprocess.PIPE) 81 output, _ = command.communicate() 82 return [os.path.realpath(p) for p in output.splitlines()] 83 84 85def _GetFilesFromCompileDB(build_directory): 86 """ Gets the list of files mentioned in the compilation database. 87 88 Args: 89 build_directory: Directory that contains the compile database. 90 """ 91 return [os.path.join(entry['directory'], entry['file']) 92 for entry in compile_db.Read(build_directory)] 93 94 95def _ExecuteTool(toolname, tool_args, build_directory, filename): 96 """Executes the clang tool. 97 98 This is defined outside the class so it can be pickled for the multiprocessing 99 module. 100 101 Args: 102 toolname: Name of the clang tool to execute. 103 tool_args: Arguments to be passed to the clang tool. Can be None. 104 build_directory: Directory that contains the compile database. 105 filename: The file to run the clang tool over. 106 107 Returns: 108 A dictionary that must contain the key "status" and a boolean value 109 associated with it. 110 111 If status is True, then the generated output is stored with the key 112 "stdout_text" in the dictionary. 113 114 Otherwise, the filename and the output from stderr are associated with the 115 keys "filename" and "stderr_text" respectively. 116 """ 117 args = [toolname, '-p', build_directory, filename] 118 if (tool_args): 119 args.extend(tool_args) 120 command = subprocess.Popen( 121 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 122 stdout_text, stderr_text = command.communicate() 123 stderr_text = re.sub( 124 r"^warning: .*'linker' input unused \[-Wunused-command-line-argument\]\n", 125 "", stderr_text, flags=re.MULTILINE) 126 if command.returncode != 0: 127 return {'status': False, 'filename': filename, 'stderr_text': stderr_text} 128 else: 129 return {'status': True, 'filename': filename, 'stdout_text': stdout_text, 130 'stderr_text': stderr_text} 131 132 133class _CompilerDispatcher(object): 134 """Multiprocessing controller for running clang tools in parallel.""" 135 136 def __init__(self, toolname, tool_args, build_directory, filenames): 137 """Initializer method. 138 139 Args: 140 toolname: Path to the tool to execute. 141 tool_args: Arguments to be passed to the tool. Can be None. 142 build_directory: Directory that contains the compile database. 143 filenames: The files to run the tool over. 144 """ 145 self.__toolname = toolname 146 self.__tool_args = tool_args 147 self.__build_directory = build_directory 148 self.__filenames = filenames 149 self.__success_count = 0 150 self.__failed_count = 0 151 152 @property 153 def failed_count(self): 154 return self.__failed_count 155 156 def Run(self): 157 """Does the grunt work.""" 158 pool = multiprocessing.Pool() 159 result_iterator = pool.imap_unordered( 160 functools.partial(_ExecuteTool, self.__toolname, self.__tool_args, 161 self.__build_directory), 162 self.__filenames) 163 for result in result_iterator: 164 self.__ProcessResult(result) 165 sys.stderr.write('\n') 166 167 def __ProcessResult(self, result): 168 """Handles result processing. 169 170 Args: 171 result: The result dictionary returned by _ExecuteTool. 172 """ 173 if result['status']: 174 self.__success_count += 1 175 sys.stdout.write(result['stdout_text']) 176 sys.stderr.write(result['stderr_text']) 177 else: 178 self.__failed_count += 1 179 sys.stderr.write('\nFailed to process %s\n' % result['filename']) 180 sys.stderr.write(result['stderr_text']) 181 sys.stderr.write('\n') 182 done_count = self.__success_count + self.__failed_count 183 percentage = (float(done_count) / len(self.__filenames)) * 100 184 sys.stderr.write( 185 'Processed %d files with %s tool (%d failures) [%.2f%%]\r' % 186 (done_count, self.__toolname, self.__failed_count, percentage)) 187 188 189def main(): 190 parser = argparse.ArgumentParser() 191 parser.add_argument('tool', help='clang tool to run') 192 parser.add_argument('--all', action='store_true') 193 parser.add_argument( 194 '--generate-compdb', 195 action='store_true', 196 help='regenerate the compile database before running the tool') 197 parser.add_argument( 198 'compile_database', 199 help='path to the directory that contains the compile database') 200 parser.add_argument( 201 'path_filter', 202 nargs='*', 203 help='optional paths to filter what files the tool is run on') 204 parser.add_argument( 205 '--tool-args', nargs='*', 206 help='optional arguments passed to the tool') 207 args = parser.parse_args() 208 209 os.environ['PATH'] = '%s%s%s' % ( 210 os.path.abspath(os.path.join( 211 os.path.dirname(__file__), 212 '../../../third_party/llvm-build/Release+Asserts/bin')), 213 os.pathsep, 214 os.environ['PATH']) 215 216 if args.generate_compdb: 217 compile_db.GenerateWithNinja(args.compile_database) 218 219 if args.all: 220 source_filenames = set(_GetFilesFromCompileDB(args.compile_database)) 221 else: 222 git_filenames = set(_GetFilesFromGit(args.path_filter)) 223 # Filter out files that aren't C/C++/Obj-C/Obj-C++. 224 extensions = frozenset(('.c', '.cc', '.cpp', '.m', '.mm')) 225 source_filenames = [f 226 for f in git_filenames 227 if os.path.splitext(f)[1] in extensions] 228 229 dispatcher = _CompilerDispatcher(args.tool, args.tool_args, 230 args.compile_database, 231 source_filenames) 232 dispatcher.Run() 233 return -dispatcher.failed_count 234 235 236if __name__ == '__main__': 237 sys.exit(main()) 238