1# -*- coding:utf-8 -*- 2# Copyright 2016 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16"""Functions that implement the actual checks.""" 17 18from __future__ import print_function 19 20import json 21import os 22import platform 23import re 24import sys 25 26_path = os.path.realpath(__file__ + '/../..') 27if sys.path[0] != _path: 28 sys.path.insert(0, _path) 29del _path 30 31import rh.results 32import rh.git 33import rh.utils 34 35 36class Placeholders(object): 37 """Holder class for replacing ${vars} in arg lists. 38 39 To add a new variable to replace in config files, just add it as a @property 40 to this class using the form. So to add support for BIRD: 41 @property 42 def var_BIRD(self): 43 return <whatever this is> 44 45 You can return either a string or an iterable (e.g. a list or tuple). 46 """ 47 48 def __init__(self, diff=()): 49 """Initialize. 50 51 Args: 52 diff: The list of files that changed. 53 """ 54 self.diff = diff 55 56 def expand_vars(self, args): 57 """Perform place holder expansion on all of |args|. 58 59 Args: 60 args: The args to perform expansion on. 61 62 Returns: 63 The updated |args| list. 64 """ 65 all_vars = set(self.vars()) 66 replacements = dict((var, self.get(var)) for var in all_vars) 67 68 ret = [] 69 for arg in args: 70 # First scan for exact matches 71 for key, val in replacements.items(): 72 var = '${%s}' % (key,) 73 if arg == var: 74 if isinstance(val, str): 75 ret.append(val) 76 else: 77 ret.extend(val) 78 # We break on first hit to avoid double expansion. 79 break 80 else: 81 # If no exact matches, do an inline replacement. 82 def replace(m): 83 val = self.get(m.group(1)) 84 if isinstance(val, str): 85 return val 86 else: 87 return ' '.join(val) 88 ret.append(re.sub(r'\$\{(%s)\}' % ('|'.join(all_vars),), 89 replace, arg)) 90 91 return ret 92 93 @classmethod 94 def vars(cls): 95 """Yield all replacement variable names.""" 96 for key in dir(cls): 97 if key.startswith('var_'): 98 yield key[4:] 99 100 def get(self, var): 101 """Helper function to get the replacement |var| value.""" 102 return getattr(self, 'var_%s' % (var,)) 103 104 @property 105 def var_PREUPLOAD_COMMIT_MESSAGE(self): 106 """The git commit message.""" 107 return os.environ.get('PREUPLOAD_COMMIT_MESSAGE', '') 108 109 @property 110 def var_PREUPLOAD_COMMIT(self): 111 """The git commit sha1.""" 112 return os.environ.get('PREUPLOAD_COMMIT', '') 113 114 @property 115 def var_PREUPLOAD_FILES(self): 116 """List of files modified in this git commit.""" 117 return [x.file for x in self.diff if x.status != 'D'] 118 119 @property 120 def var_REPO_ROOT(self): 121 """The root of the repo checkout.""" 122 return rh.git.find_repo_root() 123 124 @property 125 def var_BUILD_OS(self): 126 """The build OS (see _get_build_os_name for details).""" 127 return _get_build_os_name() 128 129 130class HookOptions(object): 131 """Holder class for hook options.""" 132 133 def __init__(self, name, args, tool_paths): 134 """Initialize. 135 136 Args: 137 name: The name of the hook. 138 args: The override commandline arguments for the hook. 139 tool_paths: A dictionary with tool names to paths. 140 """ 141 self.name = name 142 self._args = args 143 self._tool_paths = tool_paths 144 145 @staticmethod 146 def expand_vars(args, diff=()): 147 """Perform place holder expansion on all of |args|.""" 148 replacer = Placeholders(diff=diff) 149 return replacer.expand_vars(args) 150 151 def args(self, default_args=(), diff=()): 152 """Gets the hook arguments, after performing place holder expansion. 153 154 Args: 155 default_args: The list to return if |self._args| is empty. 156 diff: The list of files that changed in the current commit. 157 158 Returns: 159 A list with arguments. 160 """ 161 args = self._args 162 if not args: 163 args = default_args 164 165 return self.expand_vars(args, diff=diff) 166 167 def tool_path(self, tool_name): 168 """Gets the path in which the |tool_name| executable can be found. 169 170 This function performs expansion for some place holders. If the tool 171 does not exist in the overridden |self._tool_paths| dictionary, the tool 172 name will be returned and will be run from the user's $PATH. 173 174 Args: 175 tool_name: The name of the executable. 176 177 Returns: 178 The path of the tool with all optional place holders expanded. 179 """ 180 assert tool_name in TOOL_PATHS 181 if tool_name not in self._tool_paths: 182 return TOOL_PATHS[tool_name] 183 184 tool_path = os.path.normpath(self._tool_paths[tool_name]) 185 return self.expand_vars([tool_path])[0] 186 187 188def _run_command(cmd, **kwargs): 189 """Helper command for checks that tend to gather output.""" 190 kwargs.setdefault('redirect_stderr', True) 191 kwargs.setdefault('combine_stdout_stderr', True) 192 kwargs.setdefault('capture_output', True) 193 kwargs.setdefault('error_code_ok', True) 194 return rh.utils.run_command(cmd, **kwargs) 195 196 197def _match_regex_list(subject, expressions): 198 """Try to match a list of regular expressions to a string. 199 200 Args: 201 subject: The string to match regexes on. 202 expressions: An iterable of regular expressions to check for matches with. 203 204 Returns: 205 Whether the passed in subject matches any of the passed in regexes. 206 """ 207 for expr in expressions: 208 if re.search(expr, subject): 209 return True 210 return False 211 212 213def _filter_diff(diff, include_list, exclude_list=()): 214 """Filter out files based on the conditions passed in. 215 216 Args: 217 diff: list of diff objects to filter. 218 include_list: list of regex that when matched with a file path will cause 219 it to be added to the output list unless the file is also matched with 220 a regex in the exclude_list. 221 exclude_list: list of regex that when matched with a file will prevent it 222 from being added to the output list, even if it is also matched with a 223 regex in the include_list. 224 225 Returns: 226 A list of filepaths that contain files matched in the include_list and not 227 in the exclude_list. 228 """ 229 filtered = [] 230 for d in diff: 231 if (d.status != 'D' and 232 _match_regex_list(d.file, include_list) and 233 not _match_regex_list(d.file, exclude_list)): 234 # We've got a match! 235 filtered.append(d) 236 return filtered 237 238 239def _get_build_os_name(): 240 """Gets the build OS name. 241 242 Returns: 243 A string in a format usable to get prebuilt tool paths. 244 """ 245 system = platform.system() 246 if 'Darwin' in system or 'Macintosh' in system: 247 return 'darwin-x86' 248 else: 249 # TODO: Add more values if needed. 250 return 'linux-x86' 251 252 253def _fixup_func_caller(cmd, **kwargs): 254 """Wraps |cmd| around a callable automated fixup. 255 256 For hooks that support automatically fixing errors after running (e.g. code 257 formatters), this function provides a way to run |cmd| as the |fixup_func| 258 parameter in HookCommandResult. 259 """ 260 def wrapper(): 261 result = _run_command(cmd, **kwargs) 262 if result.returncode not in (None, 0): 263 return result.output 264 return None 265 return wrapper 266 267 268def _check_cmd(hook_name, project, commit, cmd, fixup_func=None, **kwargs): 269 """Runs |cmd| and returns its result as a HookCommandResult.""" 270 return [rh.results.HookCommandResult(hook_name, project, commit, 271 _run_command(cmd, **kwargs), 272 fixup_func=fixup_func)] 273 274 275# Where helper programs exist. 276TOOLS_DIR = os.path.realpath(__file__ + '/../../tools') 277 278def get_helper_path(tool): 279 """Return the full path to the helper |tool|.""" 280 return os.path.join(TOOLS_DIR, tool) 281 282 283def check_custom(project, commit, _desc, diff, options=None, **kwargs): 284 """Run a custom hook.""" 285 return _check_cmd(options.name, project, commit, options.args((), diff), 286 **kwargs) 287 288 289def check_checkpatch(project, commit, _desc, diff, options=None): 290 """Run |diff| through the kernel's checkpatch.pl tool.""" 291 tool = get_helper_path('checkpatch.pl') 292 cmd = ([tool, '-', '--root', project.dir] + 293 options.args(('--ignore=GERRIT_CHANGE_ID',), diff)) 294 return _check_cmd('checkpatch.pl', project, commit, cmd, 295 input=rh.git.get_patch(commit)) 296 297 298def check_clang_format(project, commit, _desc, diff, options=None): 299 """Run git clang-format on the commit.""" 300 tool = get_helper_path('clang-format.py') 301 clang_format = options.tool_path('clang-format') 302 git_clang_format = options.tool_path('git-clang-format') 303 tool_args = (['--clang-format', clang_format, '--git-clang-format', 304 git_clang_format] + 305 options.args(('--style', 'file', '--commit', commit), diff)) 306 cmd = [tool] + tool_args 307 fixup_func = _fixup_func_caller([tool, '--fix'] + tool_args) 308 return _check_cmd('clang-format', project, commit, cmd, 309 fixup_func=fixup_func) 310 311 312def check_google_java_format(project, commit, _desc, _diff, options=None): 313 """Run google-java-format on the commit.""" 314 315 tool = get_helper_path('google-java-format.py') 316 google_java_format = options.tool_path('google-java-format') 317 google_java_format_diff = options.tool_path('google-java-format-diff') 318 tool_args = ['--google-java-format', google_java_format, 319 '--google-java-format-diff', google_java_format_diff, 320 '--commit', commit] + options.args() 321 cmd = [tool] + tool_args 322 fixup_func = _fixup_func_caller([tool, '--fix'] + tool_args) 323 return _check_cmd('google-java-format', project, commit, cmd, 324 fixup_func=fixup_func) 325 326 327def check_commit_msg_bug_field(project, commit, desc, _diff, options=None): 328 """Check the commit message for a 'Bug:' line.""" 329 field = 'Bug' 330 regex = r'^%s: (None|[0-9]+(, [0-9]+)*)$' % (field,) 331 check_re = re.compile(regex) 332 333 if options.args(): 334 raise ValueError('commit msg %s check takes no options' % (field,)) 335 336 found = [] 337 for line in desc.splitlines(): 338 if check_re.match(line): 339 found.append(line) 340 341 if not found: 342 error = ('Commit message is missing a "%s:" line. It must match:\n' 343 '%s') % (field, regex) 344 else: 345 return 346 347 return [rh.results.HookResult('commit msg: "%s:" check' % (field,), 348 project, commit, error=error)] 349 350 351def check_commit_msg_changeid_field(project, commit, desc, _diff, options=None): 352 """Check the commit message for a 'Change-Id:' line.""" 353 field = 'Change-Id' 354 regex = r'^%s: I[a-f0-9]+$' % (field,) 355 check_re = re.compile(regex) 356 357 if options.args(): 358 raise ValueError('commit msg %s check takes no options' % (field,)) 359 360 found = [] 361 for line in desc.splitlines(): 362 if check_re.match(line): 363 found.append(line) 364 365 if len(found) == 0: 366 error = ('Commit message is missing a "%s:" line. It must match:\n' 367 '%s') % (field, regex) 368 elif len(found) > 1: 369 error = ('Commit message has too many "%s:" lines. There can be only ' 370 'one.') % (field,) 371 else: 372 return 373 374 return [rh.results.HookResult('commit msg: "%s:" check' % (field,), 375 project, commit, error=error)] 376 377 378TEST_MSG = """Commit message is missing a "Test:" line. It must match: 379%s 380 381The Test: stanza is free-form and should describe how you tested your change. 382As a CL author, you'll have a consistent place to describe the testing strategy 383you use for your work. As a CL reviewer, you'll be reminded to discuss testing 384as part of your code review, and you'll more easily replicate testing when you 385patch in CLs locally. 386 387Some examples below: 388 389Test: make WITH_TIDY=1 mmma art 390Test: make test-art 391Test: manual - took a photo 392Test: refactoring CL. Existing unit tests still pass. 393 394Check the git history for more examples. It's a free-form field, so we urge 395you to develop conventions that make sense for your project. Note that many 396projects use exact test commands, which are perfectly fine. 397 398Adding good automated tests with new code is critical to our goals of keeping 399the system stable and constantly improving quality. Please use Test: to 400highlight this area of your development. And reviewers, please insist on 401high-quality Test: descriptions. 402""" 403 404 405def check_commit_msg_test_field(project, commit, desc, _diff, options=None): 406 """Check the commit message for a 'Test:' line.""" 407 field = 'Test' 408 regex = r'^%s: .*$' % (field,) 409 check_re = re.compile(regex) 410 411 if options.args(): 412 raise ValueError('commit msg %s check takes no options' % (field,)) 413 414 found = [] 415 for line in desc.splitlines(): 416 if check_re.match(line): 417 found.append(line) 418 419 if not found: 420 error = TEST_MSG % (regex) 421 else: 422 return 423 424 return [rh.results.HookResult('commit msg: "%s:" check' % (field,), 425 project, commit, error=error)] 426 427 428def check_cpplint(project, commit, _desc, diff, options=None): 429 """Run cpplint.""" 430 # This list matches what cpplint expects. We could run on more (like .cxx), 431 # but cpplint would just ignore them. 432 filtered = _filter_diff(diff, [r'\.(cc|h|cpp|cu|cuh)$']) 433 if not filtered: 434 return 435 436 cpplint = options.tool_path('cpplint') 437 cmd = [cpplint] + options.args(('${PREUPLOAD_FILES}',), filtered) 438 return _check_cmd('cpplint', project, commit, cmd) 439 440 441def check_gofmt(project, commit, _desc, diff, options=None): 442 """Checks that Go files are formatted with gofmt.""" 443 filtered = _filter_diff(diff, [r'\.go$']) 444 if not filtered: 445 return 446 447 gofmt = options.tool_path('gofmt') 448 cmd = [gofmt, '-l'] + options.args((), filtered) 449 ret = [] 450 for d in filtered: 451 data = rh.git.get_file_content(commit, d.file) 452 result = _run_command(cmd, input=data) 453 if result.output: 454 ret.append(rh.results.HookResult( 455 'gofmt', project, commit, error=result.output, 456 files=(d.file,))) 457 return ret 458 459 460def check_json(project, commit, _desc, diff, options=None): 461 """Verify json files are valid.""" 462 if options.args(): 463 raise ValueError('json check takes no options') 464 465 filtered = _filter_diff(diff, [r'\.json$']) 466 if not filtered: 467 return 468 469 ret = [] 470 for d in filtered: 471 data = rh.git.get_file_content(commit, d.file) 472 try: 473 json.loads(data) 474 except ValueError as e: 475 ret.append(rh.results.HookResult( 476 'json', project, commit, error=str(e), 477 files=(d.file,))) 478 return ret 479 480 481def check_pylint(project, commit, _desc, diff, options=None): 482 """Run pylint.""" 483 filtered = _filter_diff(diff, [r'\.py$']) 484 if not filtered: 485 return 486 487 pylint = options.tool_path('pylint') 488 cmd = [ 489 get_helper_path('pylint.py'), 490 '--executable-path', pylint, 491 ] + options.args(('${PREUPLOAD_FILES}',), filtered) 492 return _check_cmd('pylint', project, commit, cmd) 493 494 495def check_xmllint(project, commit, _desc, diff, options=None): 496 """Run xmllint.""" 497 # XXX: Should we drop most of these and probe for <?xml> tags? 498 extensions = frozenset(( 499 'dbus-xml', # Generated DBUS interface. 500 'dia', # File format for Dia. 501 'dtd', # Document Type Definition. 502 'fml', # Fuzzy markup language. 503 'form', # Forms created by IntelliJ GUI Designer. 504 'fxml', # JavaFX user interfaces. 505 'glade', # Glade user interface design. 506 'grd', # GRIT translation files. 507 'iml', # Android build modules? 508 'kml', # Keyhole Markup Language. 509 'mxml', # Macromedia user interface markup language. 510 'nib', # OS X Cocoa Interface Builder. 511 'plist', # Property list (for OS X). 512 'pom', # Project Object Model (for Apache Maven). 513 'rng', # RELAX NG schemas. 514 'sgml', # Standard Generalized Markup Language. 515 'svg', # Scalable Vector Graphics. 516 'uml', # Unified Modeling Language. 517 'vcproj', # Microsoft Visual Studio project. 518 'vcxproj', # Microsoft Visual Studio project. 519 'wxs', # WiX Transform File. 520 'xhtml', # XML HTML. 521 'xib', # OS X Cocoa Interface Builder. 522 'xlb', # Android locale bundle. 523 'xml', # Extensible Markup Language. 524 'xsd', # XML Schema Definition. 525 'xsl', # Extensible Stylesheet Language. 526 )) 527 528 filtered = _filter_diff(diff, [r'\.(%s)$' % '|'.join(extensions)]) 529 if not filtered: 530 return 531 532 # TODO: Figure out how to integrate schema validation. 533 # XXX: Should we use python's XML libs instead? 534 cmd = ['xmllint'] + options.args(('${PREUPLOAD_FILES}',), filtered) 535 536 return _check_cmd('xmllint', project, commit, cmd) 537 538 539# Hooks that projects can opt into. 540# Note: Make sure to keep the top level README.md up to date when adding more! 541BUILTIN_HOOKS = { 542 'checkpatch': check_checkpatch, 543 'clang_format': check_clang_format, 544 'commit_msg_bug_field': check_commit_msg_bug_field, 545 'commit_msg_changeid_field': check_commit_msg_changeid_field, 546 'commit_msg_test_field': check_commit_msg_test_field, 547 'cpplint': check_cpplint, 548 'gofmt': check_gofmt, 549 'google_java_format': check_google_java_format, 550 'jsonlint': check_json, 551 'pylint': check_pylint, 552 'xmllint': check_xmllint, 553} 554 555# Additional tools that the hooks can call with their default values. 556# Note: Make sure to keep the top level README.md up to date when adding more! 557TOOL_PATHS = { 558 'clang-format': 'clang-format', 559 'cpplint': os.path.join(TOOLS_DIR, 'cpplint.py'), 560 'git-clang-format': 'git-clang-format', 561 'gofmt': 'gofmt', 562 'google-java-format': 'google-java-format', 563 'google-java-format-diff': 'google-java-format-diff.py', 564 'pylint': 'pylint', 565} 566