1# Copyright 2016 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Functions that implement the actual checks.""" 16 17import fnmatch 18import json 19import os 20import platform 21import re 22import sys 23from typing import Callable, NamedTuple 24 25_path = os.path.realpath(__file__ + '/../..') 26if sys.path[0] != _path: 27 sys.path.insert(0, _path) 28del _path 29 30# pylint: disable=wrong-import-position 31import rh.git 32import rh.results 33import rh.utils 34 35 36class Placeholders(object): 37 """Holder class for replacing ${vars} in arg lists. 38 39 To add a new variable to replace in config files, just add it as a @property 40 to this class using the form. So to add support for BIRD: 41 @property 42 def var_BIRD(self): 43 return <whatever this is> 44 45 You can return either a string or an iterable (e.g. a list or tuple). 46 """ 47 48 def __init__(self, diff=()): 49 """Initialize. 50 51 Args: 52 diff: The list of files that changed. 53 """ 54 self.diff = diff 55 56 def expand_vars(self, args): 57 """Perform place holder expansion on all of |args|. 58 59 Args: 60 args: The args to perform expansion on. 61 62 Returns: 63 The updated |args| list. 64 """ 65 all_vars = set(self.vars()) 66 replacements = dict((var, self.get(var)) for var in all_vars) 67 68 ret = [] 69 for arg in args: 70 if arg.endswith('${PREUPLOAD_FILES_PREFIXED}'): 71 if arg == '${PREUPLOAD_FILES_PREFIXED}': 72 assert len(ret) > 1, ('PREUPLOAD_FILES_PREFIXED cannot be ' 73 'the 1st or 2nd argument') 74 prev_arg = ret[-1] 75 ret = ret[0:-1] 76 for file in self.get('PREUPLOAD_FILES'): 77 ret.append(prev_arg) 78 ret.append(file) 79 else: 80 prefix = arg[0:-len('${PREUPLOAD_FILES_PREFIXED}')] 81 ret.extend( 82 prefix + file for file in self.get('PREUPLOAD_FILES')) 83 else: 84 # First scan for exact matches 85 for key, val in replacements.items(): 86 var = '${' + key + '}' 87 if arg == var: 88 if isinstance(val, str): 89 ret.append(val) 90 else: 91 ret.extend(val) 92 # We break on first hit to avoid double expansion. 93 break 94 else: 95 # If no exact matches, do an inline replacement. 96 def replace(m): 97 val = self.get(m.group(1)) 98 if isinstance(val, str): 99 return val 100 return ' '.join(val) 101 ret.append(re.sub(r'\$\{(' + '|'.join(all_vars) + r')\}', 102 replace, arg)) 103 return ret 104 105 @classmethod 106 def vars(cls): 107 """Yield all replacement variable names.""" 108 for key in dir(cls): 109 if key.startswith('var_'): 110 yield key[4:] 111 112 def get(self, var): 113 """Helper function to get the replacement |var| value.""" 114 return getattr(self, f'var_{var}') 115 116 @property 117 def var_PREUPLOAD_COMMIT_MESSAGE(self): 118 """The git commit message.""" 119 return os.environ.get('PREUPLOAD_COMMIT_MESSAGE', '') 120 121 @property 122 def var_PREUPLOAD_COMMIT(self): 123 """The git commit sha1.""" 124 return os.environ.get('PREUPLOAD_COMMIT', '') 125 126 @property 127 def var_PREUPLOAD_FILES(self): 128 """List of files modified in this git commit.""" 129 return [x.file for x in self.diff if x.status != 'D'] 130 131 @property 132 def var_REPO_PATH(self): 133 """The path to the project relative to the root""" 134 return os.environ.get('REPO_PATH', '') 135 136 @property 137 def var_REPO_ROOT(self): 138 """The root of the repo (sub-manifest) checkout.""" 139 return rh.git.find_repo_root() 140 141 @property 142 def var_REPO_OUTER_ROOT(self): 143 """The root of the repo (outer) checkout.""" 144 return rh.git.find_repo_root(outer=True) 145 146 @property 147 def var_BUILD_OS(self): 148 """The build OS (see _get_build_os_name for details).""" 149 return _get_build_os_name() 150 151 152class ExclusionScope(object): 153 """Exclusion scope for a hook. 154 155 An exclusion scope can be used to determine if a hook has been disabled for 156 a specific project. 157 """ 158 159 def __init__(self, scope): 160 """Initialize. 161 162 Args: 163 scope: A list of shell-style wildcards (fnmatch) or regular 164 expression. Regular expressions must start with the ^ character. 165 """ 166 self._scope = [] 167 for path in scope: 168 if path.startswith('^'): 169 self._scope.append(re.compile(path)) 170 else: 171 self._scope.append(path) 172 173 def __contains__(self, proj_dir): 174 """Checks if |proj_dir| matches the excluded paths. 175 176 Args: 177 proj_dir: The relative path of the project. 178 """ 179 for exclusion_path in self._scope: 180 if hasattr(exclusion_path, 'match'): 181 if exclusion_path.match(proj_dir): 182 return True 183 elif fnmatch.fnmatch(proj_dir, exclusion_path): 184 return True 185 return False 186 187 188class HookOptions(object): 189 """Holder class for hook options.""" 190 191 def __init__(self, name, args, tool_paths): 192 """Initialize. 193 194 Args: 195 name: The name of the hook. 196 args: The override commandline arguments for the hook. 197 tool_paths: A dictionary with tool names to paths. 198 """ 199 self.name = name 200 self._args = args 201 self._tool_paths = tool_paths 202 203 @staticmethod 204 def expand_vars(args, diff=()): 205 """Perform place holder expansion on all of |args|.""" 206 replacer = Placeholders(diff=diff) 207 return replacer.expand_vars(args) 208 209 def args(self, default_args=(), diff=()): 210 """Gets the hook arguments, after performing place holder expansion. 211 212 Args: 213 default_args: The list to return if |self._args| is empty. 214 diff: The list of files that changed in the current commit. 215 216 Returns: 217 A list with arguments. 218 """ 219 args = self._args 220 if not args: 221 args = default_args 222 223 return self.expand_vars(args, diff=diff) 224 225 def tool_path(self, tool_name): 226 """Gets the path in which the |tool_name| executable can be found. 227 228 This function performs expansion for some place holders. If the tool 229 does not exist in the overridden |self._tool_paths| dictionary, the tool 230 name will be returned and will be run from the user's $PATH. 231 232 Args: 233 tool_name: The name of the executable. 234 235 Returns: 236 The path of the tool with all optional place holders expanded. 237 """ 238 assert tool_name in TOOL_PATHS 239 if tool_name not in self._tool_paths: 240 return TOOL_PATHS[tool_name] 241 242 tool_path = os.path.normpath(self._tool_paths[tool_name]) 243 return self.expand_vars([tool_path])[0] 244 245 246class CallableHook(NamedTuple): 247 """A callable hook.""" 248 name: str 249 hook: Callable 250 scope: ExclusionScope 251 252 253def _run(cmd, **kwargs): 254 """Helper command for checks that tend to gather output.""" 255 kwargs.setdefault('combine_stdout_stderr', True) 256 kwargs.setdefault('capture_output', True) 257 kwargs.setdefault('check', False) 258 # Make sure hooks run with stdin disconnected to avoid accidentally 259 # interactive tools causing pauses. 260 kwargs.setdefault('input', '') 261 return rh.utils.run(cmd, **kwargs) 262 263 264def _match_regex_list(subject, expressions): 265 """Try to match a list of regular expressions to a string. 266 267 Args: 268 subject: The string to match regexes on. 269 expressions: An iterable of regular expressions to check for matches with. 270 271 Returns: 272 Whether the passed in subject matches any of the passed in regexes. 273 """ 274 for expr in expressions: 275 if re.search(expr, subject): 276 return True 277 return False 278 279 280def _filter_diff(diff, include_list, exclude_list=()): 281 """Filter out files based on the conditions passed in. 282 283 Args: 284 diff: list of diff objects to filter. 285 include_list: list of regex that when matched with a file path will cause 286 it to be added to the output list unless the file is also matched with 287 a regex in the exclude_list. 288 exclude_list: list of regex that when matched with a file will prevent it 289 from being added to the output list, even if it is also matched with a 290 regex in the include_list. 291 292 Returns: 293 A list of filepaths that contain files matched in the include_list and not 294 in the exclude_list. 295 """ 296 filtered = [] 297 for d in diff: 298 if (d.status != 'D' and 299 _match_regex_list(d.file, include_list) and 300 not _match_regex_list(d.file, exclude_list)): 301 # We've got a match! 302 filtered.append(d) 303 return filtered 304 305 306def _get_build_os_name(): 307 """Gets the build OS name. 308 309 Returns: 310 A string in a format usable to get prebuilt tool paths. 311 """ 312 system = platform.system() 313 if 'Darwin' in system or 'Macintosh' in system: 314 return 'darwin-x86' 315 316 # TODO: Add more values if needed. 317 return 'linux-x86' 318 319 320def _check_cmd(hook_name, project, commit, cmd, fixup_cmd=None, **kwargs): 321 """Runs |cmd| and returns its result as a HookCommandResult.""" 322 return [rh.results.HookCommandResult(hook_name, project, commit, 323 _run(cmd, **kwargs), 324 fixup_cmd=fixup_cmd)] 325 326 327# Where helper programs exist. 328TOOLS_DIR = os.path.realpath(__file__ + '/../../tools') 329 330def get_helper_path(tool): 331 """Return the full path to the helper |tool|.""" 332 return os.path.join(TOOLS_DIR, tool) 333 334 335def check_custom(project, commit, _desc, diff, options=None, **kwargs): 336 """Run a custom hook.""" 337 return _check_cmd(options.name, project, commit, options.args((), diff), 338 **kwargs) 339 340 341def check_bpfmt(project, commit, _desc, diff, options=None): 342 """Checks that Blueprint files are formatted with bpfmt.""" 343 filtered = _filter_diff(diff, [r'\.bp$']) 344 if not filtered: 345 return None 346 347 bpfmt = options.tool_path('bpfmt') 348 bpfmt_options = options.args((), filtered) 349 cmd = [bpfmt, '-l'] + bpfmt_options 350 ret = [] 351 for d in filtered: 352 data = rh.git.get_file_content(commit, d.file) 353 result = _run(cmd, input=data) 354 if result.stdout: 355 fixup_cmd = [bpfmt, '-w'] 356 if '-s' in bpfmt_options: 357 fixup_cmd.append('-s') 358 ret.append(rh.results.HookResult( 359 'bpfmt', project, commit, 360 error=result.stdout, 361 files=(d.file,), 362 fixup_cmd=fixup_cmd)) 363 return ret 364 365 366def check_checkpatch(project, commit, _desc, diff, options=None): 367 """Run |diff| through the kernel's checkpatch.pl tool.""" 368 tool = get_helper_path('checkpatch.pl') 369 cmd = ([tool, '-', '--root', project.dir] + 370 options.args(('--ignore=GERRIT_CHANGE_ID',), diff)) 371 return _check_cmd('checkpatch.pl', project, commit, cmd, 372 input=rh.git.get_patch(commit)) 373 374 375def check_clang_format(project, commit, _desc, diff, options=None): 376 """Run git clang-format on the commit.""" 377 tool = get_helper_path('clang-format.py') 378 clang_format = options.tool_path('clang-format') 379 git_clang_format = options.tool_path('git-clang-format') 380 tool_args = (['--clang-format', clang_format, '--git-clang-format', 381 git_clang_format] + 382 options.args(('--style', 'file', '--commit', commit), diff)) 383 cmd = [tool] + tool_args 384 fixup_cmd = [tool, '--fix'] + tool_args 385 return _check_cmd('clang-format', project, commit, cmd, 386 fixup_cmd=fixup_cmd) 387 388 389def check_google_java_format(project, commit, _desc, _diff, options=None): 390 """Run google-java-format on the commit.""" 391 392 tool = get_helper_path('google-java-format.py') 393 google_java_format = options.tool_path('google-java-format') 394 google_java_format_diff = options.tool_path('google-java-format-diff') 395 tool_args = ['--google-java-format', google_java_format, 396 '--google-java-format-diff', google_java_format_diff, 397 '--commit', commit] + options.args() 398 cmd = [tool] + tool_args 399 fixup_cmd = [tool, '--fix'] + tool_args 400 return _check_cmd('google-java-format', project, commit, cmd, 401 fixup_cmd=fixup_cmd) 402 403 404def check_ktfmt(project, commit, _desc, diff, options=None): 405 """Checks that kotlin files are formatted with ktfmt.""" 406 407 include_dir_args = [x for x in options.args() 408 if x.startswith('--include-dirs=')] 409 include_dirs = [x[len('--include-dirs='):].split(',') 410 for x in include_dir_args] 411 patterns = [fr'^{x}/.*\.kt$' for dir_list in include_dirs 412 for x in dir_list] 413 if not patterns: 414 patterns = [r'\.kt$'] 415 416 filtered = _filter_diff(diff, patterns) 417 418 if not filtered: 419 return None 420 421 args = [x for x in options.args() if x not in include_dir_args] 422 423 ktfmt = options.tool_path('ktfmt') 424 cmd = [ktfmt, '--dry-run'] + args + HookOptions.expand_vars( 425 ('${PREUPLOAD_FILES}',), filtered) 426 result = _run(cmd) 427 if result.stdout: 428 fixup_cmd = [ktfmt] + args 429 return [rh.results.HookResult( 430 'ktfmt', project, commit, error='Formatting errors detected', 431 files=[x.file for x in filtered], fixup_cmd=fixup_cmd)] 432 return None 433 434 435def check_commit_msg_bug_field(project, commit, desc, _diff, options=None): 436 """Check the commit message for a 'Bug:' line.""" 437 field = 'Bug' 438 regex = fr'^{field}: (None|[0-9]+(, [0-9]+)*)$' 439 check_re = re.compile(regex) 440 441 if options.args(): 442 raise ValueError(f'commit msg {field} check takes no options') 443 444 found = [] 445 for line in desc.splitlines(): 446 if check_re.match(line): 447 found.append(line) 448 449 if not found: 450 error = ( 451 f'Commit message is missing a "{field}:" line. It must match the\n' 452 f'following case-sensitive regex:\n\n {regex}' 453 ) 454 else: 455 return None 456 457 return [rh.results.HookResult(f'commit msg: "{field}:" check', 458 project, commit, error=error)] 459 460 461def check_commit_msg_changeid_field(project, commit, desc, _diff, options=None): 462 """Check the commit message for a 'Change-Id:' line.""" 463 field = 'Change-Id' 464 regex = fr'^{field}: I[a-f0-9]+$' 465 check_re = re.compile(regex) 466 467 if options.args(): 468 raise ValueError(f'commit msg {field} check takes no options') 469 470 found = [] 471 for line in desc.splitlines(): 472 if check_re.match(line): 473 found.append(line) 474 475 if not found: 476 error = ( 477 f'Commit message is missing a "{field}:" line. It must match the\n' 478 f'following case-sensitive regex:\n\n {regex}' 479 ) 480 elif len(found) > 1: 481 error = (f'Commit message has too many "{field}:" lines. There can be ' 482 'only one.') 483 else: 484 return None 485 486 return [rh.results.HookResult(f'commit msg: "{field}:" check', 487 project, commit, error=error)] 488 489 490PREBUILT_APK_MSG = """Commit message is missing required prebuilt APK 491information. To generate the information, use the aapt tool to dump badging 492information of the APKs being uploaded, specify where the APK was built, and 493specify whether the APKs are suitable for release: 494 495 for apk in $(find . -name '*.apk' | sort); do 496 echo "${apk}" 497 ${AAPT} dump badging "${apk}" | 498 grep -iE "(package: |sdkVersion:|targetSdkVersion:)" | 499 sed -e "s/' /'\\n/g" 500 echo 501 done 502 503It must match the following case-sensitive multiline regex searches: 504 505 %s 506 507For more information, see go/platform-prebuilt and go/android-prebuilt. 508 509""" 510 511 512def check_commit_msg_prebuilt_apk_fields(project, commit, desc, diff, 513 options=None): 514 """Check that prebuilt APK commits contain the required lines.""" 515 516 if options.args(): 517 raise ValueError('prebuilt apk check takes no options') 518 519 filtered = _filter_diff(diff, [r'\.apk$']) 520 if not filtered: 521 return None 522 523 regexes = [ 524 r'^package: .*$', 525 r'^sdkVersion:.*$', 526 r'^targetSdkVersion:.*$', 527 r'^Built here:.*$', 528 (r'^This build IS( NOT)? suitable for' 529 r'( preview|( preview or)? public) release' 530 r'( but IS NOT suitable for public release)?\.$') 531 ] 532 533 missing = [] 534 for regex in regexes: 535 if not re.search(regex, desc, re.MULTILINE): 536 missing.append(regex) 537 538 if missing: 539 error = PREBUILT_APK_MSG % '\n '.join(missing) 540 else: 541 return None 542 543 return [rh.results.HookResult('commit msg: "prebuilt apk:" check', 544 project, commit, error=error)] 545 546 547TEST_MSG = """Commit message is missing a "Test:" line. It must match the 548following case-sensitive regex: 549 550 %s 551 552The Test: stanza is free-form and should describe how you tested your change. 553As a CL author, you'll have a consistent place to describe the testing strategy 554you use for your work. As a CL reviewer, you'll be reminded to discuss testing 555as part of your code review, and you'll more easily replicate testing when you 556patch in CLs locally. 557 558Some examples below: 559 560Test: make WITH_TIDY=1 mmma art 561Test: make test-art 562Test: manual - took a photo 563Test: refactoring CL. Existing unit tests still pass. 564 565Check the git history for more examples. It's a free-form field, so we urge 566you to develop conventions that make sense for your project. Note that many 567projects use exact test commands, which are perfectly fine. 568 569Adding good automated tests with new code is critical to our goals of keeping 570the system stable and constantly improving quality. Please use Test: to 571highlight this area of your development. And reviewers, please insist on 572high-quality Test: descriptions. 573""" 574 575 576def check_commit_msg_test_field(project, commit, desc, _diff, options=None): 577 """Check the commit message for a 'Test:' line.""" 578 field = 'Test' 579 regex = fr'^{field}: .*$' 580 check_re = re.compile(regex) 581 582 if options.args(): 583 raise ValueError(f'commit msg {field} check takes no options') 584 585 found = [] 586 for line in desc.splitlines(): 587 if check_re.match(line): 588 found.append(line) 589 590 if not found: 591 error = TEST_MSG % (regex) 592 else: 593 return None 594 595 return [rh.results.HookResult(f'commit msg: "{field}:" check', 596 project, commit, error=error)] 597 598 599RELNOTE_MISSPELL_MSG = """Commit message contains something that looks 600similar to the "Relnote:" tag. It must match the regex: 601 602 %s 603 604The Relnote: stanza is free-form and should describe what developers need to 605know about your change. 606 607Some examples below: 608 609Relnote: "Added a new API `Class#isBetter` to determine whether or not the 610class is better" 611Relnote: Fixed an issue where the UI would hang on a double tap. 612 613Check the git history for more examples. It's a free-form field, so we urge 614you to develop conventions that make sense for your project. 615""" 616 617RELNOTE_MISSING_QUOTES_MSG = """Commit message contains something that looks 618similar to the "Relnote:" tag but might be malformatted. For multiline 619release notes, you need to include a starting and closing quote. 620 621Multi-line Relnote example: 622 623Relnote: "Added a new API `Class#getSize` to get the size of the class. 624 This is useful if you need to know the size of the class." 625 626Single-line Relnote example: 627 628Relnote: Added a new API `Class#containsData` 629""" 630 631RELNOTE_INVALID_QUOTES_MSG = """Commit message contains something that looks 632similar to the "Relnote:" tag but might be malformatted. If you are using 633quotes that do not mark the start or end of a Relnote, you need to escape them 634with a backslash. 635 636Non-starting/non-ending quote Relnote examples: 637 638Relnote: "Fixed an error with `Class#getBar()` where \"foo\" would be returned 639in edge cases." 640Relnote: Added a new API to handle strings like \"foo\" 641""" 642 643def check_commit_msg_relnote_field_format(project, commit, desc, _diff, 644 options=None): 645 """Check the commit for one correctly formatted 'Relnote:' line. 646 647 Checks the commit message for two things: 648 (1) Checks for possible misspellings of the 'Relnote:' tag. 649 (2) Ensures that multiline release notes are properly formatted with a 650 starting quote and an endling quote. 651 (3) Checks that release notes that contain non-starting or non-ending 652 quotes are escaped with a backslash. 653 """ 654 field = 'Relnote' 655 regex_relnote = fr'^{field}:.*$' 656 check_re_relnote = re.compile(regex_relnote, re.IGNORECASE) 657 658 if options.args(): 659 raise ValueError(f'commit msg {field} check takes no options') 660 661 # Check 1: Check for possible misspellings of the `Relnote:` field. 662 663 # Regex for misspelled fields. 664 possible_field_misspells = { 665 'Relnotes', 'ReleaseNote', 666 'Rel-note', 'Rel note', 667 'rel-notes', 'releasenotes', 668 'release-note', 'release-notes', 669 } 670 re_possible_field_misspells = '|'.join(possible_field_misspells) 671 regex_field_misspells = fr'^({re_possible_field_misspells}): .*$' 672 check_re_field_misspells = re.compile(regex_field_misspells, re.IGNORECASE) 673 674 ret = [] 675 for line in desc.splitlines(): 676 if check_re_field_misspells.match(line): 677 error = RELNOTE_MISSPELL_MSG % (regex_relnote, ) 678 ret.append( 679 rh.results.HookResult( 680 f'commit msg: "{field}:" tag spelling error', 681 project, commit, error=error)) 682 683 # Check 2: Check that multiline Relnotes are quoted. 684 685 check_re_empty_string = re.compile(r'^$') 686 687 # Regex to find other fields that could be used. 688 regex_other_fields = r'^[a-zA-Z0-9-]+:' 689 check_re_other_fields = re.compile(regex_other_fields) 690 691 desc_lines = desc.splitlines() 692 for i, cur_line in enumerate(desc_lines): 693 # Look for a Relnote tag that is before the last line and 694 # lacking any quotes. 695 if (check_re_relnote.match(cur_line) and 696 i < len(desc_lines) - 1 and 697 '"' not in cur_line): 698 next_line = desc_lines[i + 1] 699 # Check that the next line does not contain any other field 700 # and it's not an empty string. 701 if (not check_re_other_fields.findall(next_line) and 702 not check_re_empty_string.match(next_line)): 703 ret.append( 704 rh.results.HookResult( 705 f'commit msg: "{field}:" tag missing quotes', 706 project, commit, error=RELNOTE_MISSING_QUOTES_MSG)) 707 break 708 709 # Check 3: Check that multiline Relnotes contain matching quotes. 710 first_quote_found = False 711 second_quote_found = False 712 for cur_line in desc_lines: 713 contains_quote = '"' in cur_line 714 contains_field = check_re_other_fields.findall(cur_line) 715 # If we have found the first quote and another field, break and fail. 716 if first_quote_found and contains_field: 717 break 718 # If we have found the first quote, this line contains a quote, 719 # and this line is not another field, break and succeed. 720 if first_quote_found and contains_quote: 721 second_quote_found = True 722 break 723 # Check that the `Relnote:` tag exists and it contains a starting quote. 724 if check_re_relnote.match(cur_line) and contains_quote: 725 first_quote_found = True 726 # A single-line Relnote containing a start and ending triple quote 727 # is valid. 728 if cur_line.count('"""') == 2: 729 second_quote_found = True 730 break 731 # A single-line Relnote containing a start and ending quote 732 # is valid. 733 if cur_line.count('"') - cur_line.count('\\"') == 2: 734 second_quote_found = True 735 break 736 if first_quote_found != second_quote_found: 737 ret.append( 738 rh.results.HookResult( 739 f'commit msg: "{field}:" tag missing closing quote', 740 project, commit, error=RELNOTE_MISSING_QUOTES_MSG)) 741 742 # Check 4: Check that non-starting or non-ending quotes are escaped with a 743 # backslash. 744 line_needs_checking = False 745 uses_invalid_quotes = False 746 for cur_line in desc_lines: 747 if check_re_other_fields.findall(cur_line): 748 line_needs_checking = False 749 on_relnote_line = check_re_relnote.match(cur_line) 750 # Determine if we are parsing the base `Relnote:` line. 751 if on_relnote_line and '"' in cur_line: 752 line_needs_checking = True 753 # We don't think anyone will type '"""' and then forget to 754 # escape it, so we're not checking for this. 755 if '"""' in cur_line: 756 break 757 if line_needs_checking: 758 stripped_line = re.sub(fr'^{field}:', '', cur_line, 759 flags=re.IGNORECASE).strip() 760 for i, character in enumerate(stripped_line): 761 if i == 0: 762 # Case 1: Valid quote at the beginning of the 763 # base `Relnote:` line. 764 if on_relnote_line: 765 continue 766 # Case 2: Invalid quote at the beginning of following 767 # lines, where we are not terminating the release note. 768 if character == '"' and stripped_line != '"': 769 uses_invalid_quotes = True 770 break 771 # Case 3: Check all other cases. 772 if (character == '"' 773 and 0 < i < len(stripped_line) - 1 774 and stripped_line[i-1] != '"' 775 and stripped_line[i-1] != "\\"): 776 uses_invalid_quotes = True 777 break 778 779 if uses_invalid_quotes: 780 ret.append(rh.results.HookResult( 781 f'commit msg: "{field}:" tag using unescaped quotes', 782 project, commit, error=RELNOTE_INVALID_QUOTES_MSG)) 783 return ret 784 785 786RELNOTE_REQUIRED_CURRENT_TXT_MSG = """\ 787Commit contains a change to current.txt or public_plus_experimental_current.txt, 788but the commit message does not contain the required `Relnote:` tag. It must 789match the regex: 790 791 %s 792 793The Relnote: stanza is free-form and should describe what developers need to 794know about your change. If you are making infrastructure changes, you 795can set the Relnote: stanza to be "N/A" for the commit to not be included 796in release notes. 797 798Some examples: 799 800Relnote: "Added a new API `Class#isBetter` to determine whether or not the 801class is better" 802Relnote: Fixed an issue where the UI would hang on a double tap. 803Relnote: N/A 804 805Check the git history for more examples. 806""" 807 808def check_commit_msg_relnote_for_current_txt(project, commit, desc, diff, 809 options=None): 810 """Check changes to current.txt contain the 'Relnote:' stanza.""" 811 field = 'Relnote' 812 regex = fr'^{field}: .+$' 813 check_re = re.compile(regex, re.IGNORECASE) 814 815 if options.args(): 816 raise ValueError(f'commit msg {field} check takes no options') 817 818 filtered = _filter_diff( 819 diff, 820 [r'(^|/)(public_plus_experimental_current|current)\.txt$'] 821 ) 822 # If the commit does not contain a change to *current.txt, then this repo 823 # hook check no longer applies. 824 if not filtered: 825 return None 826 827 found = [] 828 for line in desc.splitlines(): 829 if check_re.match(line): 830 found.append(line) 831 832 if not found: 833 error = RELNOTE_REQUIRED_CURRENT_TXT_MSG % (regex) 834 else: 835 return None 836 837 return [rh.results.HookResult(f'commit msg: "{field}:" check', 838 project, commit, error=error)] 839 840 841def check_cpplint(project, commit, _desc, diff, options=None): 842 """Run cpplint.""" 843 # This list matches what cpplint expects. We could run on more (like .cxx), 844 # but cpplint would just ignore them. 845 filtered = _filter_diff(diff, [r'\.(cc|h|cpp|cu|cuh)$']) 846 if not filtered: 847 return None 848 849 cpplint = options.tool_path('cpplint') 850 cmd = [cpplint] + options.args(('${PREUPLOAD_FILES}',), filtered) 851 return _check_cmd('cpplint', project, commit, cmd) 852 853 854def check_gofmt(project, commit, _desc, diff, options=None): 855 """Checks that Go files are formatted with gofmt.""" 856 filtered = _filter_diff(diff, [r'\.go$']) 857 if not filtered: 858 return None 859 860 gofmt = options.tool_path('gofmt') 861 cmd = [gofmt, '-l'] + options.args() 862 fixup_cmd = [gofmt, '-w'] + options.args() 863 864 ret = [] 865 for d in filtered: 866 data = rh.git.get_file_content(commit, d.file) 867 result = _run(cmd, input=data) 868 if result.stdout: 869 ret.append(rh.results.HookResult( 870 'gofmt', project, commit, error=result.stdout, 871 files=(d.file,), fixup_cmd=fixup_cmd)) 872 return ret 873 874 875def check_json(project, commit, _desc, diff, options=None): 876 """Verify json files are valid.""" 877 if options.args(): 878 raise ValueError('json check takes no options') 879 880 filtered = _filter_diff(diff, [r'\.json$']) 881 if not filtered: 882 return None 883 884 ret = [] 885 for d in filtered: 886 data = rh.git.get_file_content(commit, d.file) 887 try: 888 json.loads(data) 889 except ValueError as e: 890 ret.append(rh.results.HookResult( 891 'json', project, commit, error=str(e), 892 files=(d.file,))) 893 return ret 894 895 896def _check_pylint(project, commit, _desc, diff, extra_args=None, options=None): 897 """Run pylint.""" 898 filtered = _filter_diff(diff, [r'\.py$']) 899 if not filtered: 900 return None 901 902 if extra_args is None: 903 extra_args = [] 904 905 pylint = options.tool_path('pylint') 906 cmd = [ 907 get_helper_path('pylint.py'), 908 '--executable-path', pylint, 909 ] + extra_args + options.args(('${PREUPLOAD_FILES}',), filtered) 910 return _check_cmd('pylint', project, commit, cmd) 911 912 913def check_pylint2(project, commit, desc, diff, options=None): 914 """Run pylint through Python 2.""" 915 return _check_pylint(project, commit, desc, diff, options=options) 916 917 918def check_pylint3(project, commit, desc, diff, options=None): 919 """Run pylint through Python 3.""" 920 return _check_pylint(project, commit, desc, diff, 921 extra_args=['--py3'], 922 options=options) 923 924 925def check_rustfmt(project, commit, _desc, diff, options=None): 926 """Run "rustfmt --check" on diffed rust files""" 927 filtered = _filter_diff(diff, [r'\.rs$']) 928 if not filtered: 929 return None 930 931 rustfmt = options.tool_path('rustfmt') 932 cmd = [rustfmt] + options.args((), filtered) 933 ret = [] 934 for d in filtered: 935 data = rh.git.get_file_content(commit, d.file) 936 result = _run(cmd, input=data) 937 # If the parsing failed, stdout will contain enough details on the 938 # location of the error. 939 if result.returncode: 940 ret.append(rh.results.HookResult( 941 'rustfmt', project, commit, error=result.stdout, 942 files=(d.file,))) 943 continue 944 # TODO(b/164111102): rustfmt stable does not support --check on stdin. 945 # If no error is reported, compare stdin with stdout. 946 if data != result.stdout: 947 ret.append(rh.results.HookResult( 948 'rustfmt', project, commit, error='Files not formatted', 949 files=(d.file,), fixup_cmd=cmd)) 950 return ret 951 952 953def check_xmllint(project, commit, _desc, diff, options=None): 954 """Run xmllint.""" 955 # XXX: Should we drop most of these and probe for <?xml> tags? 956 extensions = frozenset(( 957 'dbus-xml', # Generated DBUS interface. 958 'dia', # File format for Dia. 959 'dtd', # Document Type Definition. 960 'fml', # Fuzzy markup language. 961 'form', # Forms created by IntelliJ GUI Designer. 962 'fxml', # JavaFX user interfaces. 963 'glade', # Glade user interface design. 964 'grd', # GRIT translation files. 965 'iml', # Android build modules? 966 'kml', # Keyhole Markup Language. 967 'mxml', # Macromedia user interface markup language. 968 'nib', # OS X Cocoa Interface Builder. 969 'plist', # Property list (for OS X). 970 'pom', # Project Object Model (for Apache Maven). 971 'rng', # RELAX NG schemas. 972 'sgml', # Standard Generalized Markup Language. 973 'svg', # Scalable Vector Graphics. 974 'uml', # Unified Modeling Language. 975 'vcproj', # Microsoft Visual Studio project. 976 'vcxproj', # Microsoft Visual Studio project. 977 'wxs', # WiX Transform File. 978 'xhtml', # XML HTML. 979 'xib', # OS X Cocoa Interface Builder. 980 'xlb', # Android locale bundle. 981 'xml', # Extensible Markup Language. 982 'xsd', # XML Schema Definition. 983 'xsl', # Extensible Stylesheet Language. 984 )) 985 986 filtered = _filter_diff(diff, [r'\.(' + '|'.join(extensions) + r')$']) 987 if not filtered: 988 return None 989 990 # TODO: Figure out how to integrate schema validation. 991 # XXX: Should we use python's XML libs instead? 992 cmd = ['xmllint'] + options.args(('${PREUPLOAD_FILES}',), filtered) 993 994 return _check_cmd('xmllint', project, commit, cmd) 995 996 997def check_android_test_mapping(project, commit, _desc, diff, options=None): 998 """Verify Android TEST_MAPPING files are valid.""" 999 if options.args(): 1000 raise ValueError('Android TEST_MAPPING check takes no options') 1001 filtered = _filter_diff(diff, [r'(^|.*/)TEST_MAPPING$']) 1002 if not filtered: 1003 return None 1004 1005 testmapping_format = options.tool_path('android-test-mapping-format') 1006 testmapping_args = ['--commit', commit] 1007 cmd = [testmapping_format] + options.args( 1008 (project.dir, '${PREUPLOAD_FILES}'), filtered) + testmapping_args 1009 return _check_cmd('android-test-mapping-format', project, commit, cmd) 1010 1011 1012def check_aidl_format(project, commit, _desc, diff, options=None): 1013 """Checks that AIDL files are formatted with aidl-format.""" 1014 # All *.aidl files except for those under aidl_api directory. 1015 filtered = _filter_diff(diff, [r'\.aidl$'], [r'(^|/)aidl_api/']) 1016 if not filtered: 1017 return None 1018 aidl_format = options.tool_path('aidl-format') 1019 clang_format = options.tool_path('clang-format') 1020 diff_cmd = [aidl_format, '-d', '--clang-format-path', clang_format] + \ 1021 options.args((), filtered) 1022 ret = [] 1023 for d in filtered: 1024 data = rh.git.get_file_content(commit, d.file) 1025 result = _run(diff_cmd, input=data) 1026 if result.stdout: 1027 fixup_cmd = [aidl_format, '-w', '--clang-format-path', clang_format] 1028 ret.append(rh.results.HookResult( 1029 'aidl-format', project, commit, error=result.stdout, 1030 files=(d.file,), fixup_cmd=fixup_cmd)) 1031 return ret 1032 1033 1034# Hooks that projects can opt into. 1035# Note: Make sure to keep the top level README.md up to date when adding more! 1036BUILTIN_HOOKS = { 1037 'aidl_format': check_aidl_format, 1038 'android_test_mapping_format': check_android_test_mapping, 1039 'bpfmt': check_bpfmt, 1040 'checkpatch': check_checkpatch, 1041 'clang_format': check_clang_format, 1042 'commit_msg_bug_field': check_commit_msg_bug_field, 1043 'commit_msg_changeid_field': check_commit_msg_changeid_field, 1044 'commit_msg_prebuilt_apk_fields': check_commit_msg_prebuilt_apk_fields, 1045 'commit_msg_relnote_field_format': check_commit_msg_relnote_field_format, 1046 'commit_msg_relnote_for_current_txt': 1047 check_commit_msg_relnote_for_current_txt, 1048 'commit_msg_test_field': check_commit_msg_test_field, 1049 'cpplint': check_cpplint, 1050 'gofmt': check_gofmt, 1051 'google_java_format': check_google_java_format, 1052 'jsonlint': check_json, 1053 'ktfmt': check_ktfmt, 1054 'pylint': check_pylint2, 1055 'pylint2': check_pylint2, 1056 'pylint3': check_pylint3, 1057 'rustfmt': check_rustfmt, 1058 'xmllint': check_xmllint, 1059} 1060 1061# Additional tools that the hooks can call with their default values. 1062# Note: Make sure to keep the top level README.md up to date when adding more! 1063TOOL_PATHS = { 1064 'aidl-format': 'aidl-format', 1065 'android-test-mapping-format': 1066 os.path.join(TOOLS_DIR, 'android_test_mapping_format.py'), 1067 'bpfmt': 'bpfmt', 1068 'clang-format': 'clang-format', 1069 'cpplint': os.path.join(TOOLS_DIR, 'cpplint.py'), 1070 'git-clang-format': 'git-clang-format', 1071 'gofmt': 'gofmt', 1072 'google-java-format': 'google-java-format', 1073 'google-java-format-diff': 'google-java-format-diff.py', 1074 'ktfmt': 'ktfmt', 1075 'pylint': 'pylint', 1076 'rustfmt': 'rustfmt', 1077} 1078