1# Copyright 2016 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Functions that implement the actual checks.""" 16 17import fnmatch 18import json 19import os 20import platform 21import re 22import sys 23from typing import Callable, NamedTuple 24 25_path = os.path.realpath(__file__ + '/../..') 26if sys.path[0] != _path: 27 sys.path.insert(0, _path) 28del _path 29 30# pylint: disable=wrong-import-position 31import rh.git 32import rh.results 33import rh.utils 34 35 36class Placeholders(object): 37 """Holder class for replacing ${vars} in arg lists. 38 39 To add a new variable to replace in config files, just add it as a @property 40 to this class using the form. So to add support for BIRD: 41 @property 42 def var_BIRD(self): 43 return <whatever this is> 44 45 You can return either a string or an iterable (e.g. a list or tuple). 46 """ 47 48 def __init__(self, diff=()): 49 """Initialize. 50 51 Args: 52 diff: The list of files that changed. 53 """ 54 self.diff = diff 55 56 def expand_vars(self, args): 57 """Perform place holder expansion on all of |args|. 58 59 Args: 60 args: The args to perform expansion on. 61 62 Returns: 63 The updated |args| list. 64 """ 65 all_vars = set(self.vars()) 66 replacements = dict((var, self.get(var)) for var in all_vars) 67 68 ret = [] 69 for arg in args: 70 if arg.endswith('${PREUPLOAD_FILES_PREFIXED}'): 71 if arg == '${PREUPLOAD_FILES_PREFIXED}': 72 assert len(ret) > 1, ('PREUPLOAD_FILES_PREFIXED cannot be ' 73 'the 1st or 2nd argument') 74 prev_arg = ret[-1] 75 ret = ret[0:-1] 76 for file in self.get('PREUPLOAD_FILES'): 77 ret.append(prev_arg) 78 ret.append(file) 79 else: 80 prefix = arg[0:-len('${PREUPLOAD_FILES_PREFIXED}')] 81 ret.extend( 82 prefix + file for file in self.get('PREUPLOAD_FILES')) 83 else: 84 # First scan for exact matches 85 for key, val in replacements.items(): 86 var = '${' + key + '}' 87 if arg == var: 88 if isinstance(val, str): 89 ret.append(val) 90 else: 91 ret.extend(val) 92 # We break on first hit to avoid double expansion. 93 break 94 else: 95 # If no exact matches, do an inline replacement. 96 def replace(m): 97 val = self.get(m.group(1)) 98 if isinstance(val, str): 99 return val 100 return ' '.join(val) 101 ret.append(re.sub(r'\$\{(' + '|'.join(all_vars) + r')\}', 102 replace, arg)) 103 return ret 104 105 @classmethod 106 def vars(cls): 107 """Yield all replacement variable names.""" 108 for key in dir(cls): 109 if key.startswith('var_'): 110 yield key[4:] 111 112 def get(self, var): 113 """Helper function to get the replacement |var| value.""" 114 return getattr(self, f'var_{var}') 115 116 @property 117 def var_PREUPLOAD_COMMIT_MESSAGE(self): 118 """The git commit message.""" 119 return os.environ.get('PREUPLOAD_COMMIT_MESSAGE', '') 120 121 @property 122 def var_PREUPLOAD_COMMIT(self): 123 """The git commit sha1.""" 124 return os.environ.get('PREUPLOAD_COMMIT', '') 125 126 @property 127 def var_PREUPLOAD_FILES(self): 128 """List of files modified in this git commit.""" 129 return [x.file for x in self.diff if x.status != 'D'] 130 131 @property 132 def var_REPO_PATH(self): 133 """The path to the project relative to the root""" 134 return os.environ.get('REPO_PATH', '') 135 136 @property 137 def var_REPO_PROJECT(self): 138 """The name of the project""" 139 return os.environ.get('REPO_PROJECT', '') 140 141 @property 142 def var_REPO_ROOT(self): 143 """The root of the repo (sub-manifest) checkout.""" 144 return rh.git.find_repo_root() 145 146 @property 147 def var_REPO_OUTER_ROOT(self): 148 """The root of the repo (outer) checkout.""" 149 return rh.git.find_repo_root(outer=True) 150 151 @property 152 def var_BUILD_OS(self): 153 """The build OS (see _get_build_os_name for details).""" 154 return _get_build_os_name() 155 156 157class ExclusionScope(object): 158 """Exclusion scope for a hook. 159 160 An exclusion scope can be used to determine if a hook has been disabled for 161 a specific project. 162 """ 163 164 def __init__(self, scope): 165 """Initialize. 166 167 Args: 168 scope: A list of shell-style wildcards (fnmatch) or regular 169 expression. Regular expressions must start with the ^ character. 170 """ 171 self._scope = [] 172 for path in scope: 173 if path.startswith('^'): 174 self._scope.append(re.compile(path)) 175 else: 176 self._scope.append(path) 177 178 def __contains__(self, proj_dir): 179 """Checks if |proj_dir| matches the excluded paths. 180 181 Args: 182 proj_dir: The relative path of the project. 183 """ 184 for exclusion_path in self._scope: 185 if hasattr(exclusion_path, 'match'): 186 if exclusion_path.match(proj_dir): 187 return True 188 elif fnmatch.fnmatch(proj_dir, exclusion_path): 189 return True 190 return False 191 192 193class HookOptions(object): 194 """Holder class for hook options.""" 195 196 def __init__(self, name, args, tool_paths): 197 """Initialize. 198 199 Args: 200 name: The name of the hook. 201 args: The override commandline arguments for the hook. 202 tool_paths: A dictionary with tool names to paths. 203 """ 204 self.name = name 205 self._args = args 206 self._tool_paths = tool_paths 207 208 @staticmethod 209 def expand_vars(args, diff=()): 210 """Perform place holder expansion on all of |args|.""" 211 replacer = Placeholders(diff=diff) 212 return replacer.expand_vars(args) 213 214 def args(self, default_args=(), diff=()): 215 """Gets the hook arguments, after performing place holder expansion. 216 217 Args: 218 default_args: The list to return if |self._args| is empty. 219 diff: The list of files that changed in the current commit. 220 221 Returns: 222 A list with arguments. 223 """ 224 args = self._args 225 if not args: 226 args = default_args 227 228 return self.expand_vars(args, diff=diff) 229 230 def tool_path(self, tool_name): 231 """Gets the path in which the |tool_name| executable can be found. 232 233 This function performs expansion for some place holders. If the tool 234 does not exist in the overridden |self._tool_paths| dictionary, the tool 235 name will be returned and will be run from the user's $PATH. 236 237 Args: 238 tool_name: The name of the executable. 239 240 Returns: 241 The path of the tool with all optional place holders expanded. 242 """ 243 assert tool_name in TOOL_PATHS 244 if tool_name not in self._tool_paths: 245 return TOOL_PATHS[tool_name] 246 247 tool_path = os.path.normpath(self._tool_paths[tool_name]) 248 return self.expand_vars([tool_path])[0] 249 250 251class CallableHook(NamedTuple): 252 """A callable hook.""" 253 name: str 254 hook: Callable 255 scope: ExclusionScope 256 257 258def _run(cmd, **kwargs): 259 """Helper command for checks that tend to gather output.""" 260 kwargs.setdefault('combine_stdout_stderr', True) 261 kwargs.setdefault('capture_output', True) 262 kwargs.setdefault('check', False) 263 # Make sure hooks run with stdin disconnected to avoid accidentally 264 # interactive tools causing pauses. 265 kwargs.setdefault('input', '') 266 return rh.utils.run(cmd, **kwargs) 267 268 269def _match_regex_list(subject, expressions): 270 """Try to match a list of regular expressions to a string. 271 272 Args: 273 subject: The string to match regexes on. 274 expressions: An iterable of regular expressions to check for matches with. 275 276 Returns: 277 Whether the passed in subject matches any of the passed in regexes. 278 """ 279 for expr in expressions: 280 if re.search(expr, subject): 281 return True 282 return False 283 284 285def _filter_diff(diff, include_list, exclude_list=()): 286 """Filter out files based on the conditions passed in. 287 288 Args: 289 diff: list of diff objects to filter. 290 include_list: list of regex that when matched with a file path will cause 291 it to be added to the output list unless the file is also matched with 292 a regex in the exclude_list. 293 exclude_list: list of regex that when matched with a file will prevent it 294 from being added to the output list, even if it is also matched with a 295 regex in the include_list. 296 297 Returns: 298 A list of filepaths that contain files matched in the include_list and not 299 in the exclude_list. 300 """ 301 filtered = [] 302 for d in diff: 303 if (d.status != 'D' and 304 _match_regex_list(d.file, include_list) and 305 not _match_regex_list(d.file, exclude_list)): 306 # We've got a match! 307 filtered.append(d) 308 return filtered 309 310 311def _get_build_os_name(): 312 """Gets the build OS name. 313 314 Returns: 315 A string in a format usable to get prebuilt tool paths. 316 """ 317 system = platform.system() 318 if 'Darwin' in system or 'Macintosh' in system: 319 return 'darwin-x86' 320 321 # TODO: Add more values if needed. 322 return 'linux-x86' 323 324 325def _check_cmd(hook_name, project, commit, cmd, fixup_cmd=None, **kwargs): 326 """Runs |cmd| and returns its result as a HookCommandResult.""" 327 return [rh.results.HookCommandResult(hook_name, project, commit, 328 _run(cmd, **kwargs), 329 fixup_cmd=fixup_cmd)] 330 331 332# Where helper programs exist. 333TOOLS_DIR = os.path.realpath(__file__ + '/../../tools') 334 335def get_helper_path(tool): 336 """Return the full path to the helper |tool|.""" 337 return os.path.join(TOOLS_DIR, tool) 338 339 340def check_custom(project, commit, _desc, diff, options=None, **kwargs): 341 """Run a custom hook.""" 342 return _check_cmd(options.name, project, commit, options.args((), diff), 343 **kwargs) 344 345 346def check_aosp_license(project, commit, _desc, diff, options=None): 347 """Checks that if all new added files has AOSP licenses""" 348 349 exclude_dir_args = [x for x in options.args() 350 if x.startswith('--exclude-dirs=')] 351 exclude_dirs = [x[len('--exclude-dirs='):].split(',') 352 for x in exclude_dir_args] 353 exclude_list = [fr'^{x}/.*$' for dir_list in exclude_dirs for x in dir_list] 354 355 # Filter diff based on extension. 356 extensions = frozenset(( 357 # Coding languages and scripts. 358 'c', 359 'cc', 360 'cpp', 361 'h', 362 'java', 363 'kt', 364 'rs', 365 'py', 366 'sh', 367 368 # Build and config files. 369 'bp', 370 'mk', 371 'xml', 372 )) 373 diff = _filter_diff(diff, [r'\.(' + '|'.join(extensions) + r')$'], exclude_list) 374 375 # Only check the new-added files. 376 diff = [d for d in diff if d.status == 'A'] 377 378 if not diff: 379 return None 380 381 cmd = [get_helper_path('check_aosp_license.py'), '--commit-hash', commit] 382 cmd += HookOptions.expand_vars(('${PREUPLOAD_FILES}',), diff) 383 return _check_cmd('aosp_license', project, commit, cmd) 384 385 386def check_bpfmt(project, commit, _desc, diff, options=None): 387 """Checks that Blueprint files are formatted with bpfmt.""" 388 filtered = _filter_diff(diff, [r'\.bp$']) 389 if not filtered: 390 return None 391 392 bpfmt = options.tool_path('bpfmt') 393 bpfmt_options = options.args((), filtered) 394 cmd = [bpfmt, '-d'] + bpfmt_options 395 fixup_cmd = [bpfmt, '-w'] 396 if '-s' in bpfmt_options: 397 fixup_cmd.append('-s') 398 fixup_cmd.append('--') 399 400 ret = [] 401 for d in filtered: 402 data = rh.git.get_file_content(commit, d.file) 403 result = _run(cmd, input=data) 404 if result.stdout: 405 ret.append(rh.results.HookResult( 406 'bpfmt', project, commit, 407 error=result.stdout, 408 files=(d.file,), 409 fixup_cmd=fixup_cmd)) 410 return ret 411 412 413def check_checkpatch(project, commit, _desc, diff, options=None): 414 """Run |diff| through the kernel's checkpatch.pl tool.""" 415 tool = get_helper_path('checkpatch.pl') 416 cmd = ([tool, '-', '--root', project.dir] + 417 options.args(('--ignore=GERRIT_CHANGE_ID',), diff)) 418 return _check_cmd('checkpatch.pl', project, commit, cmd, 419 input=rh.git.get_patch(commit)) 420 421 422def check_clang_format(project, commit, _desc, diff, options=None): 423 """Run git clang-format on the commit.""" 424 tool = get_helper_path('clang-format.py') 425 clang_format = options.tool_path('clang-format') 426 git_clang_format = options.tool_path('git-clang-format') 427 tool_args = (['--clang-format', clang_format, '--git-clang-format', 428 git_clang_format] + 429 options.args(('--style', 'file', '--commit', commit), diff)) 430 cmd = [tool] + tool_args 431 fixup_cmd = [tool, '--fix'] + tool_args 432 return _check_cmd('clang-format', project, commit, cmd, 433 fixup_cmd=fixup_cmd) 434 435 436def check_google_java_format(project, commit, _desc, _diff, options=None): 437 """Run google-java-format on the commit.""" 438 include_dir_args = [x for x in options.args() 439 if x.startswith('--include-dirs=')] 440 include_dirs = [x[len('--include-dirs='):].split(',') 441 for x in include_dir_args] 442 patterns = [fr'^{x}/.*\.java$' for dir_list in include_dirs 443 for x in dir_list] 444 if not patterns: 445 patterns = [r'\.java$'] 446 447 filtered = _filter_diff(_diff, patterns) 448 449 if not filtered: 450 return None 451 452 args = [x for x in options.args() if x not in include_dir_args] 453 454 tool = get_helper_path('google-java-format.py') 455 google_java_format = options.tool_path('google-java-format') 456 google_java_format_diff = options.tool_path('google-java-format-diff') 457 tool_args = ['--google-java-format', google_java_format, 458 '--google-java-format-diff', google_java_format_diff, 459 '--commit', commit] + args 460 cmd = [tool] + tool_args + HookOptions.expand_vars( 461 ('${PREUPLOAD_FILES}',), filtered) 462 fixup_cmd = [tool, '--fix'] + tool_args 463 return [rh.results.HookCommandResult('google-java-format', project, commit, 464 _run(cmd), 465 files=[x.file for x in filtered], 466 fixup_cmd=fixup_cmd)] 467 468 469def check_ktfmt(project, commit, _desc, diff, options=None): 470 """Checks that kotlin files are formatted with ktfmt.""" 471 472 include_dir_args = [x for x in options.args() 473 if x.startswith('--include-dirs=')] 474 include_dirs = [x[len('--include-dirs='):].split(',') 475 for x in include_dir_args] 476 patterns = [fr'^{x}/.*\.kt$' for dir_list in include_dirs 477 for x in dir_list] 478 if not patterns: 479 patterns = [r'\.kt$'] 480 481 filtered = _filter_diff(diff, patterns) 482 483 if not filtered: 484 return None 485 486 args = [x for x in options.args() if x not in include_dir_args] 487 488 ktfmt = options.tool_path('ktfmt') 489 cmd = [ktfmt, '--dry-run'] + args + HookOptions.expand_vars( 490 ('${PREUPLOAD_FILES}',), filtered) 491 result = _run(cmd) 492 if result.stdout: 493 fixup_cmd = [ktfmt] + args 494 return [rh.results.HookResult( 495 'ktfmt', project, commit, error='Formatting errors detected', 496 files=[x.file for x in filtered], fixup_cmd=fixup_cmd)] 497 return None 498 499 500def check_commit_msg_bug_field(project, commit, desc, _diff, options=None): 501 """Check the commit message for a 'Bug:' or 'Fix:' line.""" 502 field = 'Bug' 503 regex = r'^(Bug|Fix): (None|[0-9]+(, [0-9]+)*)$' 504 check_re = re.compile(regex) 505 506 if options.args(): 507 raise ValueError(f'commit msg {field} check takes no options') 508 509 found = [] 510 for line in desc.splitlines(): 511 if check_re.match(line): 512 found.append(line) 513 514 if not found: 515 error = ( 516 f'Commit message is missing a "{field}:" line. It must match the\n' 517 f'following case-sensitive regex:\n\n {regex}' 518 ) 519 else: 520 return None 521 522 return [rh.results.HookResult(f'commit msg: "{field}:" check', 523 project, commit, error=error)] 524 525 526def check_commit_msg_changeid_field(project, commit, desc, _diff, options=None): 527 """Check the commit message for a 'Change-Id:' line.""" 528 field = 'Change-Id' 529 regex = fr'^{field}: I[a-f0-9]+$' 530 check_re = re.compile(regex) 531 532 if options.args(): 533 raise ValueError(f'commit msg {field} check takes no options') 534 535 found = [] 536 for line in desc.splitlines(): 537 if check_re.match(line): 538 found.append(line) 539 540 if not found: 541 error = ( 542 f'Commit message is missing a "{field}:" line. It must match the\n' 543 f'following case-sensitive regex:\n\n {regex}' 544 ) 545 elif len(found) > 1: 546 error = (f'Commit message has too many "{field}:" lines. There can be ' 547 'only one.') 548 else: 549 return None 550 551 return [rh.results.HookResult(f'commit msg: "{field}:" check', 552 project, commit, error=error)] 553 554 555PREBUILT_APK_MSG = """Commit message is missing required prebuilt APK 556information. To generate the information, use the aapt tool to dump badging 557information of the APKs being uploaded, specify where the APK was built, and 558specify whether the APKs are suitable for release: 559 560 for apk in $(find . -name '*.apk' | sort); do 561 echo "${apk}" 562 ${AAPT} dump badging "${apk}" | 563 grep -iE "(package: |sdkVersion:|targetSdkVersion:)" | 564 sed -e "s/' /'\\n/g" 565 echo 566 done 567 568It must match the following case-sensitive multiline regex searches: 569 570 %s 571 572For more information, see go/platform-prebuilt and go/android-prebuilt. 573 574""" 575 576 577def check_commit_msg_prebuilt_apk_fields(project, commit, desc, diff, 578 options=None): 579 """Check that prebuilt APK commits contain the required lines.""" 580 581 if options.args(): 582 raise ValueError('prebuilt apk check takes no options') 583 584 filtered = _filter_diff(diff, [r'\.apk$']) 585 if not filtered: 586 return None 587 588 regexes = [ 589 r'^package: .*$', 590 r'^sdkVersion:.*$', 591 r'^targetSdkVersion:.*$', 592 r'^Built here:.*$', 593 (r'^This build IS( NOT)? suitable for' 594 r'( preview|( preview or)? public) release' 595 r'( but IS NOT suitable for public release)?\.$') 596 ] 597 598 missing = [] 599 for regex in regexes: 600 if not re.search(regex, desc, re.MULTILINE): 601 missing.append(regex) 602 603 if missing: 604 error = PREBUILT_APK_MSG % '\n '.join(missing) 605 else: 606 return None 607 608 return [rh.results.HookResult('commit msg: "prebuilt apk:" check', 609 project, commit, error=error)] 610 611 612TEST_MSG = """Commit message is missing a "Test:" line. It must match the 613following case-sensitive regex: 614 615 %s 616 617The Test: stanza is free-form and should describe how you tested your change. 618As a CL author, you'll have a consistent place to describe the testing strategy 619you use for your work. As a CL reviewer, you'll be reminded to discuss testing 620as part of your code review, and you'll more easily replicate testing when you 621patch in CLs locally. 622 623Some examples below: 624 625Test: make WITH_TIDY=1 mmma art 626Test: make test-art 627Test: manual - took a photo 628Test: refactoring CL. Existing unit tests still pass. 629 630Check the git history for more examples. It's a free-form field, so we urge 631you to develop conventions that make sense for your project. Note that many 632projects use exact test commands, which are perfectly fine. 633 634Adding good automated tests with new code is critical to our goals of keeping 635the system stable and constantly improving quality. Please use Test: to 636highlight this area of your development. And reviewers, please insist on 637high-quality Test: descriptions. 638""" 639 640 641def check_commit_msg_test_field(project, commit, desc, _diff, options=None): 642 """Check the commit message for a 'Test:' line.""" 643 field = 'Test' 644 regex = fr'^{field}: .*$' 645 check_re = re.compile(regex) 646 647 if options.args(): 648 raise ValueError(f'commit msg {field} check takes no options') 649 650 found = [] 651 for line in desc.splitlines(): 652 if check_re.match(line): 653 found.append(line) 654 655 if not found: 656 error = TEST_MSG % (regex) 657 else: 658 return None 659 660 return [rh.results.HookResult(f'commit msg: "{field}:" check', 661 project, commit, error=error)] 662 663 664RELNOTE_MISSPELL_MSG = """Commit message contains something that looks 665similar to the "Relnote:" tag. It must match the regex: 666 667 %s 668 669The Relnote: stanza is free-form and should describe what developers need to 670know about your change. 671 672Some examples below: 673 674Relnote: "Added a new API `Class#isBetter` to determine whether or not the 675class is better" 676Relnote: Fixed an issue where the UI would hang on a double tap. 677 678Check the git history for more examples. It's a free-form field, so we urge 679you to develop conventions that make sense for your project. 680""" 681 682RELNOTE_MISSING_QUOTES_MSG = """Commit message contains something that looks 683similar to the "Relnote:" tag but might be malformatted. For multiline 684release notes, you need to include a starting and closing quote. 685 686Multi-line Relnote example: 687 688Relnote: "Added a new API `Class#getSize` to get the size of the class. 689 This is useful if you need to know the size of the class." 690 691Single-line Relnote example: 692 693Relnote: Added a new API `Class#containsData` 694""" 695 696RELNOTE_INVALID_QUOTES_MSG = """Commit message contains something that looks 697similar to the "Relnote:" tag but might be malformatted. If you are using 698quotes that do not mark the start or end of a Relnote, you need to escape them 699with a backslash. 700 701Non-starting/non-ending quote Relnote examples: 702 703Relnote: "Fixed an error with `Class#getBar()` where \"foo\" would be returned 704in edge cases." 705Relnote: Added a new API to handle strings like \"foo\" 706""" 707 708def check_commit_msg_relnote_field_format(project, commit, desc, _diff, 709 options=None): 710 """Check the commit for one correctly formatted 'Relnote:' line. 711 712 Checks the commit message for two things: 713 (1) Checks for possible misspellings of the 'Relnote:' tag. 714 (2) Ensures that multiline release notes are properly formatted with a 715 starting quote and an endling quote. 716 (3) Checks that release notes that contain non-starting or non-ending 717 quotes are escaped with a backslash. 718 """ 719 field = 'Relnote' 720 regex_relnote = fr'^{field}:.*$' 721 check_re_relnote = re.compile(regex_relnote, re.IGNORECASE) 722 723 if options.args(): 724 raise ValueError(f'commit msg {field} check takes no options') 725 726 # Check 1: Check for possible misspellings of the `Relnote:` field. 727 728 # Regex for misspelled fields. 729 possible_field_misspells = { 730 'Relnotes', 'ReleaseNote', 731 'Rel-note', 'Rel note', 732 'rel-notes', 'releasenotes', 733 'release-note', 'release-notes', 734 } 735 re_possible_field_misspells = '|'.join(possible_field_misspells) 736 regex_field_misspells = fr'^({re_possible_field_misspells}): .*$' 737 check_re_field_misspells = re.compile(regex_field_misspells, re.IGNORECASE) 738 739 ret = [] 740 for line in desc.splitlines(): 741 if check_re_field_misspells.match(line): 742 error = RELNOTE_MISSPELL_MSG % (regex_relnote, ) 743 ret.append( 744 rh.results.HookResult( 745 f'commit msg: "{field}:" tag spelling error', 746 project, commit, error=error)) 747 748 # Check 2: Check that multiline Relnotes are quoted. 749 750 check_re_empty_string = re.compile(r'^$') 751 752 # Regex to find other fields that could be used. 753 regex_other_fields = r'^[a-zA-Z0-9-]+:' 754 check_re_other_fields = re.compile(regex_other_fields) 755 756 desc_lines = desc.splitlines() 757 for i, cur_line in enumerate(desc_lines): 758 # Look for a Relnote tag that is before the last line and 759 # lacking any quotes. 760 if (check_re_relnote.match(cur_line) and 761 i < len(desc_lines) - 1 and 762 '"' not in cur_line): 763 next_line = desc_lines[i + 1] 764 # Check that the next line does not contain any other field 765 # and it's not an empty string. 766 if (not check_re_other_fields.findall(next_line) and 767 not check_re_empty_string.match(next_line)): 768 ret.append( 769 rh.results.HookResult( 770 f'commit msg: "{field}:" tag missing quotes', 771 project, commit, error=RELNOTE_MISSING_QUOTES_MSG)) 772 break 773 774 # Check 3: Check that multiline Relnotes contain matching quotes. 775 first_quote_found = False 776 second_quote_found = False 777 for cur_line in desc_lines: 778 contains_quote = '"' in cur_line 779 contains_field = check_re_other_fields.findall(cur_line) 780 # If we have found the first quote and another field, break and fail. 781 if first_quote_found and contains_field: 782 break 783 # If we have found the first quote, this line contains a quote, 784 # and this line is not another field, break and succeed. 785 if first_quote_found and contains_quote: 786 second_quote_found = True 787 break 788 # Check that the `Relnote:` tag exists and it contains a starting quote. 789 if check_re_relnote.match(cur_line) and contains_quote: 790 first_quote_found = True 791 # A single-line Relnote containing a start and ending triple quote 792 # is valid. 793 if cur_line.count('"""') == 2: 794 second_quote_found = True 795 break 796 # A single-line Relnote containing a start and ending quote 797 # is valid. 798 if cur_line.count('"') - cur_line.count('\\"') == 2: 799 second_quote_found = True 800 break 801 if first_quote_found != second_quote_found: 802 ret.append( 803 rh.results.HookResult( 804 f'commit msg: "{field}:" tag missing closing quote', 805 project, commit, error=RELNOTE_MISSING_QUOTES_MSG)) 806 807 # Check 4: Check that non-starting or non-ending quotes are escaped with a 808 # backslash. 809 line_needs_checking = False 810 uses_invalid_quotes = False 811 for cur_line in desc_lines: 812 if check_re_other_fields.findall(cur_line): 813 line_needs_checking = False 814 on_relnote_line = check_re_relnote.match(cur_line) 815 # Determine if we are parsing the base `Relnote:` line. 816 if on_relnote_line and '"' in cur_line: 817 line_needs_checking = True 818 # We don't think anyone will type '"""' and then forget to 819 # escape it, so we're not checking for this. 820 if '"""' in cur_line: 821 break 822 if line_needs_checking: 823 stripped_line = re.sub(fr'^{field}:', '', cur_line, 824 flags=re.IGNORECASE).strip() 825 for i, character in enumerate(stripped_line): 826 if i == 0: 827 # Case 1: Valid quote at the beginning of the 828 # base `Relnote:` line. 829 if on_relnote_line: 830 continue 831 # Case 2: Invalid quote at the beginning of following 832 # lines, where we are not terminating the release note. 833 if character == '"' and stripped_line != '"': 834 uses_invalid_quotes = True 835 break 836 # Case 3: Check all other cases. 837 if (character == '"' 838 and 0 < i < len(stripped_line) - 1 839 and stripped_line[i-1] != '"' 840 and stripped_line[i-1] != "\\"): 841 uses_invalid_quotes = True 842 break 843 844 if uses_invalid_quotes: 845 ret.append(rh.results.HookResult( 846 f'commit msg: "{field}:" tag using unescaped quotes', 847 project, commit, error=RELNOTE_INVALID_QUOTES_MSG)) 848 return ret 849 850 851RELNOTE_REQUIRED_CURRENT_TXT_MSG = """\ 852Commit contains a change to current.txt or public_plus_experimental_current.txt, 853but the commit message does not contain the required `Relnote:` tag. It must 854match the regex: 855 856 %s 857 858The Relnote: stanza is free-form and should describe what developers need to 859know about your change. If you are making infrastructure changes, you 860can set the Relnote: stanza to be "N/A" for the commit to not be included 861in release notes. 862 863Some examples: 864 865Relnote: "Added a new API `Class#isBetter` to determine whether or not the 866class is better" 867Relnote: Fixed an issue where the UI would hang on a double tap. 868Relnote: N/A 869 870Check the git history for more examples. 871""" 872 873def check_commit_msg_relnote_for_current_txt(project, commit, desc, diff, 874 options=None): 875 """Check changes to current.txt contain the 'Relnote:' stanza.""" 876 field = 'Relnote' 877 regex = fr'^{field}: .+$' 878 check_re = re.compile(regex, re.IGNORECASE) 879 880 if options.args(): 881 raise ValueError(f'commit msg {field} check takes no options') 882 883 filtered = _filter_diff( 884 diff, 885 [r'(^|/)(public_plus_experimental_current|current)\.txt$'] 886 ) 887 # If the commit does not contain a change to *current.txt, then this repo 888 # hook check no longer applies. 889 if not filtered: 890 return None 891 892 found = [] 893 for line in desc.splitlines(): 894 if check_re.match(line): 895 found.append(line) 896 897 if not found: 898 error = RELNOTE_REQUIRED_CURRENT_TXT_MSG % (regex) 899 else: 900 return None 901 902 return [rh.results.HookResult(f'commit msg: "{field}:" check', 903 project, commit, error=error)] 904 905 906def check_cpplint(project, commit, _desc, diff, options=None): 907 """Run cpplint.""" 908 # This list matches what cpplint expects. We could run on more (like .cxx), 909 # but cpplint would just ignore them. 910 filtered = _filter_diff(diff, [r'\.(cc|h|cpp|cu|cuh)$']) 911 if not filtered: 912 return None 913 914 cpplint = options.tool_path('cpplint') 915 cmd = [cpplint] + options.args(('${PREUPLOAD_FILES}',), filtered) 916 return _check_cmd('cpplint', project, commit, cmd) 917 918 919def check_gofmt(project, commit, _desc, diff, options=None): 920 """Checks that Go files are formatted with gofmt.""" 921 filtered = _filter_diff(diff, [r'\.go$']) 922 if not filtered: 923 return None 924 925 gofmt = options.tool_path('gofmt') 926 cmd = [gofmt, '-l'] + options.args() 927 fixup_cmd = [gofmt, '-w'] + options.args() 928 929 ret = [] 930 for d in filtered: 931 data = rh.git.get_file_content(commit, d.file) 932 result = _run(cmd, input=data) 933 if result.stdout: 934 ret.append(rh.results.HookResult( 935 'gofmt', project, commit, error=result.stdout, 936 files=(d.file,), fixup_cmd=fixup_cmd)) 937 return ret 938 939 940def check_json(project, commit, _desc, diff, options=None): 941 """Verify json files are valid.""" 942 if options.args(): 943 raise ValueError('json check takes no options') 944 945 filtered = _filter_diff(diff, [r'\.json$']) 946 if not filtered: 947 return None 948 949 ret = [] 950 for d in filtered: 951 data = rh.git.get_file_content(commit, d.file) 952 try: 953 json.loads(data) 954 except ValueError as e: 955 ret.append(rh.results.HookResult( 956 'json', project, commit, error=str(e), 957 files=(d.file,))) 958 return ret 959 960 961def _check_pylint(project, commit, _desc, diff, extra_args=None, options=None): 962 """Run pylint.""" 963 filtered = _filter_diff(diff, [r'\.py$']) 964 if not filtered: 965 return None 966 967 if extra_args is None: 968 extra_args = [] 969 970 pylint = options.tool_path('pylint') 971 cmd = [ 972 get_helper_path('pylint.py'), 973 '--executable-path', pylint, 974 ] + extra_args + options.args(('${PREUPLOAD_FILES}',), filtered) 975 return _check_cmd('pylint', project, commit, cmd) 976 977 978def check_pylint2(project, commit, desc, diff, options=None): 979 """Run pylint through Python 2. 980 981 This hook is not supported anymore, but we keep it registered to avoid 982 breaking in older branches with old configs that still have it. 983 """ 984 del desc, diff, options 985 return [rh.results.HookResult( 986 'pylint2', project, commit, 987 ('The pylint2 check is no longer supported. ' 988 'Please delete from PREUPLOAD.cfg.'), 989 warning=True)] 990 991 992def check_pylint3(project, commit, desc, diff, options=None): 993 """Run pylint through Python 3.""" 994 return _check_pylint(project, commit, desc, diff, options=options) 995 996 997def check_rustfmt(project, commit, _desc, diff, options=None): 998 """Run "rustfmt --check" on diffed rust files""" 999 filtered = _filter_diff(diff, [r'\.rs$']) 1000 if not filtered: 1001 return None 1002 1003 rustfmt = options.tool_path('rustfmt') 1004 cmd = [rustfmt] + options.args((), filtered) 1005 ret = [] 1006 for d in filtered: 1007 data = rh.git.get_file_content(commit, d.file) 1008 result = _run(cmd, input=data) 1009 # If the parsing failed, stdout will contain enough details on the 1010 # location of the error. 1011 if result.returncode: 1012 ret.append(rh.results.HookResult( 1013 'rustfmt', project, commit, error=result.stdout, 1014 files=(d.file,))) 1015 continue 1016 # TODO(b/164111102): rustfmt stable does not support --check on stdin. 1017 # If no error is reported, compare stdin with stdout. 1018 if data != result.stdout: 1019 ret.append(rh.results.HookResult( 1020 'rustfmt', project, commit, error='Files not formatted', 1021 files=(d.file,), fixup_cmd=cmd)) 1022 return ret 1023 1024 1025def check_xmllint(project, commit, _desc, diff, options=None): 1026 """Run xmllint.""" 1027 # XXX: Should we drop most of these and probe for <?xml> tags? 1028 extensions = frozenset(( 1029 'dbus-xml', # Generated DBUS interface. 1030 'dia', # File format for Dia. 1031 'dtd', # Document Type Definition. 1032 'fml', # Fuzzy markup language. 1033 'form', # Forms created by IntelliJ GUI Designer. 1034 'fxml', # JavaFX user interfaces. 1035 'glade', # Glade user interface design. 1036 'grd', # GRIT translation files. 1037 'iml', # Android build modules? 1038 'kml', # Keyhole Markup Language. 1039 'mxml', # Macromedia user interface markup language. 1040 'nib', # OS X Cocoa Interface Builder. 1041 'plist', # Property list (for OS X). 1042 'pom', # Project Object Model (for Apache Maven). 1043 'rng', # RELAX NG schemas. 1044 'sgml', # Standard Generalized Markup Language. 1045 'svg', # Scalable Vector Graphics. 1046 'uml', # Unified Modeling Language. 1047 'vcproj', # Microsoft Visual Studio project. 1048 'vcxproj', # Microsoft Visual Studio project. 1049 'wxs', # WiX Transform File. 1050 'xhtml', # XML HTML. 1051 'xib', # OS X Cocoa Interface Builder. 1052 'xlb', # Android locale bundle. 1053 'xml', # Extensible Markup Language. 1054 'xsd', # XML Schema Definition. 1055 'xsl', # Extensible Stylesheet Language. 1056 )) 1057 1058 filtered = _filter_diff(diff, [r'\.(' + '|'.join(extensions) + r')$']) 1059 if not filtered: 1060 return None 1061 1062 # TODO: Figure out how to integrate schema validation. 1063 # XXX: Should we use python's XML libs instead? 1064 cmd = ['xmllint'] + options.args(('${PREUPLOAD_FILES}',), filtered) 1065 1066 return _check_cmd('xmllint', project, commit, cmd) 1067 1068 1069def check_android_test_mapping(project, commit, _desc, diff, options=None): 1070 """Verify Android TEST_MAPPING files are valid.""" 1071 if options.args(): 1072 raise ValueError('Android TEST_MAPPING check takes no options') 1073 filtered = _filter_diff(diff, [r'(^|.*/)TEST_MAPPING$']) 1074 if not filtered: 1075 return None 1076 1077 testmapping_format = options.tool_path('android-test-mapping-format') 1078 testmapping_args = ['--commit', commit] 1079 cmd = [testmapping_format] + options.args( 1080 (project.dir, '${PREUPLOAD_FILES}'), filtered) + testmapping_args 1081 return _check_cmd('android-test-mapping-format', project, commit, cmd) 1082 1083 1084def check_aidl_format(project, commit, _desc, diff, options=None): 1085 """Checks that AIDL files are formatted with aidl-format.""" 1086 # All *.aidl files except for those under aidl_api directory. 1087 filtered = _filter_diff(diff, [r'\.aidl$'], [r'(^|/)aidl_api/']) 1088 if not filtered: 1089 return None 1090 aidl_format = options.tool_path('aidl-format') 1091 clang_format = options.tool_path('clang-format') 1092 diff_cmd = [aidl_format, '-d', '--clang-format-path', clang_format] + \ 1093 options.args((), filtered) 1094 ret = [] 1095 for d in filtered: 1096 data = rh.git.get_file_content(commit, d.file) 1097 result = _run(diff_cmd, input=data) 1098 if result.stdout: 1099 fixup_cmd = [aidl_format, '-w', '--clang-format-path', clang_format] 1100 ret.append(rh.results.HookResult( 1101 'aidl-format', project, commit, error=result.stdout, 1102 files=(d.file,), fixup_cmd=fixup_cmd)) 1103 return ret 1104 1105 1106# Hooks that projects can opt into. 1107# Note: Make sure to keep the top level README.md up to date when adding more! 1108BUILTIN_HOOKS = { 1109 'aidl_format': check_aidl_format, 1110 'android_test_mapping_format': check_android_test_mapping, 1111 'aosp_license': check_aosp_license, 1112 'bpfmt': check_bpfmt, 1113 'checkpatch': check_checkpatch, 1114 'clang_format': check_clang_format, 1115 'commit_msg_bug_field': check_commit_msg_bug_field, 1116 'commit_msg_changeid_field': check_commit_msg_changeid_field, 1117 'commit_msg_prebuilt_apk_fields': check_commit_msg_prebuilt_apk_fields, 1118 'commit_msg_relnote_field_format': check_commit_msg_relnote_field_format, 1119 'commit_msg_relnote_for_current_txt': 1120 check_commit_msg_relnote_for_current_txt, 1121 'commit_msg_test_field': check_commit_msg_test_field, 1122 'cpplint': check_cpplint, 1123 'gofmt': check_gofmt, 1124 'google_java_format': check_google_java_format, 1125 'jsonlint': check_json, 1126 'ktfmt': check_ktfmt, 1127 'pylint': check_pylint3, 1128 'pylint2': check_pylint2, 1129 'pylint3': check_pylint3, 1130 'rustfmt': check_rustfmt, 1131 'xmllint': check_xmllint, 1132} 1133 1134# Additional tools that the hooks can call with their default values. 1135# Note: Make sure to keep the top level README.md up to date when adding more! 1136TOOL_PATHS = { 1137 'aidl-format': 'aidl-format', 1138 'android-test-mapping-format': 1139 os.path.join(TOOLS_DIR, 'android_test_mapping_format.py'), 1140 'bpfmt': 'bpfmt', 1141 'clang-format': 'clang-format', 1142 'cpplint': os.path.join(TOOLS_DIR, 'cpplint.py'), 1143 'git-clang-format': 'git-clang-format', 1144 'gofmt': 'gofmt', 1145 'google-java-format': 'google-java-format', 1146 'google-java-format-diff': 'google-java-format-diff.py', 1147 'ktfmt': 'ktfmt', 1148 'pylint': 'pylint', 1149 'rustfmt': 'rustfmt', 1150} 1151