1# Copyright 2016 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Functions that implement the actual checks.""" 16 17import fnmatch 18import json 19import os 20import platform 21import re 22import sys 23from typing import Callable, NamedTuple 24 25_path = os.path.realpath(__file__ + '/../..') 26if sys.path[0] != _path: 27 sys.path.insert(0, _path) 28del _path 29 30# pylint: disable=wrong-import-position 31import rh.git 32import rh.results 33import rh.utils 34 35 36class Placeholders(object): 37 """Holder class for replacing ${vars} in arg lists. 38 39 To add a new variable to replace in config files, just add it as a @property 40 to this class using the form. So to add support for BIRD: 41 @property 42 def var_BIRD(self): 43 return <whatever this is> 44 45 You can return either a string or an iterable (e.g. a list or tuple). 46 """ 47 48 def __init__(self, diff=()): 49 """Initialize. 50 51 Args: 52 diff: The list of files that changed. 53 """ 54 self.diff = diff 55 56 def expand_vars(self, args): 57 """Perform place holder expansion on all of |args|. 58 59 Args: 60 args: The args to perform expansion on. 61 62 Returns: 63 The updated |args| list. 64 """ 65 all_vars = set(self.vars()) 66 replacements = dict((var, self.get(var)) for var in all_vars) 67 68 ret = [] 69 for arg in args: 70 if arg.endswith('${PREUPLOAD_FILES_PREFIXED}'): 71 if arg == '${PREUPLOAD_FILES_PREFIXED}': 72 assert len(ret) > 1, ('PREUPLOAD_FILES_PREFIXED cannot be ' 73 'the 1st or 2nd argument') 74 prev_arg = ret[-1] 75 ret = ret[0:-1] 76 for file in self.get('PREUPLOAD_FILES'): 77 ret.append(prev_arg) 78 ret.append(file) 79 else: 80 prefix = arg[0:-len('${PREUPLOAD_FILES_PREFIXED}')] 81 ret.extend( 82 prefix + file for file in self.get('PREUPLOAD_FILES')) 83 else: 84 # First scan for exact matches 85 for key, val in replacements.items(): 86 var = '${' + key + '}' 87 if arg == var: 88 if isinstance(val, str): 89 ret.append(val) 90 else: 91 ret.extend(val) 92 # We break on first hit to avoid double expansion. 93 break 94 else: 95 # If no exact matches, do an inline replacement. 96 def replace(m): 97 val = self.get(m.group(1)) 98 if isinstance(val, str): 99 return val 100 return ' '.join(val) 101 ret.append(re.sub(r'\$\{(' + '|'.join(all_vars) + r')\}', 102 replace, arg)) 103 return ret 104 105 @classmethod 106 def vars(cls): 107 """Yield all replacement variable names.""" 108 for key in dir(cls): 109 if key.startswith('var_'): 110 yield key[4:] 111 112 def get(self, var): 113 """Helper function to get the replacement |var| value.""" 114 return getattr(self, f'var_{var}') 115 116 @property 117 def var_PREUPLOAD_COMMIT_MESSAGE(self): 118 """The git commit message.""" 119 return os.environ.get('PREUPLOAD_COMMIT_MESSAGE', '') 120 121 @property 122 def var_PREUPLOAD_COMMIT(self): 123 """The git commit sha1.""" 124 return os.environ.get('PREUPLOAD_COMMIT', '') 125 126 @property 127 def var_PREUPLOAD_FILES(self): 128 """List of files modified in this git commit.""" 129 return [x.file for x in self.diff if x.status != 'D'] 130 131 @property 132 def var_REPO_PATH(self): 133 """The path to the project relative to the root""" 134 return os.environ.get('REPO_PATH', '') 135 136 @property 137 def var_REPO_PROJECT(self): 138 """The name of the project""" 139 return os.environ.get('REPO_PROJECT', '') 140 141 @property 142 def var_REPO_ROOT(self): 143 """The root of the repo (sub-manifest) checkout.""" 144 return rh.git.find_repo_root() 145 146 @property 147 def var_REPO_OUTER_ROOT(self): 148 """The root of the repo (outer) checkout.""" 149 return rh.git.find_repo_root(outer=True) 150 151 @property 152 def var_BUILD_OS(self): 153 """The build OS (see _get_build_os_name for details).""" 154 return _get_build_os_name() 155 156 157class ExclusionScope(object): 158 """Exclusion scope for a hook. 159 160 An exclusion scope can be used to determine if a hook has been disabled for 161 a specific project. 162 """ 163 164 def __init__(self, scope): 165 """Initialize. 166 167 Args: 168 scope: A list of shell-style wildcards (fnmatch) or regular 169 expression. Regular expressions must start with the ^ character. 170 """ 171 self._scope = [] 172 for path in scope: 173 if path.startswith('^'): 174 self._scope.append(re.compile(path)) 175 else: 176 self._scope.append(path) 177 178 def __contains__(self, proj_dir): 179 """Checks if |proj_dir| matches the excluded paths. 180 181 Args: 182 proj_dir: The relative path of the project. 183 """ 184 for exclusion_path in self._scope: 185 if hasattr(exclusion_path, 'match'): 186 if exclusion_path.match(proj_dir): 187 return True 188 elif fnmatch.fnmatch(proj_dir, exclusion_path): 189 return True 190 return False 191 192 193class HookOptions(object): 194 """Holder class for hook options.""" 195 196 def __init__(self, name, args, tool_paths): 197 """Initialize. 198 199 Args: 200 name: The name of the hook. 201 args: The override commandline arguments for the hook. 202 tool_paths: A dictionary with tool names to paths. 203 """ 204 self.name = name 205 self._args = args 206 self._tool_paths = tool_paths 207 208 @staticmethod 209 def expand_vars(args, diff=()): 210 """Perform place holder expansion on all of |args|.""" 211 replacer = Placeholders(diff=diff) 212 return replacer.expand_vars(args) 213 214 def args(self, default_args=(), diff=()): 215 """Gets the hook arguments, after performing place holder expansion. 216 217 Args: 218 default_args: The list to return if |self._args| is empty. 219 diff: The list of files that changed in the current commit. 220 221 Returns: 222 A list with arguments. 223 """ 224 args = self._args 225 if not args: 226 args = default_args 227 228 return self.expand_vars(args, diff=diff) 229 230 def tool_path(self, tool_name): 231 """Gets the path in which the |tool_name| executable can be found. 232 233 This function performs expansion for some place holders. If the tool 234 does not exist in the overridden |self._tool_paths| dictionary, the tool 235 name will be returned and will be run from the user's $PATH. 236 237 Args: 238 tool_name: The name of the executable. 239 240 Returns: 241 The path of the tool with all optional place holders expanded. 242 """ 243 assert tool_name in TOOL_PATHS 244 if tool_name not in self._tool_paths: 245 return TOOL_PATHS[tool_name] 246 247 tool_path = os.path.normpath(self._tool_paths[tool_name]) 248 return self.expand_vars([tool_path])[0] 249 250 251class CallableHook(NamedTuple): 252 """A callable hook.""" 253 name: str 254 hook: Callable 255 scope: ExclusionScope 256 257 258def _run(cmd, **kwargs): 259 """Helper command for checks that tend to gather output.""" 260 kwargs.setdefault('combine_stdout_stderr', True) 261 kwargs.setdefault('capture_output', True) 262 kwargs.setdefault('check', False) 263 # Make sure hooks run with stdin disconnected to avoid accidentally 264 # interactive tools causing pauses. 265 kwargs.setdefault('input', '') 266 return rh.utils.run(cmd, **kwargs) 267 268 269def _match_regex_list(subject, expressions): 270 """Try to match a list of regular expressions to a string. 271 272 Args: 273 subject: The string to match regexes on. 274 expressions: An iterable of regular expressions to check for matches with. 275 276 Returns: 277 Whether the passed in subject matches any of the passed in regexes. 278 """ 279 for expr in expressions: 280 if re.search(expr, subject): 281 return True 282 return False 283 284 285def _filter_diff(diff, include_list, exclude_list=()): 286 """Filter out files based on the conditions passed in. 287 288 Args: 289 diff: list of diff objects to filter. 290 include_list: list of regex that when matched with a file path will cause 291 it to be added to the output list unless the file is also matched with 292 a regex in the exclude_list. 293 exclude_list: list of regex that when matched with a file will prevent it 294 from being added to the output list, even if it is also matched with a 295 regex in the include_list. 296 297 Returns: 298 A list of filepaths that contain files matched in the include_list and not 299 in the exclude_list. 300 """ 301 filtered = [] 302 for d in diff: 303 if (d.status != 'D' and 304 _match_regex_list(d.file, include_list) and 305 not _match_regex_list(d.file, exclude_list)): 306 # We've got a match! 307 filtered.append(d) 308 return filtered 309 310 311def _get_build_os_name(): 312 """Gets the build OS name. 313 314 Returns: 315 A string in a format usable to get prebuilt tool paths. 316 """ 317 system = platform.system() 318 if 'Darwin' in system or 'Macintosh' in system: 319 return 'darwin-x86' 320 321 # TODO: Add more values if needed. 322 return 'linux-x86' 323 324 325def _check_cmd(hook_name, project, commit, cmd, fixup_cmd=None, **kwargs): 326 """Runs |cmd| and returns its result as a HookCommandResult.""" 327 return [rh.results.HookCommandResult(hook_name, project, commit, 328 _run(cmd, **kwargs), 329 fixup_cmd=fixup_cmd)] 330 331 332# Where helper programs exist. 333TOOLS_DIR = os.path.realpath(__file__ + '/../../tools') 334 335def get_helper_path(tool): 336 """Return the full path to the helper |tool|.""" 337 return os.path.join(TOOLS_DIR, tool) 338 339 340def check_custom(project, commit, _desc, diff, options=None, **kwargs): 341 """Run a custom hook.""" 342 return _check_cmd(options.name, project, commit, options.args((), diff), 343 **kwargs) 344 345 346def check_bpfmt(project, commit, _desc, diff, options=None): 347 """Checks that Blueprint files are formatted with bpfmt.""" 348 filtered = _filter_diff(diff, [r'\.bp$']) 349 if not filtered: 350 return None 351 352 bpfmt = options.tool_path('bpfmt') 353 bpfmt_options = options.args((), filtered) 354 cmd = [bpfmt, '-d'] + bpfmt_options 355 fixup_cmd = [bpfmt, '-w'] 356 if '-s' in bpfmt_options: 357 fixup_cmd.append('-s') 358 fixup_cmd.append('--') 359 360 ret = [] 361 for d in filtered: 362 data = rh.git.get_file_content(commit, d.file) 363 result = _run(cmd, input=data) 364 if result.stdout: 365 ret.append(rh.results.HookResult( 366 'bpfmt', project, commit, 367 error=result.stdout, 368 files=(d.file,), 369 fixup_cmd=fixup_cmd)) 370 return ret 371 372 373def check_checkpatch(project, commit, _desc, diff, options=None): 374 """Run |diff| through the kernel's checkpatch.pl tool.""" 375 tool = get_helper_path('checkpatch.pl') 376 cmd = ([tool, '-', '--root', project.dir] + 377 options.args(('--ignore=GERRIT_CHANGE_ID',), diff)) 378 return _check_cmd('checkpatch.pl', project, commit, cmd, 379 input=rh.git.get_patch(commit)) 380 381 382def check_clang_format(project, commit, _desc, diff, options=None): 383 """Run git clang-format on the commit.""" 384 tool = get_helper_path('clang-format.py') 385 clang_format = options.tool_path('clang-format') 386 git_clang_format = options.tool_path('git-clang-format') 387 tool_args = (['--clang-format', clang_format, '--git-clang-format', 388 git_clang_format] + 389 options.args(('--style', 'file', '--commit', commit), diff)) 390 cmd = [tool] + tool_args 391 fixup_cmd = [tool, '--fix'] + tool_args 392 return _check_cmd('clang-format', project, commit, cmd, 393 fixup_cmd=fixup_cmd) 394 395 396def check_google_java_format(project, commit, _desc, _diff, options=None): 397 """Run google-java-format on the commit.""" 398 include_dir_args = [x for x in options.args() 399 if x.startswith('--include-dirs=')] 400 include_dirs = [x[len('--include-dirs='):].split(',') 401 for x in include_dir_args] 402 patterns = [fr'^{x}/.*\.java$' for dir_list in include_dirs 403 for x in dir_list] 404 if not patterns: 405 patterns = [r'\.java$'] 406 407 filtered = _filter_diff(_diff, patterns) 408 409 if not filtered: 410 return None 411 412 args = [x for x in options.args() if x not in include_dir_args] 413 414 tool = get_helper_path('google-java-format.py') 415 google_java_format = options.tool_path('google-java-format') 416 google_java_format_diff = options.tool_path('google-java-format-diff') 417 tool_args = ['--google-java-format', google_java_format, 418 '--google-java-format-diff', google_java_format_diff, 419 '--commit', commit] + args 420 cmd = [tool] + tool_args + HookOptions.expand_vars( 421 ('${PREUPLOAD_FILES}',), filtered) 422 fixup_cmd = [tool, '--fix'] + tool_args 423 return [rh.results.HookCommandResult('google-java-format', project, commit, 424 _run(cmd), 425 files=[x.file for x in filtered], 426 fixup_cmd=fixup_cmd)] 427 428 429def check_ktfmt(project, commit, _desc, diff, options=None): 430 """Checks that kotlin files are formatted with ktfmt.""" 431 432 include_dir_args = [x for x in options.args() 433 if x.startswith('--include-dirs=')] 434 include_dirs = [x[len('--include-dirs='):].split(',') 435 for x in include_dir_args] 436 patterns = [fr'^{x}/.*\.kt$' for dir_list in include_dirs 437 for x in dir_list] 438 if not patterns: 439 patterns = [r'\.kt$'] 440 441 filtered = _filter_diff(diff, patterns) 442 443 if not filtered: 444 return None 445 446 args = [x for x in options.args() if x not in include_dir_args] 447 448 ktfmt = options.tool_path('ktfmt') 449 cmd = [ktfmt, '--dry-run'] + args + HookOptions.expand_vars( 450 ('${PREUPLOAD_FILES}',), filtered) 451 result = _run(cmd) 452 if result.stdout: 453 fixup_cmd = [ktfmt] + args 454 return [rh.results.HookResult( 455 'ktfmt', project, commit, error='Formatting errors detected', 456 files=[x.file for x in filtered], fixup_cmd=fixup_cmd)] 457 return None 458 459 460def check_commit_msg_bug_field(project, commit, desc, _diff, options=None): 461 """Check the commit message for a 'Bug:' line.""" 462 field = 'Bug' 463 regex = fr'^{field}: (None|[0-9]+(, [0-9]+)*)$' 464 check_re = re.compile(regex) 465 466 if options.args(): 467 raise ValueError(f'commit msg {field} check takes no options') 468 469 found = [] 470 for line in desc.splitlines(): 471 if check_re.match(line): 472 found.append(line) 473 474 if not found: 475 error = ( 476 f'Commit message is missing a "{field}:" line. It must match the\n' 477 f'following case-sensitive regex:\n\n {regex}' 478 ) 479 else: 480 return None 481 482 return [rh.results.HookResult(f'commit msg: "{field}:" check', 483 project, commit, error=error)] 484 485 486def check_commit_msg_changeid_field(project, commit, desc, _diff, options=None): 487 """Check the commit message for a 'Change-Id:' line.""" 488 field = 'Change-Id' 489 regex = fr'^{field}: I[a-f0-9]+$' 490 check_re = re.compile(regex) 491 492 if options.args(): 493 raise ValueError(f'commit msg {field} check takes no options') 494 495 found = [] 496 for line in desc.splitlines(): 497 if check_re.match(line): 498 found.append(line) 499 500 if not found: 501 error = ( 502 f'Commit message is missing a "{field}:" line. It must match the\n' 503 f'following case-sensitive regex:\n\n {regex}' 504 ) 505 elif len(found) > 1: 506 error = (f'Commit message has too many "{field}:" lines. There can be ' 507 'only one.') 508 else: 509 return None 510 511 return [rh.results.HookResult(f'commit msg: "{field}:" check', 512 project, commit, error=error)] 513 514 515PREBUILT_APK_MSG = """Commit message is missing required prebuilt APK 516information. To generate the information, use the aapt tool to dump badging 517information of the APKs being uploaded, specify where the APK was built, and 518specify whether the APKs are suitable for release: 519 520 for apk in $(find . -name '*.apk' | sort); do 521 echo "${apk}" 522 ${AAPT} dump badging "${apk}" | 523 grep -iE "(package: |sdkVersion:|targetSdkVersion:)" | 524 sed -e "s/' /'\\n/g" 525 echo 526 done 527 528It must match the following case-sensitive multiline regex searches: 529 530 %s 531 532For more information, see go/platform-prebuilt and go/android-prebuilt. 533 534""" 535 536 537def check_commit_msg_prebuilt_apk_fields(project, commit, desc, diff, 538 options=None): 539 """Check that prebuilt APK commits contain the required lines.""" 540 541 if options.args(): 542 raise ValueError('prebuilt apk check takes no options') 543 544 filtered = _filter_diff(diff, [r'\.apk$']) 545 if not filtered: 546 return None 547 548 regexes = [ 549 r'^package: .*$', 550 r'^sdkVersion:.*$', 551 r'^targetSdkVersion:.*$', 552 r'^Built here:.*$', 553 (r'^This build IS( NOT)? suitable for' 554 r'( preview|( preview or)? public) release' 555 r'( but IS NOT suitable for public release)?\.$') 556 ] 557 558 missing = [] 559 for regex in regexes: 560 if not re.search(regex, desc, re.MULTILINE): 561 missing.append(regex) 562 563 if missing: 564 error = PREBUILT_APK_MSG % '\n '.join(missing) 565 else: 566 return None 567 568 return [rh.results.HookResult('commit msg: "prebuilt apk:" check', 569 project, commit, error=error)] 570 571 572TEST_MSG = """Commit message is missing a "Test:" line. It must match the 573following case-sensitive regex: 574 575 %s 576 577The Test: stanza is free-form and should describe how you tested your change. 578As a CL author, you'll have a consistent place to describe the testing strategy 579you use for your work. As a CL reviewer, you'll be reminded to discuss testing 580as part of your code review, and you'll more easily replicate testing when you 581patch in CLs locally. 582 583Some examples below: 584 585Test: make WITH_TIDY=1 mmma art 586Test: make test-art 587Test: manual - took a photo 588Test: refactoring CL. Existing unit tests still pass. 589 590Check the git history for more examples. It's a free-form field, so we urge 591you to develop conventions that make sense for your project. Note that many 592projects use exact test commands, which are perfectly fine. 593 594Adding good automated tests with new code is critical to our goals of keeping 595the system stable and constantly improving quality. Please use Test: to 596highlight this area of your development. And reviewers, please insist on 597high-quality Test: descriptions. 598""" 599 600 601def check_commit_msg_test_field(project, commit, desc, _diff, options=None): 602 """Check the commit message for a 'Test:' line.""" 603 field = 'Test' 604 regex = fr'^{field}: .*$' 605 check_re = re.compile(regex) 606 607 if options.args(): 608 raise ValueError(f'commit msg {field} check takes no options') 609 610 found = [] 611 for line in desc.splitlines(): 612 if check_re.match(line): 613 found.append(line) 614 615 if not found: 616 error = TEST_MSG % (regex) 617 else: 618 return None 619 620 return [rh.results.HookResult(f'commit msg: "{field}:" check', 621 project, commit, error=error)] 622 623 624RELNOTE_MISSPELL_MSG = """Commit message contains something that looks 625similar to the "Relnote:" tag. It must match the regex: 626 627 %s 628 629The Relnote: stanza is free-form and should describe what developers need to 630know about your change. 631 632Some examples below: 633 634Relnote: "Added a new API `Class#isBetter` to determine whether or not the 635class is better" 636Relnote: Fixed an issue where the UI would hang on a double tap. 637 638Check the git history for more examples. It's a free-form field, so we urge 639you to develop conventions that make sense for your project. 640""" 641 642RELNOTE_MISSING_QUOTES_MSG = """Commit message contains something that looks 643similar to the "Relnote:" tag but might be malformatted. For multiline 644release notes, you need to include a starting and closing quote. 645 646Multi-line Relnote example: 647 648Relnote: "Added a new API `Class#getSize` to get the size of the class. 649 This is useful if you need to know the size of the class." 650 651Single-line Relnote example: 652 653Relnote: Added a new API `Class#containsData` 654""" 655 656RELNOTE_INVALID_QUOTES_MSG = """Commit message contains something that looks 657similar to the "Relnote:" tag but might be malformatted. If you are using 658quotes that do not mark the start or end of a Relnote, you need to escape them 659with a backslash. 660 661Non-starting/non-ending quote Relnote examples: 662 663Relnote: "Fixed an error with `Class#getBar()` where \"foo\" would be returned 664in edge cases." 665Relnote: Added a new API to handle strings like \"foo\" 666""" 667 668def check_commit_msg_relnote_field_format(project, commit, desc, _diff, 669 options=None): 670 """Check the commit for one correctly formatted 'Relnote:' line. 671 672 Checks the commit message for two things: 673 (1) Checks for possible misspellings of the 'Relnote:' tag. 674 (2) Ensures that multiline release notes are properly formatted with a 675 starting quote and an endling quote. 676 (3) Checks that release notes that contain non-starting or non-ending 677 quotes are escaped with a backslash. 678 """ 679 field = 'Relnote' 680 regex_relnote = fr'^{field}:.*$' 681 check_re_relnote = re.compile(regex_relnote, re.IGNORECASE) 682 683 if options.args(): 684 raise ValueError(f'commit msg {field} check takes no options') 685 686 # Check 1: Check for possible misspellings of the `Relnote:` field. 687 688 # Regex for misspelled fields. 689 possible_field_misspells = { 690 'Relnotes', 'ReleaseNote', 691 'Rel-note', 'Rel note', 692 'rel-notes', 'releasenotes', 693 'release-note', 'release-notes', 694 } 695 re_possible_field_misspells = '|'.join(possible_field_misspells) 696 regex_field_misspells = fr'^({re_possible_field_misspells}): .*$' 697 check_re_field_misspells = re.compile(regex_field_misspells, re.IGNORECASE) 698 699 ret = [] 700 for line in desc.splitlines(): 701 if check_re_field_misspells.match(line): 702 error = RELNOTE_MISSPELL_MSG % (regex_relnote, ) 703 ret.append( 704 rh.results.HookResult( 705 f'commit msg: "{field}:" tag spelling error', 706 project, commit, error=error)) 707 708 # Check 2: Check that multiline Relnotes are quoted. 709 710 check_re_empty_string = re.compile(r'^$') 711 712 # Regex to find other fields that could be used. 713 regex_other_fields = r'^[a-zA-Z0-9-]+:' 714 check_re_other_fields = re.compile(regex_other_fields) 715 716 desc_lines = desc.splitlines() 717 for i, cur_line in enumerate(desc_lines): 718 # Look for a Relnote tag that is before the last line and 719 # lacking any quotes. 720 if (check_re_relnote.match(cur_line) and 721 i < len(desc_lines) - 1 and 722 '"' not in cur_line): 723 next_line = desc_lines[i + 1] 724 # Check that the next line does not contain any other field 725 # and it's not an empty string. 726 if (not check_re_other_fields.findall(next_line) and 727 not check_re_empty_string.match(next_line)): 728 ret.append( 729 rh.results.HookResult( 730 f'commit msg: "{field}:" tag missing quotes', 731 project, commit, error=RELNOTE_MISSING_QUOTES_MSG)) 732 break 733 734 # Check 3: Check that multiline Relnotes contain matching quotes. 735 first_quote_found = False 736 second_quote_found = False 737 for cur_line in desc_lines: 738 contains_quote = '"' in cur_line 739 contains_field = check_re_other_fields.findall(cur_line) 740 # If we have found the first quote and another field, break and fail. 741 if first_quote_found and contains_field: 742 break 743 # If we have found the first quote, this line contains a quote, 744 # and this line is not another field, break and succeed. 745 if first_quote_found and contains_quote: 746 second_quote_found = True 747 break 748 # Check that the `Relnote:` tag exists and it contains a starting quote. 749 if check_re_relnote.match(cur_line) and contains_quote: 750 first_quote_found = True 751 # A single-line Relnote containing a start and ending triple quote 752 # is valid. 753 if cur_line.count('"""') == 2: 754 second_quote_found = True 755 break 756 # A single-line Relnote containing a start and ending quote 757 # is valid. 758 if cur_line.count('"') - cur_line.count('\\"') == 2: 759 second_quote_found = True 760 break 761 if first_quote_found != second_quote_found: 762 ret.append( 763 rh.results.HookResult( 764 f'commit msg: "{field}:" tag missing closing quote', 765 project, commit, error=RELNOTE_MISSING_QUOTES_MSG)) 766 767 # Check 4: Check that non-starting or non-ending quotes are escaped with a 768 # backslash. 769 line_needs_checking = False 770 uses_invalid_quotes = False 771 for cur_line in desc_lines: 772 if check_re_other_fields.findall(cur_line): 773 line_needs_checking = False 774 on_relnote_line = check_re_relnote.match(cur_line) 775 # Determine if we are parsing the base `Relnote:` line. 776 if on_relnote_line and '"' in cur_line: 777 line_needs_checking = True 778 # We don't think anyone will type '"""' and then forget to 779 # escape it, so we're not checking for this. 780 if '"""' in cur_line: 781 break 782 if line_needs_checking: 783 stripped_line = re.sub(fr'^{field}:', '', cur_line, 784 flags=re.IGNORECASE).strip() 785 for i, character in enumerate(stripped_line): 786 if i == 0: 787 # Case 1: Valid quote at the beginning of the 788 # base `Relnote:` line. 789 if on_relnote_line: 790 continue 791 # Case 2: Invalid quote at the beginning of following 792 # lines, where we are not terminating the release note. 793 if character == '"' and stripped_line != '"': 794 uses_invalid_quotes = True 795 break 796 # Case 3: Check all other cases. 797 if (character == '"' 798 and 0 < i < len(stripped_line) - 1 799 and stripped_line[i-1] != '"' 800 and stripped_line[i-1] != "\\"): 801 uses_invalid_quotes = True 802 break 803 804 if uses_invalid_quotes: 805 ret.append(rh.results.HookResult( 806 f'commit msg: "{field}:" tag using unescaped quotes', 807 project, commit, error=RELNOTE_INVALID_QUOTES_MSG)) 808 return ret 809 810 811RELNOTE_REQUIRED_CURRENT_TXT_MSG = """\ 812Commit contains a change to current.txt or public_plus_experimental_current.txt, 813but the commit message does not contain the required `Relnote:` tag. It must 814match the regex: 815 816 %s 817 818The Relnote: stanza is free-form and should describe what developers need to 819know about your change. If you are making infrastructure changes, you 820can set the Relnote: stanza to be "N/A" for the commit to not be included 821in release notes. 822 823Some examples: 824 825Relnote: "Added a new API `Class#isBetter` to determine whether or not the 826class is better" 827Relnote: Fixed an issue where the UI would hang on a double tap. 828Relnote: N/A 829 830Check the git history for more examples. 831""" 832 833def check_commit_msg_relnote_for_current_txt(project, commit, desc, diff, 834 options=None): 835 """Check changes to current.txt contain the 'Relnote:' stanza.""" 836 field = 'Relnote' 837 regex = fr'^{field}: .+$' 838 check_re = re.compile(regex, re.IGNORECASE) 839 840 if options.args(): 841 raise ValueError(f'commit msg {field} check takes no options') 842 843 filtered = _filter_diff( 844 diff, 845 [r'(^|/)(public_plus_experimental_current|current)\.txt$'] 846 ) 847 # If the commit does not contain a change to *current.txt, then this repo 848 # hook check no longer applies. 849 if not filtered: 850 return None 851 852 found = [] 853 for line in desc.splitlines(): 854 if check_re.match(line): 855 found.append(line) 856 857 if not found: 858 error = RELNOTE_REQUIRED_CURRENT_TXT_MSG % (regex) 859 else: 860 return None 861 862 return [rh.results.HookResult(f'commit msg: "{field}:" check', 863 project, commit, error=error)] 864 865 866def check_cpplint(project, commit, _desc, diff, options=None): 867 """Run cpplint.""" 868 # This list matches what cpplint expects. We could run on more (like .cxx), 869 # but cpplint would just ignore them. 870 filtered = _filter_diff(diff, [r'\.(cc|h|cpp|cu|cuh)$']) 871 if not filtered: 872 return None 873 874 cpplint = options.tool_path('cpplint') 875 cmd = [cpplint] + options.args(('${PREUPLOAD_FILES}',), filtered) 876 return _check_cmd('cpplint', project, commit, cmd) 877 878 879def check_gofmt(project, commit, _desc, diff, options=None): 880 """Checks that Go files are formatted with gofmt.""" 881 filtered = _filter_diff(diff, [r'\.go$']) 882 if not filtered: 883 return None 884 885 gofmt = options.tool_path('gofmt') 886 cmd = [gofmt, '-l'] + options.args() 887 fixup_cmd = [gofmt, '-w'] + options.args() 888 889 ret = [] 890 for d in filtered: 891 data = rh.git.get_file_content(commit, d.file) 892 result = _run(cmd, input=data) 893 if result.stdout: 894 ret.append(rh.results.HookResult( 895 'gofmt', project, commit, error=result.stdout, 896 files=(d.file,), fixup_cmd=fixup_cmd)) 897 return ret 898 899 900def check_json(project, commit, _desc, diff, options=None): 901 """Verify json files are valid.""" 902 if options.args(): 903 raise ValueError('json check takes no options') 904 905 filtered = _filter_diff(diff, [r'\.json$']) 906 if not filtered: 907 return None 908 909 ret = [] 910 for d in filtered: 911 data = rh.git.get_file_content(commit, d.file) 912 try: 913 json.loads(data) 914 except ValueError as e: 915 ret.append(rh.results.HookResult( 916 'json', project, commit, error=str(e), 917 files=(d.file,))) 918 return ret 919 920 921def _check_pylint(project, commit, _desc, diff, extra_args=None, options=None): 922 """Run pylint.""" 923 filtered = _filter_diff(diff, [r'\.py$']) 924 if not filtered: 925 return None 926 927 if extra_args is None: 928 extra_args = [] 929 930 pylint = options.tool_path('pylint') 931 cmd = [ 932 get_helper_path('pylint.py'), 933 '--executable-path', pylint, 934 ] + extra_args + options.args(('${PREUPLOAD_FILES}',), filtered) 935 return _check_cmd('pylint', project, commit, cmd) 936 937 938def check_pylint2(project, commit, desc, diff, options=None): 939 """Run pylint through Python 2.""" 940 return _check_pylint(project, commit, desc, diff, options=options) 941 942 943def check_pylint3(project, commit, desc, diff, options=None): 944 """Run pylint through Python 3.""" 945 return _check_pylint(project, commit, desc, diff, 946 extra_args=['--py3'], 947 options=options) 948 949 950def check_rustfmt(project, commit, _desc, diff, options=None): 951 """Run "rustfmt --check" on diffed rust files""" 952 filtered = _filter_diff(diff, [r'\.rs$']) 953 if not filtered: 954 return None 955 956 rustfmt = options.tool_path('rustfmt') 957 cmd = [rustfmt] + options.args((), filtered) 958 ret = [] 959 for d in filtered: 960 data = rh.git.get_file_content(commit, d.file) 961 result = _run(cmd, input=data) 962 # If the parsing failed, stdout will contain enough details on the 963 # location of the error. 964 if result.returncode: 965 ret.append(rh.results.HookResult( 966 'rustfmt', project, commit, error=result.stdout, 967 files=(d.file,))) 968 continue 969 # TODO(b/164111102): rustfmt stable does not support --check on stdin. 970 # If no error is reported, compare stdin with stdout. 971 if data != result.stdout: 972 ret.append(rh.results.HookResult( 973 'rustfmt', project, commit, error='Files not formatted', 974 files=(d.file,), fixup_cmd=cmd)) 975 return ret 976 977 978def check_xmllint(project, commit, _desc, diff, options=None): 979 """Run xmllint.""" 980 # XXX: Should we drop most of these and probe for <?xml> tags? 981 extensions = frozenset(( 982 'dbus-xml', # Generated DBUS interface. 983 'dia', # File format for Dia. 984 'dtd', # Document Type Definition. 985 'fml', # Fuzzy markup language. 986 'form', # Forms created by IntelliJ GUI Designer. 987 'fxml', # JavaFX user interfaces. 988 'glade', # Glade user interface design. 989 'grd', # GRIT translation files. 990 'iml', # Android build modules? 991 'kml', # Keyhole Markup Language. 992 'mxml', # Macromedia user interface markup language. 993 'nib', # OS X Cocoa Interface Builder. 994 'plist', # Property list (for OS X). 995 'pom', # Project Object Model (for Apache Maven). 996 'rng', # RELAX NG schemas. 997 'sgml', # Standard Generalized Markup Language. 998 'svg', # Scalable Vector Graphics. 999 'uml', # Unified Modeling Language. 1000 'vcproj', # Microsoft Visual Studio project. 1001 'vcxproj', # Microsoft Visual Studio project. 1002 'wxs', # WiX Transform File. 1003 'xhtml', # XML HTML. 1004 'xib', # OS X Cocoa Interface Builder. 1005 'xlb', # Android locale bundle. 1006 'xml', # Extensible Markup Language. 1007 'xsd', # XML Schema Definition. 1008 'xsl', # Extensible Stylesheet Language. 1009 )) 1010 1011 filtered = _filter_diff(diff, [r'\.(' + '|'.join(extensions) + r')$']) 1012 if not filtered: 1013 return None 1014 1015 # TODO: Figure out how to integrate schema validation. 1016 # XXX: Should we use python's XML libs instead? 1017 cmd = ['xmllint'] + options.args(('${PREUPLOAD_FILES}',), filtered) 1018 1019 return _check_cmd('xmllint', project, commit, cmd) 1020 1021 1022def check_android_test_mapping(project, commit, _desc, diff, options=None): 1023 """Verify Android TEST_MAPPING files are valid.""" 1024 if options.args(): 1025 raise ValueError('Android TEST_MAPPING check takes no options') 1026 filtered = _filter_diff(diff, [r'(^|.*/)TEST_MAPPING$']) 1027 if not filtered: 1028 return None 1029 1030 testmapping_format = options.tool_path('android-test-mapping-format') 1031 testmapping_args = ['--commit', commit] 1032 cmd = [testmapping_format] + options.args( 1033 (project.dir, '${PREUPLOAD_FILES}'), filtered) + testmapping_args 1034 return _check_cmd('android-test-mapping-format', project, commit, cmd) 1035 1036 1037def check_aidl_format(project, commit, _desc, diff, options=None): 1038 """Checks that AIDL files are formatted with aidl-format.""" 1039 # All *.aidl files except for those under aidl_api directory. 1040 filtered = _filter_diff(diff, [r'\.aidl$'], [r'(^|/)aidl_api/']) 1041 if not filtered: 1042 return None 1043 aidl_format = options.tool_path('aidl-format') 1044 clang_format = options.tool_path('clang-format') 1045 diff_cmd = [aidl_format, '-d', '--clang-format-path', clang_format] + \ 1046 options.args((), filtered) 1047 ret = [] 1048 for d in filtered: 1049 data = rh.git.get_file_content(commit, d.file) 1050 result = _run(diff_cmd, input=data) 1051 if result.stdout: 1052 fixup_cmd = [aidl_format, '-w', '--clang-format-path', clang_format] 1053 ret.append(rh.results.HookResult( 1054 'aidl-format', project, commit, error=result.stdout, 1055 files=(d.file,), fixup_cmd=fixup_cmd)) 1056 return ret 1057 1058 1059# Hooks that projects can opt into. 1060# Note: Make sure to keep the top level README.md up to date when adding more! 1061BUILTIN_HOOKS = { 1062 'aidl_format': check_aidl_format, 1063 'android_test_mapping_format': check_android_test_mapping, 1064 'bpfmt': check_bpfmt, 1065 'checkpatch': check_checkpatch, 1066 'clang_format': check_clang_format, 1067 'commit_msg_bug_field': check_commit_msg_bug_field, 1068 'commit_msg_changeid_field': check_commit_msg_changeid_field, 1069 'commit_msg_prebuilt_apk_fields': check_commit_msg_prebuilt_apk_fields, 1070 'commit_msg_relnote_field_format': check_commit_msg_relnote_field_format, 1071 'commit_msg_relnote_for_current_txt': 1072 check_commit_msg_relnote_for_current_txt, 1073 'commit_msg_test_field': check_commit_msg_test_field, 1074 'cpplint': check_cpplint, 1075 'gofmt': check_gofmt, 1076 'google_java_format': check_google_java_format, 1077 'jsonlint': check_json, 1078 'ktfmt': check_ktfmt, 1079 'pylint': check_pylint2, 1080 'pylint2': check_pylint2, 1081 'pylint3': check_pylint3, 1082 'rustfmt': check_rustfmt, 1083 'xmllint': check_xmllint, 1084} 1085 1086# Additional tools that the hooks can call with their default values. 1087# Note: Make sure to keep the top level README.md up to date when adding more! 1088TOOL_PATHS = { 1089 'aidl-format': 'aidl-format', 1090 'android-test-mapping-format': 1091 os.path.join(TOOLS_DIR, 'android_test_mapping_format.py'), 1092 'bpfmt': 'bpfmt', 1093 'clang-format': 'clang-format', 1094 'cpplint': os.path.join(TOOLS_DIR, 'cpplint.py'), 1095 'git-clang-format': 'git-clang-format', 1096 'gofmt': 'gofmt', 1097 'google-java-format': 'google-java-format', 1098 'google-java-format-diff': 'google-java-format-diff.py', 1099 'ktfmt': 'ktfmt', 1100 'pylint': 'pylint', 1101 'rustfmt': 'rustfmt', 1102} 1103