1# Copyright 2020 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Tools for running presubmit checks in a Git repository. 15 16Presubmit checks are defined as a function or other callable. The function may 17take either no arguments or a list of the paths on which to run. Presubmit 18checks communicate failure by raising any exception. 19 20For example, either of these functions may be used as presubmit checks: 21 22 @pw_presubmit.filter_paths(endswith='.py') 23 def file_contains_ni(ctx: PresubmitContext): 24 for path in ctx.paths: 25 with open(path) as file: 26 contents = file.read() 27 if 'ni' not in contents and 'nee' not in contents: 28 raise PresumitFailure('Files must say "ni"!', path=path) 29 30 def run_the_build(): 31 subprocess.run(['make', 'release'], check=True) 32 33Presubmit checks that accept a list of paths may use the filter_paths decorator 34to automatically filter the paths list for file types they care about. See the 35pragma_once function for an example. 36 37See pigweed_presbumit.py for an example of how to define presubmit checks. 38""" 39 40from __future__ import annotations 41 42import collections 43import contextlib 44import copy 45import dataclasses 46import enum 47from inspect import Parameter, signature 48import itertools 49import json 50import logging 51import os 52from pathlib import Path 53import re 54import shutil 55import signal 56import subprocess 57import sys 58import tempfile as tf 59import time 60import types 61from typing import ( 62 Any, 63 Callable, 64 Collection, 65 Dict, 66 Iterable, 67 Iterator, 68 List, 69 Optional, 70 Pattern, 71 Sequence, 72 Set, 73 Tuple, 74 Union, 75) 76import urllib 77 78import pw_cli.color 79import pw_cli.env 80import pw_env_setup.config_file 81from pw_package import package_manager 82from pw_presubmit import git_repo, tools 83from pw_presubmit.tools import plural 84 85_LOG: logging.Logger = logging.getLogger(__name__) 86 87_COLOR = pw_cli.color.colors() 88 89_SUMMARY_BOX = '══╦╗ ║║══╩╝' 90_CHECK_UPPER = '━━━┓ ' 91_CHECK_LOWER = ' ━━━┛' 92 93WIDTH = 80 94 95_LEFT = 7 96_RIGHT = 11 97 98 99def _title(msg, style=_SUMMARY_BOX) -> str: 100 msg = f' {msg} '.center(WIDTH - 2) 101 return tools.make_box('^').format(*style, section1=msg, width1=len(msg)) 102 103 104def _format_time(time_s: float) -> str: 105 minutes, seconds = divmod(time_s, 60) 106 if minutes < 60: 107 return f' {int(minutes)}:{seconds:04.1f}' 108 hours, minutes = divmod(minutes, 60) 109 return f'{int(hours):d}:{int(minutes):02}:{int(seconds):02}' 110 111 112def _box(style, left, middle, right, box=tools.make_box('><>')) -> str: 113 return box.format( 114 *style, 115 section1=left + ('' if left.endswith(' ') else ' '), 116 width1=_LEFT, 117 section2=' ' + middle, 118 width2=WIDTH - _LEFT - _RIGHT - 4, 119 section3=right + ' ', 120 width3=_RIGHT, 121 ) 122 123 124class PresubmitFailure(Exception): 125 """Optional exception to use for presubmit failures.""" 126 127 def __init__( 128 self, 129 description: str = '', 130 path: Optional[Path] = None, 131 line: Optional[int] = None, 132 ): 133 line_part: str = '' 134 if line is not None: 135 line_part = f'{line}:' 136 super().__init__( 137 f'{path}:{line_part} {description}' if path else description 138 ) 139 140 141class PresubmitResult(enum.Enum): 142 PASS = 'PASSED' # Check completed successfully. 143 FAIL = 'FAILED' # Check failed. 144 CANCEL = 'CANCEL' # Check didn't complete. 145 146 def colorized(self, width: int, invert: bool = False) -> str: 147 if self is PresubmitResult.PASS: 148 color = _COLOR.black_on_green if invert else _COLOR.green 149 elif self is PresubmitResult.FAIL: 150 color = _COLOR.black_on_red if invert else _COLOR.red 151 elif self is PresubmitResult.CANCEL: 152 color = _COLOR.yellow 153 else: 154 color = lambda value: value 155 156 padding = (width - len(self.value)) // 2 * ' ' 157 return padding + color(self.value) + padding 158 159 160class Program(collections.abc.Sequence): 161 """A sequence of presubmit checks; basically a tuple with a name.""" 162 163 def __init__(self, name: str, steps: Iterable[Callable]): 164 self.name = name 165 166 def ensure_check(step): 167 if isinstance(step, Check): 168 return step 169 return Check(step) 170 171 self._steps: tuple[Check, ...] = tuple( 172 {ensure_check(s): None for s in tools.flatten(steps)} 173 ) 174 175 def __getitem__(self, i): 176 return self._steps[i] 177 178 def __len__(self): 179 return len(self._steps) 180 181 def __str__(self): 182 return self.name 183 184 def title(self): 185 return f'{self.name if self.name else ""} presubmit checks'.strip() 186 187 188class Programs(collections.abc.Mapping): 189 """A mapping of presubmit check programs. 190 191 Use is optional. Helpful when managing multiple presubmit check programs. 192 """ 193 194 def __init__(self, **programs: Sequence): 195 """Initializes a name: program mapping from the provided keyword args. 196 197 A program is a sequence of presubmit check functions. The sequence may 198 contain nested sequences, which are flattened. 199 """ 200 self._programs: Dict[str, Program] = { 201 name: Program(name, checks) for name, checks in programs.items() 202 } 203 204 def all_steps(self) -> Dict[str, Check]: 205 return {c.name: c for c in itertools.chain(*self.values())} 206 207 def __getitem__(self, item: str) -> Program: 208 return self._programs[item] 209 210 def __iter__(self) -> Iterator[str]: 211 return iter(self._programs) 212 213 def __len__(self) -> int: 214 return len(self._programs) 215 216 217@dataclasses.dataclass(frozen=True) 218class FormatOptions: 219 python_formatter: Optional[str] = 'yapf' 220 black_path: Optional[str] = 'black' 221 222 # TODO(b/264578594) Add exclude to pigweed.json file. 223 # exclude: Sequence[re.Pattern] = dataclasses.field(default_factory=list) 224 225 @staticmethod 226 def load() -> 'FormatOptions': 227 config = pw_env_setup.config_file.load() 228 fmt = config.get('pw', {}).get('pw_presubmit', {}).get('format', {}) 229 return FormatOptions( 230 python_formatter=fmt.get('python_formatter', 'yapf'), 231 black_path=fmt.get('black_path', 'black'), 232 # exclude=tuple(re.compile(x) for x in fmt.get('exclude', ())), 233 ) 234 235 236@dataclasses.dataclass 237class LuciPipeline: 238 round: int 239 builds_from_previous_iteration: Sequence[str] 240 241 @staticmethod 242 def create( 243 bbid: int, 244 fake_pipeline_props: Optional[Dict[str, Any]] = None, 245 ) -> Optional['LuciPipeline']: 246 pipeline_props: Dict[str, Any] 247 if fake_pipeline_props is not None: 248 pipeline_props = fake_pipeline_props 249 else: 250 pipeline_props = ( 251 get_buildbucket_info(bbid) 252 .get('input', {}) 253 .get('properties', {}) 254 .get('$pigweed/pipeline', {}) 255 ) 256 if not pipeline_props.get('inside_a_pipeline', False): 257 return None 258 259 return LuciPipeline( 260 round=int(pipeline_props['round']), 261 builds_from_previous_iteration=list( 262 pipeline_props['builds_from_previous_iteration'] 263 ), 264 ) 265 266 267def get_buildbucket_info(bbid) -> Dict[str, Any]: 268 if not bbid or not shutil.which('bb'): 269 return {} 270 271 output = subprocess.check_output( 272 ['bb', 'get', '-json', '-p', f'{bbid}'], text=True 273 ) 274 return json.loads(output) 275 276 277def download_cas_artifact( 278 ctx: PresubmitContext, digest: str, output_dir: str 279) -> None: 280 """Downloads the given digest to the given outputdirectory 281 282 Args: 283 ctx: the presubmit context 284 digest: 285 a string digest in the form "<digest hash>/<size bytes>" 286 i.e 693a04e41374150d9d4b645fccb49d6f96e10b527c7a24b1e17b331f508aa73b/86 287 output_dir: the directory we want to download the artifacts to 288 """ 289 if ctx.luci is None: 290 raise PresubmitFailure('Lucicontext is None') 291 cmd = [ 292 'cas', 293 'download', 294 '-cas-instance', 295 ctx.luci.cas_instance, 296 '-digest', 297 digest, 298 '-dir', 299 output_dir, 300 ] 301 try: 302 subprocess.check_call(cmd) 303 except subprocess.CalledProcessError as failure: 304 raise PresubmitFailure('cas download failed') from failure 305 306 307def archive_cas_artifact( 308 ctx: PresubmitContext, root: str, upload_paths: List[str] 309) -> str: 310 """Uploads the given artifacts into cas 311 312 Args: 313 ctx: the presubmit context 314 root: root directory of archived tree, should be absolutepath. 315 paths: path to archived files/dirs, should be absolute path. 316 If empty, [root] will be used. 317 318 Returns: 319 A string digest in the form "<digest hash>/<size bytes>" 320 i.e 693a04e41374150d9d4b645fccb49d6f96e10b527c7a24b1e17b331f508aa73b/86 321 """ 322 if ctx.luci is None: 323 raise PresubmitFailure('Lucicontext is None') 324 assert os.path.abspath(root) 325 if not upload_paths: 326 upload_paths = [root] 327 for path in upload_paths: 328 assert os.path.abspath(path) 329 330 with tf.NamedTemporaryFile(mode='w+t') as tmp_digest_file: 331 with tf.NamedTemporaryFile(mode='w+t') as tmp_paths_file: 332 json_paths = json.dumps( 333 [ 334 [str(root), str(os.path.relpath(path, root))] 335 for path in upload_paths 336 ] 337 ) 338 tmp_paths_file.write(json_paths) 339 tmp_paths_file.seek(0) 340 cmd = [ 341 'cas', 342 'archive', 343 '-cas-instance', 344 ctx.luci.cas_instance, 345 '-paths-json', 346 tmp_paths_file.name, 347 '-dump-digest', 348 tmp_digest_file.name, 349 ] 350 try: 351 subprocess.check_call(cmd) 352 except subprocess.CalledProcessError as failure: 353 raise PresubmitFailure('cas archive failed') from failure 354 355 tmp_digest_file.seek(0) 356 uploaded_digest = tmp_digest_file.read() 357 return uploaded_digest 358 359 360@dataclasses.dataclass 361class LuciTrigger: 362 """Details the pending change or submitted commit triggering the build.""" 363 364 number: int 365 remote: str 366 branch: str 367 ref: str 368 gerrit_name: str 369 submitted: bool 370 371 @property 372 def gerrit_url(self): 373 if not self.number: 374 return self.gitiles_url 375 return 'https://{}-review.googlesource.com/c/{}'.format( 376 self.gerrit_name, self.number 377 ) 378 379 @property 380 def gitiles_url(self): 381 return '{}/+/{}'.format(self.remote, self.ref) 382 383 @staticmethod 384 def create_from_environment( 385 env: Optional[Dict[str, str]] = None, 386 ) -> Sequence['LuciTrigger']: 387 if not env: 388 env = os.environ.copy() 389 raw_path = env.get('TRIGGERING_CHANGES_JSON') 390 if not raw_path: 391 return () 392 path = Path(raw_path) 393 if not path.is_file(): 394 return () 395 396 result = [] 397 with open(path, 'r') as ins: 398 for trigger in json.load(ins): 399 keys = { 400 'number', 401 'remote', 402 'branch', 403 'ref', 404 'gerrit_name', 405 'submitted', 406 } 407 if keys <= trigger.keys(): 408 result.append(LuciTrigger(**{x: trigger[x] for x in keys})) 409 410 return tuple(result) 411 412 @staticmethod 413 def create_for_testing(): 414 change = { 415 'number': 123456, 416 'remote': 'https://pigweed.googlesource.com/pigweed/pigweed', 417 'branch': 'main', 418 'ref': 'refs/changes/56/123456/1', 419 'gerrit_name': 'pigweed', 420 'submitted': True, 421 } 422 with tf.TemporaryDirectory() as tempdir: 423 changes_json = Path(tempdir) / 'changes.json' 424 with changes_json.open('w') as outs: 425 json.dump([change], outs) 426 env = {'TRIGGERING_CHANGES_JSON': changes_json} 427 return LuciTrigger.create_from_environment(env) 428 429 430@dataclasses.dataclass 431class LuciContext: 432 """LUCI-specific information about the environment.""" 433 434 buildbucket_id: int 435 build_number: int 436 project: str 437 bucket: str 438 builder: str 439 swarming_server: str 440 swarming_task_id: str 441 cas_instance: str 442 pipeline: Optional[LuciPipeline] 443 triggers: Sequence[LuciTrigger] = dataclasses.field(default_factory=tuple) 444 445 @staticmethod 446 def create_from_environment( 447 env: Optional[Dict[str, str]] = None, 448 fake_pipeline_props: Optional[Dict[str, Any]] = None, 449 ) -> Optional['LuciContext']: 450 """Create a LuciContext from the environment.""" 451 452 if not env: 453 env = os.environ.copy() 454 455 luci_vars = [ 456 'BUILDBUCKET_ID', 457 'BUILDBUCKET_NAME', 458 'BUILD_NUMBER', 459 'SWARMING_TASK_ID', 460 'SWARMING_SERVER', 461 ] 462 if any(x for x in luci_vars if x not in env): 463 return None 464 465 project, bucket, builder = env['BUILDBUCKET_NAME'].split(':') 466 467 bbid: int = 0 468 pipeline: Optional[LuciPipeline] = None 469 try: 470 bbid = int(env['BUILDBUCKET_ID']) 471 pipeline = LuciPipeline.create(bbid, fake_pipeline_props) 472 473 except ValueError: 474 pass 475 476 # Logic to identify cas instance from swarming server is derived from 477 # https://chromium.googlesource.com/infra/luci/recipes-py/+/main/recipe_modules/cas/api.py 478 swarm_server = env['SWARMING_SERVER'] 479 cas_project = urllib.parse.urlparse(swarm_server).netloc.split('.')[0] 480 cas_instance = f'projects/{cas_project}/instances/default_instance' 481 482 result = LuciContext( 483 buildbucket_id=bbid, 484 build_number=int(env['BUILD_NUMBER']), 485 project=project, 486 bucket=bucket, 487 builder=builder, 488 swarming_server=env['SWARMING_SERVER'], 489 swarming_task_id=env['SWARMING_TASK_ID'], 490 cas_instance=cas_instance, 491 pipeline=pipeline, 492 triggers=LuciTrigger.create_from_environment(env), 493 ) 494 _LOG.debug('%r', result) 495 return result 496 497 @staticmethod 498 def create_for_testing(): 499 env = { 500 'BUILDBUCKET_ID': '881234567890', 501 'BUILDBUCKET_NAME': 'pigweed:bucket.try:builder-name', 502 'BUILD_NUMBER': '123', 503 'SWARMING_SERVER': 'https://chromium-swarm.appspot.com', 504 'SWARMING_TASK_ID': 'cd2dac62d2', 505 } 506 return LuciContext.create_from_environment(env, {}) 507 508 509@dataclasses.dataclass 510class FormatContext: 511 """Context passed into formatting helpers. 512 513 This class is a subset of PresubmitContext containing only what's needed by 514 formatters. 515 516 For full documentation on the members see the PresubmitContext section of 517 pw_presubmit/docs.rst. 518 519 Args: 520 root: Source checkout root directory 521 output_dir: Output directory for this specific language 522 paths: Modified files for the presubmit step to check (often used in 523 formatting steps but ignored in compile steps) 524 package_root: Root directory for pw package installations 525 format_options: Formatting options, derived from pigweed.json 526 """ 527 528 root: Optional[Path] 529 output_dir: Path 530 paths: Tuple[Path, ...] 531 package_root: Path 532 format_options: FormatOptions 533 534 535@dataclasses.dataclass 536class PresubmitContext: # pylint: disable=too-many-instance-attributes 537 """Context passed into presubmit checks. 538 539 For full documentation on the members see pw_presubmit/docs.rst. 540 541 Args: 542 root: Source checkout root directory 543 repos: Repositories (top-level and submodules) processed by 544 pw presubmit 545 output_dir: Output directory for this specific presubmit step 546 failure_summary_log: Path where steps should write a brief summary of 547 any failures encountered for use by other tooling. 548 paths: Modified files for the presubmit step to check (often used in 549 formatting steps but ignored in compile steps) 550 all_paths: All files in the tree. 551 package_root: Root directory for pw package installations 552 override_gn_args: Additional GN args processed by build.gn_gen() 553 luci: Information about the LUCI build or None if not running in LUCI 554 format_options: Formatting options, derived from pigweed.json 555 num_jobs: Number of jobs to run in parallel 556 continue_after_build_error: For steps that compile, don't exit on the 557 first compilation error 558 """ 559 560 root: Path 561 repos: Tuple[Path, ...] 562 output_dir: Path 563 failure_summary_log: Path 564 paths: Tuple[Path, ...] 565 all_paths: Tuple[Path, ...] 566 package_root: Path 567 luci: Optional[LuciContext] 568 override_gn_args: Dict[str, str] 569 format_options: FormatOptions 570 num_jobs: Optional[int] = None 571 continue_after_build_error: bool = False 572 _failed: bool = False 573 574 @property 575 def failed(self) -> bool: 576 return self._failed 577 578 def fail( 579 self, 580 description: str, 581 path: Optional[Path] = None, 582 line: Optional[int] = None, 583 ): 584 """Add a failure to this presubmit step. 585 586 If this is called at least once the step fails, but not immediately—the 587 check is free to continue and possibly call this method again. 588 """ 589 _LOG.warning('%s', PresubmitFailure(description, path, line)) 590 self._failed = True 591 592 @staticmethod 593 def create_for_testing(): 594 parsed_env = pw_cli.env.pigweed_environment() 595 root = parsed_env.PW_PROJECT_ROOT 596 presubmit_root = root / 'out' / 'presubmit' 597 return PresubmitContext( 598 root=root, 599 repos=(root,), 600 output_dir=presubmit_root / 'test', 601 failure_summary_log=presubmit_root / 'failure-summary.log', 602 paths=(root / 'foo.cc', root / 'foo.py'), 603 all_paths=(root / 'BUILD.gn', root / 'foo.cc', root / 'foo.py'), 604 package_root=root / 'environment' / 'packages', 605 luci=None, 606 override_gn_args={}, 607 format_options=FormatOptions(), 608 ) 609 610 611class FileFilter: 612 """Allows checking if a path matches a series of filters. 613 614 Positive filters (e.g. the file name matches a regex) and negative filters 615 (path does not match a regular expression) may be applied. 616 """ 617 618 _StrOrPattern = Union[Pattern, str] 619 620 def __init__( 621 self, 622 *, 623 exclude: Iterable[_StrOrPattern] = (), 624 endswith: Iterable[str] = (), 625 name: Iterable[_StrOrPattern] = (), 626 suffix: Iterable[str] = (), 627 ) -> None: 628 """Creates a FileFilter with the provided filters. 629 630 Args: 631 endswith: True if the end of the path is equal to any of the passed 632 strings 633 exclude: If any of the passed regular expresion match return False. 634 This overrides and other matches. 635 name: Regexs to match with file names(pathlib.Path.name). True if 636 the resulting regex matches the entire file name. 637 suffix: True if final suffix (as determined by pathlib.Path) is 638 matched by any of the passed str. 639 """ 640 self.exclude = tuple(re.compile(i) for i in exclude) 641 642 self.endswith = tuple(endswith) 643 self.name = tuple(re.compile(i) for i in name) 644 self.suffix = tuple(suffix) 645 646 def matches(self, path: Union[str, Path]) -> bool: 647 """Returns true if the path matches any filter but not an exclude. 648 649 If no positive filters are specified, any paths that do not match a 650 negative filter are considered to match. 651 652 If 'path' is a Path object it is rendered as a posix path (i.e. 653 using "/" as the path seperator) before testing with 'exclude' and 654 'endswith'. 655 """ 656 657 posix_path = path.as_posix() if isinstance(path, Path) else path 658 if any(bool(exp.search(posix_path)) for exp in self.exclude): 659 return False 660 661 # If there are no positive filters set, accept all paths. 662 no_filters = not self.endswith and not self.name and not self.suffix 663 664 path_obj = Path(path) 665 return ( 666 no_filters 667 or path_obj.suffix in self.suffix 668 or any(regex.fullmatch(path_obj.name) for regex in self.name) 669 or any(posix_path.endswith(end) for end in self.endswith) 670 ) 671 672 def filter(self, paths: Sequence[Union[str, Path]]) -> Sequence[Path]: 673 return [Path(x) for x in paths if self.matches(x)] 674 675 def apply_to_check(self, always_run: bool = False) -> Callable: 676 def wrapper(func: Callable) -> Check: 677 if isinstance(func, Check): 678 clone = copy.copy(func) 679 clone.filter = self 680 clone.always_run = clone.always_run or always_run 681 return clone 682 683 return Check(check=func, path_filter=self, always_run=always_run) 684 685 return wrapper 686 687 688def _print_ui(*args) -> None: 689 """Prints to stdout and flushes to stay in sync with logs on stderr.""" 690 print(*args, flush=True) 691 692 693@dataclasses.dataclass 694class FilteredCheck: 695 check: Check 696 paths: Sequence[Path] 697 substep: Optional[str] = None 698 699 @property 700 def name(self) -> str: 701 return self.check.name 702 703 def run(self, ctx: PresubmitContext, count: int, total: int): 704 return self.check.run(ctx, count, total, self.substep) 705 706 707class Presubmit: 708 """Runs a series of presubmit checks on a list of files.""" 709 710 def __init__( 711 self, 712 root: Path, 713 repos: Sequence[Path], 714 output_directory: Path, 715 paths: Sequence[Path], 716 all_paths: Sequence[Path], 717 package_root: Path, 718 override_gn_args: Dict[str, str], 719 continue_after_build_error: bool, 720 ): 721 self._root = root.resolve() 722 self._repos = tuple(repos) 723 self._output_directory = output_directory.resolve() 724 self._paths = tuple(paths) 725 self._all_paths = tuple(all_paths) 726 self._relative_paths = tuple( 727 tools.relative_paths(self._paths, self._root) 728 ) 729 self._package_root = package_root.resolve() 730 self._override_gn_args = override_gn_args 731 self._continue_after_build_error = continue_after_build_error 732 733 def run( 734 self, 735 program: Program, 736 keep_going: bool = False, 737 substep: Optional[str] = None, 738 ) -> bool: 739 """Executes a series of presubmit checks on the paths.""" 740 741 checks = self.apply_filters(program) 742 if substep: 743 assert ( 744 len(checks) == 1 745 ), 'substeps not supported with multiple steps' 746 checks[0].substep = substep 747 748 _LOG.debug('Running %s for %s', program.title(), self._root.name) 749 _print_ui(_title(f'{self._root.name}: {program.title()}')) 750 751 _LOG.info( 752 '%d of %d checks apply to %s in %s', 753 len(checks), 754 len(program), 755 plural(self._paths, 'file'), 756 self._root, 757 ) 758 759 _print_ui() 760 for line in tools.file_summary(self._relative_paths): 761 _print_ui(line) 762 _print_ui() 763 764 if not self._paths: 765 _print_ui(_COLOR.yellow('No files are being checked!')) 766 767 _LOG.debug('Checks:\n%s', '\n'.join(c.name for c in checks)) 768 769 start_time: float = time.time() 770 passed, failed, skipped = self._execute_checks(checks, keep_going) 771 self._log_summary(time.time() - start_time, passed, failed, skipped) 772 773 return not failed and not skipped 774 775 def apply_filters(self, program: Sequence[Callable]) -> List[FilteredCheck]: 776 """Returns list of FilteredCheck for checks that should run.""" 777 checks = [c if isinstance(c, Check) else Check(c) for c in program] 778 filter_to_checks: Dict[ 779 FileFilter, List[Check] 780 ] = collections.defaultdict(list) 781 782 for chk in checks: 783 filter_to_checks[chk.filter].append(chk) 784 785 check_to_paths = self._map_checks_to_paths(filter_to_checks) 786 return [ 787 FilteredCheck(c, check_to_paths[c]) 788 for c in checks 789 if c in check_to_paths 790 ] 791 792 def _map_checks_to_paths( 793 self, filter_to_checks: Dict[FileFilter, List[Check]] 794 ) -> Dict[Check, Sequence[Path]]: 795 checks_to_paths: Dict[Check, Sequence[Path]] = {} 796 797 posix_paths = tuple(p.as_posix() for p in self._relative_paths) 798 799 for filt, checks in filter_to_checks.items(): 800 filtered_paths = tuple( 801 path 802 for path, filter_path in zip(self._paths, posix_paths) 803 if filt.matches(filter_path) 804 ) 805 806 for chk in checks: 807 if filtered_paths or chk.always_run: 808 checks_to_paths[chk] = filtered_paths 809 else: 810 _LOG.debug('Skipping "%s": no relevant files', chk.name) 811 812 return checks_to_paths 813 814 def _log_summary( 815 self, time_s: float, passed: int, failed: int, skipped: int 816 ) -> None: 817 summary_items = [] 818 if passed: 819 summary_items.append(f'{passed} passed') 820 if failed: 821 summary_items.append(f'{failed} failed') 822 if skipped: 823 summary_items.append(f'{skipped} not run') 824 summary = ', '.join(summary_items) or 'nothing was done' 825 826 if failed or skipped: 827 result = PresubmitResult.FAIL 828 else: 829 result = PresubmitResult.PASS 830 total = passed + failed + skipped 831 832 _LOG.debug( 833 'Finished running %d checks on %s in %.1f s', 834 total, 835 plural(self._paths, 'file'), 836 time_s, 837 ) 838 _LOG.debug('Presubmit checks %s: %s', result.value, summary) 839 840 _print_ui( 841 _box( 842 _SUMMARY_BOX, 843 result.colorized(_LEFT, invert=True), 844 f'{total} checks on {plural(self._paths, "file")}: {summary}', 845 _format_time(time_s), 846 ) 847 ) 848 849 def _create_presubmit_context( # pylint: disable=no-self-use 850 self, **kwargs 851 ): 852 """Create a PresubmitContext. Override if needed in subclasses.""" 853 return PresubmitContext(**kwargs) 854 855 @contextlib.contextmanager 856 def _context(self, filtered_check: FilteredCheck): 857 # There are many characters banned from filenames on Windows. To 858 # simplify things, just strip everything that's not a letter, digit, 859 # or underscore. 860 sanitized_name = re.sub(r'[\W_]+', '_', filtered_check.name).lower() 861 output_directory = self._output_directory.joinpath(sanitized_name) 862 os.makedirs(output_directory, exist_ok=True) 863 864 failure_summary_log = output_directory / 'failure-summary.log' 865 failure_summary_log.unlink(missing_ok=True) 866 867 handler = logging.FileHandler( 868 output_directory.joinpath('step.log'), mode='w' 869 ) 870 handler.setLevel(logging.DEBUG) 871 872 try: 873 _LOG.addHandler(handler) 874 875 yield self._create_presubmit_context( 876 root=self._root, 877 repos=self._repos, 878 output_dir=output_directory, 879 failure_summary_log=failure_summary_log, 880 paths=filtered_check.paths, 881 all_paths=self._all_paths, 882 package_root=self._package_root, 883 override_gn_args=self._override_gn_args, 884 continue_after_build_error=self._continue_after_build_error, 885 luci=LuciContext.create_from_environment(), 886 format_options=FormatOptions.load(), 887 ) 888 889 finally: 890 _LOG.removeHandler(handler) 891 892 def _execute_checks( 893 self, program: List[FilteredCheck], keep_going: bool 894 ) -> Tuple[int, int, int]: 895 """Runs presubmit checks; returns (passed, failed, skipped) lists.""" 896 passed = failed = 0 897 898 for i, filtered_check in enumerate(program, 1): 899 with self._context(filtered_check) as ctx: 900 result = filtered_check.run(ctx, i, len(program)) 901 902 if result is PresubmitResult.PASS: 903 passed += 1 904 elif result is PresubmitResult.CANCEL: 905 break 906 else: 907 failed += 1 908 if not keep_going: 909 break 910 911 return passed, failed, len(program) - passed - failed 912 913 914def _process_pathspecs( 915 repos: Iterable[Path], pathspecs: Iterable[str] 916) -> Dict[Path, List[str]]: 917 pathspecs_by_repo: Dict[Path, List[str]] = {repo: [] for repo in repos} 918 repos_with_paths: Set[Path] = set() 919 920 for pathspec in pathspecs: 921 # If the pathspec is a path to an existing file, only use it for the 922 # repo it is in. 923 if os.path.exists(pathspec): 924 # Raise an exception if the path exists but is not in a known repo. 925 repo = git_repo.within_repo(pathspec) 926 if repo not in pathspecs_by_repo: 927 raise ValueError( 928 f'{pathspec} is not in a Git repository in this presubmit' 929 ) 930 931 # Make the path relative to the repo's root. 932 pathspecs_by_repo[repo].append(os.path.relpath(pathspec, repo)) 933 repos_with_paths.add(repo) 934 else: 935 # Pathspecs that are not paths (e.g. '*.h') are used for all repos. 936 for patterns in pathspecs_by_repo.values(): 937 patterns.append(pathspec) 938 939 # If any paths were specified, only search for paths in those repos. 940 if repos_with_paths: 941 for repo in set(pathspecs_by_repo) - repos_with_paths: 942 del pathspecs_by_repo[repo] 943 944 return pathspecs_by_repo 945 946 947def run( # pylint: disable=too-many-arguments,too-many-locals 948 program: Sequence[Check], 949 root: Path, 950 repos: Collection[Path] = (), 951 base: Optional[str] = None, 952 paths: Sequence[str] = (), 953 exclude: Sequence[Pattern] = (), 954 output_directory: Optional[Path] = None, 955 package_root: Optional[Path] = None, 956 only_list_steps: bool = False, 957 override_gn_args: Sequence[Tuple[str, str]] = (), 958 keep_going: bool = False, 959 continue_after_build_error: bool = False, 960 presubmit_class: type = Presubmit, 961 list_steps_file: Optional[Path] = None, 962 substep: Optional[str] = None, 963) -> bool: 964 """Lists files in the current Git repo and runs a Presubmit with them. 965 966 This changes the directory to the root of the Git repository after listing 967 paths, so all presubmit checks can assume they run from there. 968 969 The paths argument contains Git pathspecs. If no pathspecs are provided, all 970 paths in all repos are included. If paths to files or directories are 971 provided, only files within those repositories are searched. Patterns are 972 searched across all repositories. For example, if the pathspecs "my_module/" 973 and "*.h", paths under "my_module/" in the containing repo and paths in all 974 repos matching "*.h" will be included in the presubmit. 975 976 Args: 977 program: list of presubmit check functions to run 978 root: root path of the project 979 repos: paths to the roots of Git repositories to check 980 name: name to use to refer to this presubmit check run 981 base: optional base Git commit to list files against 982 paths: optional list of Git pathspecs to run the checks against 983 exclude: regular expressions for Posix-style paths to exclude 984 output_directory: where to place output files 985 package_root: where to place package files 986 only_list_steps: print step names instead of running them 987 override_gn_args: additional GN args to set on steps 988 keep_going: continue running presubmit steps after a step fails 989 continue_after_build_error: continue building if a build step fails 990 presubmit_class: class to use to run Presubmits, should inherit from 991 Presubmit class above 992 list_steps_file: File created by --only-list-steps, used to keep from 993 recalculating affected files. 994 substep: run only part of a single check 995 996 Returns: 997 True if all presubmit checks succeeded 998 """ 999 repos = [repo.resolve() for repo in repos] 1000 1001 non_empty_repos = [] 1002 for repo in repos: 1003 if list(repo.iterdir()): 1004 non_empty_repos.append(repo) 1005 if git_repo.root(repo) != repo: 1006 raise ValueError( 1007 f'{repo} is not the root of a Git repo; ' 1008 'presubmit checks must be run from a Git repo' 1009 ) 1010 repos = non_empty_repos 1011 1012 pathspecs_by_repo = _process_pathspecs(repos, paths) 1013 1014 all_files: List[Path] = [] 1015 modified_files: List[Path] = [] 1016 list_steps_data: Dict[str, Any] = {} 1017 1018 if list_steps_file: 1019 with list_steps_file.open() as ins: 1020 list_steps_data = json.load(ins) 1021 all_files.extend(list_steps_data['all_files']) 1022 for step in list_steps_data['steps']: 1023 modified_files.extend(Path(x) for x in step.get("paths", ())) 1024 modified_files = sorted(set(modified_files)) 1025 _LOG.info( 1026 'Loaded %d paths from file %s', 1027 len(modified_files), 1028 list_steps_file, 1029 ) 1030 1031 else: 1032 for repo, pathspecs in pathspecs_by_repo.items(): 1033 all_files_repo = tuple( 1034 tools.exclude_paths( 1035 exclude, git_repo.list_files(None, pathspecs, repo), root 1036 ) 1037 ) 1038 all_files += all_files_repo 1039 1040 if base is None: 1041 modified_files += all_files_repo 1042 else: 1043 modified_files += tools.exclude_paths( 1044 exclude, git_repo.list_files(base, pathspecs, repo), root 1045 ) 1046 1047 _LOG.info( 1048 'Checking %s', 1049 git_repo.describe_files( 1050 repo, repo, base, pathspecs, exclude, root 1051 ), 1052 ) 1053 1054 if output_directory is None: 1055 output_directory = root / '.presubmit' 1056 1057 if package_root is None: 1058 package_root = output_directory / 'packages' 1059 1060 presubmit = presubmit_class( 1061 root=root, 1062 repos=repos, 1063 output_directory=output_directory, 1064 paths=modified_files, 1065 all_paths=all_files, 1066 package_root=package_root, 1067 override_gn_args=dict(override_gn_args or {}), 1068 continue_after_build_error=continue_after_build_error, 1069 ) 1070 1071 if only_list_steps: 1072 steps: List[Dict] = [] 1073 for filtered_check in presubmit.apply_filters(program): 1074 step = { 1075 'name': filtered_check.name, 1076 'paths': [str(x) for x in filtered_check.paths], 1077 } 1078 substeps = filtered_check.check.substeps() 1079 if len(substeps) > 1: 1080 step['substeps'] = [x.name for x in substeps] 1081 steps.append(step) 1082 1083 list_steps_data = { 1084 'steps': steps, 1085 'all_files': [str(x) for x in all_files], 1086 } 1087 json.dump(list_steps_data, sys.stdout, indent=2) 1088 sys.stdout.write('\n') 1089 return True 1090 1091 if not isinstance(program, Program): 1092 program = Program('', program) 1093 1094 return presubmit.run(program, keep_going, substep=substep) 1095 1096 1097def _make_str_tuple(value: Union[Iterable[str], str]) -> Tuple[str, ...]: 1098 return tuple([value] if isinstance(value, str) else value) 1099 1100 1101def check(*args, **kwargs): 1102 """Turn a function into a presubmit check. 1103 1104 Args: 1105 *args: Passed through to function. 1106 *kwargs: Passed through to function. 1107 1108 If only one argument is provided and it's a function, this function acts 1109 as a decorator and creates a Check from the function. Example of this kind 1110 of usage: 1111 1112 @check 1113 def pragma_once(ctx: PresubmitContext): 1114 pass 1115 1116 Otherwise, save the arguments, and return a decorator that turns a function 1117 into a Check, but with the arguments added onto the Check constructor. 1118 Example of this kind of usage: 1119 1120 @check(name='pragma_twice') 1121 def pragma_once(ctx: PresubmitContext): 1122 pass 1123 """ 1124 if ( 1125 len(args) == 1 1126 and isinstance(args[0], types.FunctionType) 1127 and not kwargs 1128 ): 1129 # Called as a regular decorator. 1130 return Check(args[0]) 1131 1132 def decorator(check_function): 1133 return Check(check_function, *args, **kwargs) 1134 1135 return decorator 1136 1137 1138@dataclasses.dataclass 1139class SubStep: 1140 name: Optional[str] 1141 _func: Callable[..., PresubmitResult] 1142 args: Sequence[Any] = () 1143 kwargs: Dict[str, Any] = dataclasses.field(default_factory=lambda: {}) 1144 1145 def __call__(self, ctx: PresubmitContext) -> PresubmitResult: 1146 if self.name: 1147 _LOG.info('%s', self.name) 1148 return self._func(ctx, *self.args, **self.kwargs) 1149 1150 1151class Check: 1152 """Wraps a presubmit check function. 1153 1154 This class consolidates the logic for running and logging a presubmit check. 1155 It also supports filtering the paths passed to the presubmit check. 1156 """ 1157 1158 def __init__( 1159 self, 1160 check: Union[ # pylint: disable=redefined-outer-name 1161 Callable, Iterable[SubStep] 1162 ], 1163 path_filter: FileFilter = FileFilter(), 1164 always_run: bool = True, 1165 name: Optional[str] = None, 1166 doc: Optional[str] = None, 1167 ) -> None: 1168 # Since Check wraps a presubmit function, adopt that function's name. 1169 self.name: str = '' 1170 self.doc: str = '' 1171 if isinstance(check, Check): 1172 self.name = check.name 1173 self.doc = check.doc 1174 elif callable(check): 1175 self.name = check.__name__ 1176 self.doc = check.__doc__ or '' 1177 1178 if name: 1179 self.name = name 1180 if doc: 1181 self.doc = doc 1182 1183 if not self.name: 1184 raise ValueError('no name for step') 1185 1186 self._substeps_raw: Iterable[SubStep] 1187 if isinstance(check, collections.abc.Iterator): 1188 self._substeps_raw = check 1189 else: 1190 assert callable(check) 1191 _ensure_is_valid_presubmit_check_function(check) 1192 self._substeps_raw = iter((SubStep(None, check),)) 1193 self._substeps_saved: Sequence[SubStep] = () 1194 1195 self.filter = path_filter 1196 self.always_run: bool = always_run 1197 1198 def substeps(self) -> Sequence[SubStep]: 1199 """Return the SubSteps of the current step. 1200 1201 This is where the list of SubSteps is actually evaluated. It can't be 1202 evaluated in the constructor because the Iterable passed into the 1203 constructor might not be ready yet. 1204 """ 1205 if not self._substeps_saved: 1206 self._substeps_saved = tuple(self._substeps_raw) 1207 return self._substeps_saved 1208 1209 def __repr__(self): 1210 # This returns just the name so it's easy to show the entire list of 1211 # steps with '--help'. 1212 return self.name 1213 1214 def unfiltered(self) -> Check: 1215 """Create a new check identical to this one, but without the filter.""" 1216 clone = copy.copy(self) 1217 clone.filter = FileFilter() 1218 return clone 1219 1220 def with_filter( 1221 self, 1222 *, 1223 endswith: Iterable[str] = (), 1224 exclude: Iterable[Union[Pattern[str], str]] = (), 1225 ) -> Check: 1226 """Create a new check identical to this one, but with extra filters. 1227 1228 Add to the existing filter, perhaps to exclude an additional directory. 1229 1230 Args: 1231 endswith: Passed through to FileFilter. 1232 exclude: Passed through to FileFilter. 1233 1234 Returns a new check. 1235 """ 1236 return self.with_file_filter( 1237 FileFilter(endswith=_make_str_tuple(endswith), exclude=exclude) 1238 ) 1239 1240 def with_file_filter(self, file_filter: FileFilter) -> Check: 1241 """Create a new check identical to this one, but with extra filters. 1242 1243 Add to the existing filter, perhaps to exclude an additional directory. 1244 1245 Args: 1246 file_filter: Additional filter rules. 1247 1248 Returns a new check. 1249 """ 1250 clone = copy.copy(self) 1251 if clone.filter: 1252 clone.filter.exclude = clone.filter.exclude + file_filter.exclude 1253 clone.filter.endswith = clone.filter.endswith + file_filter.endswith 1254 clone.filter.name = file_filter.name or clone.filter.name 1255 clone.filter.suffix = clone.filter.suffix + file_filter.suffix 1256 else: 1257 clone.filter = file_filter 1258 return clone 1259 1260 def run( 1261 self, 1262 ctx: PresubmitContext, 1263 count: int, 1264 total: int, 1265 substep: Optional[str] = None, 1266 ) -> PresubmitResult: 1267 """Runs the presubmit check on the provided paths.""" 1268 1269 _print_ui( 1270 _box( 1271 _CHECK_UPPER, 1272 f'{count}/{total}', 1273 self.name, 1274 plural(ctx.paths, "file"), 1275 ) 1276 ) 1277 1278 substep_part = f'.{substep}' if substep else '' 1279 _LOG.debug( 1280 '[%d/%d] Running %s%s on %s', 1281 count, 1282 total, 1283 self.name, 1284 substep_part, 1285 plural(ctx.paths, "file"), 1286 ) 1287 1288 start_time_s = time.time() 1289 result: PresubmitResult 1290 if substep: 1291 result = self.run_substep(ctx, substep) 1292 else: 1293 result = self(ctx) 1294 time_str = _format_time(time.time() - start_time_s) 1295 _LOG.debug('%s %s', self.name, result.value) 1296 1297 _print_ui( 1298 _box(_CHECK_LOWER, result.colorized(_LEFT), self.name, time_str) 1299 ) 1300 _LOG.debug('%s duration:%s', self.name, time_str) 1301 1302 return result 1303 1304 def _try_call( 1305 self, 1306 func: Callable, 1307 ctx, 1308 *args, 1309 **kwargs, 1310 ) -> PresubmitResult: 1311 try: 1312 result = func(ctx, *args, **kwargs) 1313 if ctx.failed: 1314 return PresubmitResult.FAIL 1315 if isinstance(result, PresubmitResult): 1316 return result 1317 return PresubmitResult.PASS 1318 1319 except PresubmitFailure as failure: 1320 if str(failure): 1321 _LOG.warning('%s', failure) 1322 return PresubmitResult.FAIL 1323 1324 except Exception as _failure: # pylint: disable=broad-except 1325 _LOG.exception('Presubmit check %s failed!', self.name) 1326 return PresubmitResult.FAIL 1327 1328 except KeyboardInterrupt: 1329 _print_ui() 1330 return PresubmitResult.CANCEL 1331 1332 def run_substep( 1333 self, ctx: PresubmitContext, name: Optional[str] 1334 ) -> PresubmitResult: 1335 for substep in self.substeps(): 1336 if substep.name == name: 1337 return substep(ctx) 1338 1339 expected = ', '.join(repr(s.name) for s in self.substeps()) 1340 raise LookupError(f'bad substep name: {name!r} (expected: {expected})') 1341 1342 def __call__(self, ctx: PresubmitContext) -> PresubmitResult: 1343 """Calling a Check calls its underlying substeps directly. 1344 1345 This makes it possible to call functions wrapped by @filter_paths. The 1346 prior filters are ignored, so new filters may be applied. 1347 """ 1348 result: PresubmitResult 1349 for substep in self.substeps(): 1350 result = self._try_call(substep, ctx) 1351 if result and result != PresubmitResult.PASS: 1352 return result 1353 return PresubmitResult.PASS 1354 1355 1356def _required_args(function: Callable) -> Iterable[Parameter]: 1357 """Returns the required arguments for a function.""" 1358 optional_types = Parameter.VAR_POSITIONAL, Parameter.VAR_KEYWORD 1359 1360 for param in signature(function).parameters.values(): 1361 if param.default is param.empty and param.kind not in optional_types: 1362 yield param 1363 1364 1365def _ensure_is_valid_presubmit_check_function(chk: Callable) -> None: 1366 """Checks if a Callable can be used as a presubmit check.""" 1367 try: 1368 required_args = tuple(_required_args(chk)) 1369 except (TypeError, ValueError): 1370 raise TypeError( 1371 'Presubmit checks must be callable, but ' 1372 f'{chk!r} is a {type(chk).__name__}' 1373 ) 1374 1375 if len(required_args) != 1: 1376 raise TypeError( 1377 f'Presubmit check functions must have exactly one required ' 1378 f'positional argument (the PresubmitContext), but ' 1379 f'{chk.__name__} has {len(required_args)} required arguments' 1380 + ( 1381 f' ({", ".join(a.name for a in required_args)})' 1382 if required_args 1383 else '' 1384 ) 1385 ) 1386 1387 1388def filter_paths( 1389 *, 1390 endswith: Iterable[str] = (), 1391 exclude: Iterable[Union[Pattern[str], str]] = (), 1392 file_filter: Optional[FileFilter] = None, 1393 always_run: bool = False, 1394) -> Callable[[Callable], Check]: 1395 """Decorator for filtering the paths list for a presubmit check function. 1396 1397 Path filters only apply when the function is used as a presubmit check. 1398 Filters are ignored when the functions are called directly. This makes it 1399 possible to reuse functions wrapped in @filter_paths in other presubmit 1400 checks, potentially with different path filtering rules. 1401 1402 Args: 1403 endswith: str or iterable of path endings to include 1404 exclude: regular expressions of paths to exclude 1405 file_filter: FileFilter used to select files 1406 always_run: Run check even when no files match 1407 Returns: 1408 a wrapped version of the presubmit function 1409 """ 1410 1411 if file_filter: 1412 real_file_filter = file_filter 1413 if endswith or exclude: 1414 raise ValueError( 1415 'Must specify either file_filter or ' 1416 'endswith/exclude args, not both' 1417 ) 1418 else: 1419 # TODO(b/238426363): Remove these arguments and use FileFilter only. 1420 real_file_filter = FileFilter( 1421 endswith=_make_str_tuple(endswith), exclude=exclude 1422 ) 1423 1424 def filter_paths_for_function(function: Callable): 1425 return Check(function, real_file_filter, always_run=always_run) 1426 1427 return filter_paths_for_function 1428 1429 1430def call(*args, **kwargs) -> None: 1431 """Optional subprocess wrapper that causes a PresubmitFailure on errors.""" 1432 attributes, command = tools.format_command(args, kwargs) 1433 _LOG.debug('[RUN] %s\n%s', attributes, command) 1434 1435 tee = kwargs.pop('tee', None) 1436 propagate_sigterm = kwargs.pop('propagate_sigterm', False) 1437 1438 env = pw_cli.env.pigweed_environment() 1439 kwargs['stdout'] = subprocess.PIPE 1440 kwargs['stderr'] = subprocess.STDOUT 1441 1442 process = subprocess.Popen(args, **kwargs) 1443 assert process.stdout 1444 1445 # Set up signal handler if requested. 1446 signaled = False 1447 if propagate_sigterm: 1448 1449 def signal_handler(_signal_number: int, _stack_frame: Any) -> None: 1450 nonlocal signaled 1451 signaled = True 1452 process.terminate() 1453 1454 previous_signal_handler = signal.signal(signal.SIGTERM, signal_handler) 1455 1456 if env.PW_PRESUBMIT_DISABLE_SUBPROCESS_CAPTURE: 1457 while True: 1458 line = process.stdout.readline().decode(errors='backslashreplace') 1459 if not line: 1460 break 1461 _LOG.info(line.rstrip()) 1462 if tee: 1463 tee.write(line) 1464 1465 stdout, _ = process.communicate() 1466 if tee: 1467 tee.write(stdout.decode(errors='backslashreplace')) 1468 1469 logfunc = _LOG.warning if process.returncode else _LOG.debug 1470 logfunc('[FINISHED]\n%s', command) 1471 logfunc( 1472 '[RESULT] %s with return code %d', 1473 'Failed' if process.returncode else 'Passed', 1474 process.returncode, 1475 ) 1476 if stdout: 1477 logfunc('[OUTPUT]\n%s', stdout.decode(errors='backslashreplace')) 1478 1479 if propagate_sigterm: 1480 signal.signal(signal.SIGTERM, previous_signal_handler) 1481 if signaled: 1482 _LOG.warning('Exiting due to SIGTERM.') 1483 sys.exit(1) 1484 1485 if process.returncode: 1486 raise PresubmitFailure 1487 1488 1489def install_package( 1490 ctx: Union[FormatContext, PresubmitContext], 1491 name: str, 1492 force: bool = False, 1493) -> None: 1494 """Install package with given name in given path.""" 1495 root = ctx.package_root 1496 mgr = package_manager.PackageManager(root) 1497 1498 if not mgr.list(): 1499 raise PresubmitFailure( 1500 'no packages configured, please import your pw_package ' 1501 'configuration module' 1502 ) 1503 1504 if not mgr.status(name) or force: 1505 mgr.install(name, force=force) 1506