1# Copyright 2023 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""pw_presubmit ContextVar.""" 15 16from __future__ import annotations 17 18from contextvars import ContextVar 19import dataclasses 20import enum 21import inspect 22import logging 23import json 24import os 25from pathlib import Path 26import re 27import shlex 28import shutil 29import subprocess 30import tempfile 31from typing import ( 32 Any, 33 Iterable, 34 NamedTuple, 35 Sequence, 36 TYPE_CHECKING, 37) 38import urllib 39 40import pw_cli.color 41import pw_cli.env 42import pw_env_setup.config_file 43 44if TYPE_CHECKING: 45 from pw_presubmit.presubmit import Check 46 47_COLOR = pw_cli.color.colors() 48_LOG: logging.Logger = logging.getLogger(__name__) 49 50PRESUBMIT_CHECK_TRACE: ContextVar[ 51 dict[str, list[PresubmitCheckTrace]] 52] = ContextVar('pw_presubmit_check_trace', default={}) 53 54 55@dataclasses.dataclass(frozen=True) 56class FormatOptions: 57 python_formatter: str | None = 'black' 58 black_path: str | None = 'black' 59 exclude: Sequence[re.Pattern] = dataclasses.field(default_factory=list) 60 61 @staticmethod 62 def load(env: dict[str, str] | None = None) -> FormatOptions: 63 config = pw_env_setup.config_file.load(env=env) 64 fmt = config.get('pw', {}).get('pw_presubmit', {}).get('format', {}) 65 return FormatOptions( 66 python_formatter=fmt.get('python_formatter', 'black'), 67 black_path=fmt.get('black_path', 'black'), 68 exclude=tuple(re.compile(x) for x in fmt.get('exclude', ())), 69 ) 70 71 def filter_paths(self, paths: Iterable[Path]) -> tuple[Path, ...]: 72 root = Path(pw_cli.env.pigweed_environment().PW_PROJECT_ROOT) 73 relpaths = [x.relative_to(root) for x in paths] 74 75 for filt in self.exclude: 76 relpaths = [x for x in relpaths if not filt.search(str(x))] 77 return tuple(root / x for x in relpaths) 78 79 80def get_buildbucket_info(bbid) -> dict[str, Any]: 81 if not bbid or not shutil.which('bb'): 82 return {} 83 84 output = subprocess.check_output( 85 ['bb', 'get', '-json', '-p', f'{bbid}'], text=True 86 ) 87 return json.loads(output) 88 89 90@dataclasses.dataclass 91class LuciPipeline: 92 """Details of previous builds in this pipeline, if applicable. 93 94 Attributes: 95 round: The zero-indexed round number. 96 builds_from_previous_iteration: A list of the buildbucket ids from the 97 previous round, if any. 98 """ 99 100 round: int 101 builds_from_previous_iteration: Sequence[int] 102 103 @staticmethod 104 def create( 105 bbid: int, 106 fake_pipeline_props: dict[str, Any] | None = None, 107 ) -> LuciPipeline | None: 108 pipeline_props: dict[str, Any] 109 if fake_pipeline_props is not None: 110 pipeline_props = fake_pipeline_props 111 else: 112 pipeline_props = ( 113 get_buildbucket_info(bbid) 114 .get('input', {}) 115 .get('properties', {}) 116 .get('$pigweed/pipeline', {}) 117 ) 118 if not pipeline_props.get('inside_a_pipeline', False): 119 return None 120 121 return LuciPipeline( 122 round=int(pipeline_props['round']), 123 builds_from_previous_iteration=list( 124 int(x) for x in pipeline_props['builds_from_previous_iteration'] 125 ), 126 ) 127 128 129@dataclasses.dataclass 130class LuciTrigger: 131 """Details the pending change or submitted commit triggering the build. 132 133 Attributes: 134 number: The number of the change in Gerrit. 135 patchset: The number of the patchset of the change. 136 remote: The full URL of the remote. 137 project: The name of the project in Gerrit. 138 branch: The name of the branch on which this change is being/was 139 submitted. 140 ref: The "refs/changes/.." path that can be used to reference the 141 patch for unsubmitted changes and the hash for submitted changes. 142 gerrit_name: The name of the googlesource.com Gerrit host. 143 submitted: Whether the change has been submitted or is still pending. 144 primary: Whether this change was the change that triggered a build or 145 if it was imported by that triggering change. 146 gerrit_host: The scheme and hostname of the googlesource.com Gerrit 147 host. 148 gerrit_url: The full URL to this change on the Gerrit host. 149 gitiles_url: The full URL to this commit in Gitiles. 150 """ 151 152 number: int 153 patchset: int 154 remote: str 155 project: str 156 branch: str 157 ref: str 158 gerrit_name: str 159 submitted: bool 160 primary: bool 161 162 @property 163 def gerrit_host(self): 164 return f'https://{self.gerrit_name}-review.googlesource.com' 165 166 @property 167 def gerrit_url(self): 168 if not self.number: 169 return self.gitiles_url 170 return f'{self.gerrit_host}/c/{self.number}' 171 172 @property 173 def gitiles_url(self): 174 return f'{self.remote}/+/{self.ref}' 175 176 @staticmethod 177 def create_from_environment( 178 env: dict[str, str] | None = None, 179 ) -> Sequence['LuciTrigger']: 180 """Create a LuciTrigger from the environment.""" 181 if not env: 182 env = os.environ.copy() 183 raw_path = env.get('TRIGGERING_CHANGES_JSON') 184 if not raw_path: 185 return () 186 path = Path(raw_path) 187 if not path.is_file(): 188 return () 189 190 result = [] 191 with open(path, 'r') as ins: 192 for trigger in json.load(ins): 193 keys = { 194 'number', 195 'patchset', 196 'remote', 197 'project', 198 'branch', 199 'ref', 200 'gerrit_name', 201 'submitted', 202 'primary', 203 } 204 if keys <= trigger.keys(): 205 result.append(LuciTrigger(**{x: trigger[x] for x in keys})) 206 207 return tuple(result) 208 209 @staticmethod 210 def create_for_testing(**kwargs): 211 """Create a LuciTrigger for testing.""" 212 change = { 213 'number': 123456, 214 'patchset': 1, 215 'remote': 'https://pigweed.googlesource.com/pigweed/pigweed', 216 'project': 'pigweed/pigweed', 217 'branch': 'main', 218 'ref': 'refs/changes/56/123456/1', 219 'gerrit_name': 'pigweed', 220 'submitted': True, 221 'primary': True, 222 } 223 change.update(kwargs) 224 225 with tempfile.TemporaryDirectory() as tempdir: 226 changes_json = Path(tempdir) / 'changes.json' 227 with changes_json.open('w') as outs: 228 json.dump([change], outs) 229 env = {'TRIGGERING_CHANGES_JSON': changes_json} 230 return LuciTrigger.create_from_environment(env) 231 232 233@dataclasses.dataclass 234class LuciContext: 235 """LUCI-specific information about the environment. 236 237 Attributes: 238 buildbucket_id: The globally-unique buildbucket id of the build. 239 build_number: The builder-specific incrementing build number, if 240 configured for this builder. 241 project: The LUCI project under which this build is running (often 242 "pigweed" or "pigweed-internal"). 243 bucket: The LUCI bucket under which this build is running (often ends 244 with "ci" or "try"). 245 builder: The builder being run. 246 swarming_server: The swarming server on which this build is running. 247 swarming_task_id: The swarming task id of this build. 248 cas_instance: The CAS instance accessible from this build. 249 context_file: The path to the LUCI_CONTEXT file. 250 pipeline: Information about the build pipeline, if applicable. 251 triggers: Information about triggering commits, if applicable. 252 is_try: True if the bucket is a try bucket. 253 is_ci: True if the bucket is a ci bucket. 254 is_dev: True if the bucket is a dev bucket. 255 is_shadow: True if the bucket is a shadow bucket. 256 is_prod: True if both is_dev and is_shadow are False. 257 """ 258 259 buildbucket_id: int 260 build_number: int 261 project: str 262 bucket: str 263 builder: str 264 swarming_server: str 265 swarming_task_id: str 266 cas_instance: str 267 context_file: Path 268 pipeline: LuciPipeline | None 269 triggers: Sequence[LuciTrigger] = dataclasses.field(default_factory=tuple) 270 271 @property 272 def is_try(self): 273 return re.search(r'\btry$', self.bucket) 274 275 @property 276 def is_ci(self): 277 return re.search(r'\bci$', self.bucket) 278 279 @property 280 def is_dev(self): 281 return re.search(r'\bdev\b', self.bucket) 282 283 @property 284 def is_shadow(self): 285 return re.search(r'\bshadow\b', self.bucket) 286 287 @property 288 def is_prod(self): 289 return not self.is_dev and not self.is_shadow 290 291 @staticmethod 292 def create_from_environment( 293 env: dict[str, str] | None = None, 294 fake_pipeline_props: dict[str, Any] | None = None, 295 ) -> LuciContext | None: 296 """Create a LuciContext from the environment.""" 297 298 if not env: 299 env = os.environ.copy() 300 301 luci_vars = [ 302 'BUILDBUCKET_ID', 303 'BUILDBUCKET_NAME', 304 'BUILD_NUMBER', 305 'LUCI_CONTEXT', 306 'SWARMING_TASK_ID', 307 'SWARMING_SERVER', 308 ] 309 if any(x for x in luci_vars if x not in env): 310 return None 311 312 project, bucket, builder = env['BUILDBUCKET_NAME'].split(':') 313 314 bbid: int = 0 315 pipeline: LuciPipeline | None = None 316 try: 317 bbid = int(env['BUILDBUCKET_ID']) 318 pipeline = LuciPipeline.create(bbid, fake_pipeline_props) 319 320 except ValueError: 321 pass 322 323 # Logic to identify cas instance from swarming server is derived from 324 # https://chromium.googlesource.com/infra/luci/recipes-py/+/main/recipe_modules/cas/api.py 325 swarm_server = env['SWARMING_SERVER'] 326 cas_project = urllib.parse.urlparse(swarm_server).netloc.split('.')[0] 327 cas_instance = f'projects/{cas_project}/instances/default_instance' 328 329 result = LuciContext( 330 buildbucket_id=bbid, 331 build_number=int(env['BUILD_NUMBER']), 332 project=project, 333 bucket=bucket, 334 builder=builder, 335 swarming_server=env['SWARMING_SERVER'], 336 swarming_task_id=env['SWARMING_TASK_ID'], 337 cas_instance=cas_instance, 338 pipeline=pipeline, 339 triggers=LuciTrigger.create_from_environment(env), 340 context_file=Path(env['LUCI_CONTEXT']), 341 ) 342 _LOG.debug('%r', result) 343 return result 344 345 @staticmethod 346 def create_for_testing(**kwargs): 347 env = { 348 'BUILDBUCKET_ID': '881234567890', 349 'BUILDBUCKET_NAME': 'pigweed:bucket.try:builder-name', 350 'BUILD_NUMBER': '123', 351 'LUCI_CONTEXT': '/path/to/context/file.json', 352 'SWARMING_SERVER': 'https://chromium-swarm.appspot.com', 353 'SWARMING_TASK_ID': 'cd2dac62d2', 354 } 355 env.update(kwargs) 356 357 return LuciContext.create_from_environment(env, {}) 358 359 360@dataclasses.dataclass 361class FormatContext: 362 """Context passed into formatting helpers. 363 364 This class is a subset of PresubmitContext containing only what's needed by 365 formatters. 366 367 For full documentation on the members see the PresubmitContext section of 368 pw_presubmit/docs.rst. 369 370 Attributes: 371 root: Source checkout root directory 372 output_dir: Output directory for this specific language. 373 paths: Modified files for the presubmit step to check (often used in 374 formatting steps but ignored in compile steps). 375 package_root: Root directory for pw package installations. 376 format_options: Formatting options, derived from pigweed.json. 377 dry_run: Whether to just report issues or also fix them. 378 """ 379 380 root: Path | None 381 output_dir: Path 382 paths: tuple[Path, ...] 383 package_root: Path 384 format_options: FormatOptions 385 dry_run: bool = False 386 387 def append_check_command(self, *command_args, **command_kwargs) -> None: 388 """Empty append_check_command.""" 389 390 391class PresubmitFailure(Exception): 392 """Optional exception to use for presubmit failures.""" 393 394 def __init__( 395 self, 396 description: str = '', 397 path: Path | None = None, 398 line: int | None = None, 399 ): 400 line_part: str = '' 401 if line is not None: 402 line_part = f'{line}:' 403 super().__init__( 404 f'{path}:{line_part} {description}' if path else description 405 ) 406 407 408@dataclasses.dataclass 409class PresubmitContext: # pylint: disable=too-many-instance-attributes 410 """Context passed into presubmit checks. 411 412 For full documentation on the members see pw_presubmit/docs.rst. 413 414 Attributes: 415 root: Source checkout root directory. 416 repos: Repositories (top-level and submodules) processed by 417 `pw presubmit`. 418 output_dir: Output directory for this specific presubmit step. 419 failure_summary_log: Path where steps should write a brief summary of 420 any failures encountered for use by other tooling. 421 paths: Modified files for the presubmit step to check (often used in 422 formatting steps but ignored in compile steps). 423 all_paths: All files in the tree. 424 package_root: Root directory for pw package installations. 425 override_gn_args: Additional GN args processed by `build.gn_gen()`. 426 luci: Information about the LUCI build or None if not running in LUCI. 427 format_options: Formatting options, derived from pigweed.json. 428 num_jobs: Number of jobs to run in parallel. 429 continue_after_build_error: For steps that compile, don't exit on the 430 first compilation error. 431 rng_seed: Seed for a random number generator, for the few steps that 432 need one. 433 full: Whether this is a full or incremental presubmit run. 434 _failed: Whether the presubmit step in question has failed. Set to True 435 by calling ctx.fail(). 436 dry_run: Whether to actually execute commands or just log them. 437 use_remote_cache: Whether to tell the build system to use RBE. 438 pw_root: The path to the Pigweed repository. 439 """ 440 441 root: Path 442 repos: tuple[Path, ...] 443 output_dir: Path 444 failure_summary_log: Path 445 paths: tuple[Path, ...] 446 all_paths: tuple[Path, ...] 447 package_root: Path 448 luci: LuciContext | None 449 override_gn_args: dict[str, str] 450 format_options: FormatOptions 451 num_jobs: int | None = None 452 continue_after_build_error: bool = False 453 rng_seed: int = 1 454 full: bool = False 455 _failed: bool = False 456 dry_run: bool = False 457 use_remote_cache: bool = False 458 pw_root: Path = pw_cli.env.pigweed_environment().PW_ROOT 459 460 @property 461 def failed(self) -> bool: 462 return self._failed 463 464 @property 465 def incremental(self) -> bool: 466 return not self.full 467 468 def fail( 469 self, 470 description: str, 471 path: Path | None = None, 472 line: int | None = None, 473 ): 474 """Add a failure to this presubmit step. 475 476 If this is called at least once the step fails, but not immediately—the 477 check is free to continue and possibly call this method again. 478 """ 479 _LOG.warning('%s', PresubmitFailure(description, path, line)) 480 self._failed = True 481 482 @staticmethod 483 def create_for_testing(**kwargs): 484 parsed_env = pw_cli.env.pigweed_environment() 485 root = parsed_env.PW_PROJECT_ROOT 486 presubmit_root = root / 'out' / 'presubmit' 487 presubmit_kwargs = { 488 'root': root, 489 'repos': (root,), 490 'output_dir': presubmit_root / 'test', 491 'failure_summary_log': presubmit_root / 'failure-summary.log', 492 'paths': (root / 'foo.cc', root / 'foo.py'), 493 'all_paths': (root / 'BUILD.gn', root / 'foo.cc', root / 'foo.py'), 494 'package_root': root / 'environment' / 'packages', 495 'luci': None, 496 'override_gn_args': {}, 497 'format_options': FormatOptions(), 498 } 499 presubmit_kwargs.update(kwargs) 500 return PresubmitContext(**presubmit_kwargs) 501 502 def append_check_command( 503 self, 504 *command_args, 505 call_annotation: dict[Any, Any] | None = None, 506 **command_kwargs, 507 ) -> None: 508 """Save a subprocess command annotation to this presubmit context. 509 510 This is used to capture commands that will be run for display in ``pw 511 presubmit --dry-run.`` 512 513 Args: 514 515 command_args: All args that would normally be passed to 516 subprocess.run 517 518 call_annotation: Optional key value pairs of data to save for this 519 command. Examples: 520 521 :: 522 523 call_annotation={'pw_package_install': 'teensy'} 524 call_annotation={'build_system': 'bazel'} 525 call_annotation={'build_system': 'ninja'} 526 527 command_kwargs: keyword args that would normally be passed to 528 subprocess.run. 529 """ 530 call_annotation = call_annotation if call_annotation else {} 531 calling_func: str | None = None 532 calling_check = None 533 534 # Loop through the current call stack looking for `self`, and stopping 535 # when self is a Check() instance and if the __call__ or _try_call 536 # functions are in the stack. 537 538 # This used to be an isinstance(obj, Check) call, but it was changed to 539 # this so Check wouldn't need to be imported here. Doing so would create 540 # a dependency loop. 541 def is_check_object(obj): 542 return getattr(obj, '_is_presubmit_check_object', False) 543 544 for frame_info in inspect.getouterframes(inspect.currentframe()): 545 self_obj = frame_info.frame.f_locals.get('self', None) 546 if ( 547 self_obj 548 and is_check_object(self_obj) 549 and frame_info.function in ['_try_call', '__call__'] 550 ): 551 calling_func = frame_info.function 552 calling_check = self_obj 553 554 save_check_trace( 555 self.output_dir, 556 PresubmitCheckTrace( 557 self, 558 calling_check, 559 calling_func, 560 command_args, 561 command_kwargs, 562 call_annotation, 563 ), 564 ) 565 566 def __post_init__(self) -> None: 567 PRESUBMIT_CONTEXT.set(self) 568 569 def __hash__(self): 570 return hash( 571 tuple( 572 tuple(attribute.items()) 573 if isinstance(attribute, dict) 574 else attribute 575 for attribute in dataclasses.astuple(self) 576 ) 577 ) 578 579 580PRESUBMIT_CONTEXT: ContextVar[PresubmitContext | None] = ContextVar( 581 'pw_presubmit_context', default=None 582) 583 584 585def get_presubmit_context(): 586 return PRESUBMIT_CONTEXT.get() 587 588 589class PresubmitCheckTraceType(enum.Enum): 590 BAZEL = 'BAZEL' 591 CMAKE = 'CMAKE' 592 GN_NINJA = 'GN_NINJA' 593 PW_PACKAGE = 'PW_PACKAGE' 594 595 596class PresubmitCheckTrace(NamedTuple): 597 ctx: PresubmitContext 598 check: Check | None 599 func: str | None 600 args: Iterable[Any] 601 kwargs: dict[Any, Any] 602 call_annotation: dict[Any, Any] 603 604 def __repr__(self) -> str: 605 return f'''CheckTrace( 606 ctx={self.ctx.output_dir} 607 id(ctx)={id(self.ctx)} 608 check={self.check} 609 args={self.args} 610 kwargs={self.kwargs.keys()} 611 call_annotation={self.call_annotation} 612)''' 613 614 615def save_check_trace(output_dir: Path, trace: PresubmitCheckTrace) -> None: 616 trace_key = str(output_dir.resolve()) 617 trace_list = PRESUBMIT_CHECK_TRACE.get().get(trace_key, []) 618 trace_list.append(trace) 619 PRESUBMIT_CHECK_TRACE.get()[trace_key] = trace_list 620 621 622def get_check_traces(ctx: PresubmitContext) -> list[PresubmitCheckTrace]: 623 trace_key = str(ctx.output_dir.resolve()) 624 return PRESUBMIT_CHECK_TRACE.get().get(trace_key, []) 625 626 627def log_check_traces(ctx: PresubmitContext) -> None: 628 traces = PRESUBMIT_CHECK_TRACE.get() 629 630 for _output_dir, check_traces in traces.items(): 631 for check_trace in check_traces: 632 if check_trace.ctx != ctx: 633 continue 634 635 quoted_command_args = ' '.join( 636 shlex.quote(str(arg)) for arg in check_trace.args 637 ) 638 _LOG.info( 639 '%s %s', 640 _COLOR.blue('Run ==>'), 641 quoted_command_args, 642 ) 643 644 645def apply_exclusions( 646 ctx: PresubmitContext, 647 paths: Sequence[Path] | None = None, 648) -> tuple[Path, ...]: 649 return ctx.format_options.filter_paths(paths or ctx.paths) 650