1#!/usr/bin/env python 2# Copyright 2022 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15"""Watch build config dataclasses.""" 16 17from dataclasses import dataclass, field 18import logging 19from pathlib import Path 20import shlex 21from typing import Callable, Dict, List, Optional, TYPE_CHECKING 22 23from prompt_toolkit.formatted_text import ANSI, StyleAndTextTuples 24from prompt_toolkit.formatted_text.base import OneStyleAndTextTuple 25 26if TYPE_CHECKING: 27 from pw_build.project_builder import ProjectBuilder 28 from pw_build.project_builder_prefs import ProjectBuilderPrefs 29 30_LOG = logging.getLogger('pw_build.watch') 31 32 33class UnknownBuildSystem(Exception): 34 """Exception for requesting unsupported build systems.""" 35 36 37class UnknownBuildDir(Exception): 38 """Exception for an unknown build dir before command running.""" 39 40 41@dataclass 42class BuildCommand: 43 """Store details of a single build step. 44 45 Example usage: 46 47 .. code-block:: python 48 49 from pw_build.build_recipe import BuildCommand, BuildRecipe 50 51 def should_gen_gn(out: Path): 52 return not (out / 'build.ninja').is_file() 53 54 cmd1 = BuildCommand(build_dir='out', 55 command=['gn', 'gen', '{build_dir}'], 56 run_if=should_gen_gn) 57 58 cmd2 = BuildCommand(build_dir='out', 59 build_system_command='ninja', 60 build_system_extra_args=['-k', '0'], 61 targets=['default']), 62 63 Args: 64 build_dir: Output directory for this build command. This can be omitted 65 if the BuildCommand is included in the steps of a BuildRecipe. 66 build_system_command: This command should end with ``ninja``, ``make``, 67 or ``bazel``. 68 build_system_extra_args: A list of extra arguments passed to the 69 build_system_command. If running ``bazel test`` include ``test`` as 70 an extra arg here. 71 targets: Optional list of targets to build in the build_dir. 72 command: List of strings to run as a command. These are passed to 73 subprocess.run(). Any instances of the ``'{build_dir}'`` string 74 literal will be replaced at run time with the out directory. 75 run_if: A callable function to run before executing this 76 BuildCommand. The callable takes one Path arg for the build_dir. If 77 the callable returns true this command is executed. All 78 BuildCommands are run by default. 79 """ 80 81 build_dir: Optional[Path] = None 82 build_system_command: Optional[str] = None 83 build_system_extra_args: List[str] = field(default_factory=list) 84 targets: List[str] = field(default_factory=list) 85 command: List[str] = field(default_factory=list) 86 run_if: Callable[[Path], bool] = lambda _build_dir: True 87 88 def __post_init__(self) -> None: 89 # Copy self._expanded_args from the command list. 90 self._expanded_args: List[str] = [] 91 if self.command: 92 self._expanded_args = self.command 93 94 def should_run(self) -> bool: 95 """Return True if this build command should be run.""" 96 if self.build_dir: 97 return self.run_if(self.build_dir) 98 return True 99 100 def _get_starting_build_system_args(self) -> List[str]: 101 """Return flags that appear immediately after the build command.""" 102 assert self.build_system_command 103 assert self.build_dir 104 if self.build_system_command.endswith('bazel'): 105 return ['--output_base', str(self.build_dir)] 106 return [] 107 108 def _get_build_system_args(self) -> List[str]: 109 assert self.build_system_command 110 assert self.build_dir 111 112 # Both make and ninja use -C for a build directory. 113 if self.build_system_command.endswith( 114 'make' 115 ) or self.build_system_command.endswith('ninja'): 116 return ['-C', str(self.build_dir), *self.targets] 117 118 # Bazel relies on --output_base which is handled by the 119 # _get_starting_build_system_args() function. 120 if self.build_system_command.endswith('bazel'): 121 return [*self.targets] 122 123 raise UnknownBuildSystem( 124 f'\n\nUnknown build system command "{self.build_system_command}" ' 125 f'for build directory "{self.build_dir}".\n' 126 'Supported commands: ninja, bazel, make' 127 ) 128 129 def _resolve_expanded_args(self) -> List[str]: 130 """Replace instances of '{build_dir}' with the self.build_dir.""" 131 resolved_args = [] 132 for arg in self._expanded_args: 133 if arg == "{build_dir}": 134 if not self.build_dir: 135 raise UnknownBuildDir( 136 '\n\nUnknown "{build_dir}" value for command:\n' 137 f' {self._expanded_args}\n' 138 f'In BuildCommand: {repr(self)}\n\n' 139 'Check build_dir is set for the above BuildCommand' 140 'or included as a step to a BuildRecipe.' 141 ) 142 resolved_args.append(str(self.build_dir.resolve())) 143 else: 144 resolved_args.append(arg) 145 return resolved_args 146 147 def ninja_command(self) -> bool: 148 if self.build_system_command and self.build_system_command.endswith( 149 'ninja' 150 ): 151 return True 152 return False 153 154 def bazel_command(self) -> bool: 155 if self.build_system_command and self.build_system_command.endswith( 156 'bazel' 157 ): 158 return True 159 return False 160 161 def bazel_build_command(self) -> bool: 162 if self.bazel_command() and 'build' in self.build_system_extra_args: 163 return True 164 return False 165 166 def bazel_clean_command(self) -> bool: 167 if self.bazel_command() and 'clean' in self.build_system_extra_args: 168 return True 169 return False 170 171 def get_args( 172 self, 173 additional_ninja_args: Optional[List[str]] = None, 174 additional_bazel_args: Optional[List[str]] = None, 175 additional_bazel_build_args: Optional[List[str]] = None, 176 ) -> List[str]: 177 """Return all args required to launch this BuildCommand.""" 178 # If this is a plain command step, return self._expanded_args as-is. 179 if not self.build_system_command: 180 return self._resolve_expanded_args() 181 182 # Assmemble user-defined extra args. 183 extra_args = [] 184 extra_args.extend(self.build_system_extra_args) 185 if additional_ninja_args and self.ninja_command(): 186 extra_args.extend(additional_ninja_args) 187 188 if additional_bazel_build_args and self.bazel_build_command(): 189 extra_args.extend(additional_bazel_build_args) 190 191 if additional_bazel_args and self.bazel_command(): 192 extra_args.extend(additional_bazel_args) 193 194 build_system_target_args = [] 195 if not self.bazel_clean_command(): 196 build_system_target_args = self._get_build_system_args() 197 198 # Construct the build system command args. 199 command = [ 200 self.build_system_command, 201 *self._get_starting_build_system_args(), 202 *extra_args, 203 *build_system_target_args, 204 ] 205 return command 206 207 def __str__(self) -> str: 208 return ' '.join(shlex.quote(arg) for arg in self.get_args()) 209 210 211@dataclass 212class BuildRecipeStatus: 213 """Stores the status of a build recipe.""" 214 215 recipe: 'BuildRecipe' 216 current_step: str = '' 217 percent: float = 0.0 218 error_count: int = 0 219 return_code: Optional[int] = None 220 flag_done: bool = False 221 flag_started: bool = False 222 error_lines: Dict[int, List[str]] = field(default_factory=dict) 223 224 def pending(self) -> bool: 225 return self.return_code is None 226 227 def failed(self) -> bool: 228 if self.return_code is not None: 229 return self.return_code != 0 230 return False 231 232 def append_failure_line(self, line: str) -> None: 233 lines = self.error_lines.get(self.error_count, []) 234 lines.append(line) 235 self.error_lines[self.error_count] = lines 236 237 def increment_error_count(self, count: int = 1) -> None: 238 self.error_count += count 239 if self.error_count not in self.error_lines: 240 self.error_lines[self.error_count] = [] 241 242 def should_log_failures(self) -> bool: 243 return ( 244 self.recipe.project_builder is not None 245 and self.recipe.project_builder.separate_build_file_logging 246 and (not self.recipe.project_builder.send_recipe_logs_to_root) 247 ) 248 249 def log_last_failure(self) -> None: 250 """Log the last ninja error if available.""" 251 if not self.should_log_failures(): 252 return 253 254 logger = self.recipe.error_logger 255 if not logger: 256 return 257 258 _color = self.recipe.project_builder.color # type: ignore 259 260 lines = self.error_lines.get(self.error_count, []) 261 _LOG.error('') 262 _LOG.error(' ╔════════════════════════════════════') 263 _LOG.error( 264 ' ║ START %s Failure #%d:', 265 _color.cyan(self.recipe.display_name), 266 self.error_count, 267 ) 268 269 logger.error('') 270 for line in lines: 271 logger.error(line) 272 logger.error('') 273 274 _LOG.error( 275 ' ║ END %s Failure #%d', 276 _color.cyan(self.recipe.display_name), 277 self.error_count, 278 ) 279 _LOG.error(" ╚════════════════════════════════════") 280 _LOG.error('') 281 282 def log_entire_recipe_logfile(self) -> None: 283 """Log the entire build logfile if no ninja errors available.""" 284 if not self.should_log_failures(): 285 return 286 287 recipe_logfile = self.recipe.logfile 288 if not recipe_logfile: 289 return 290 291 _color = self.recipe.project_builder.color # type: ignore 292 293 logfile_path = str(recipe_logfile.resolve()) 294 295 _LOG.error('') 296 _LOG.error(' ╔════════════════════════════════════') 297 _LOG.error( 298 ' ║ %s Failure; Entire log below:', 299 _color.cyan(self.recipe.display_name), 300 ) 301 _LOG.error(' ║ %s %s', _color.yellow('START'), logfile_path) 302 303 logger = self.recipe.error_logger 304 if not logger: 305 return 306 307 logger.error('') 308 for line in recipe_logfile.read_text( 309 encoding='utf-8', errors='ignore' 310 ).splitlines(): 311 logger.error(line) 312 logger.error('') 313 314 _LOG.error(' ║ %s %s', _color.yellow('END'), logfile_path) 315 _LOG.error(" ╚════════════════════════════════════") 316 _LOG.error('') 317 318 def status_slug(self, restarting: bool = False) -> OneStyleAndTextTuple: 319 status = ('', '') 320 if not self.recipe.enabled: 321 return ('fg:ansidarkgray', 'Disabled') 322 323 waiting = False 324 if self.done: 325 if self.passed(): 326 status = ('fg:ansigreen', 'OK ') 327 elif self.failed(): 328 status = ('fg:ansired', 'FAIL ') 329 elif self.started: 330 status = ('fg:ansiyellow', 'Building') 331 else: 332 waiting = True 333 status = ('default', 'Waiting ') 334 335 # Only show Aborting if the process is building (or has failures). 336 if restarting and not waiting and not self.passed(): 337 status = ('fg:ansiyellow', 'Aborting') 338 return status 339 340 def current_step_formatted(self) -> StyleAndTextTuples: 341 formatted_text: StyleAndTextTuples = [] 342 if self.passed(): 343 return formatted_text 344 345 if self.current_step: 346 if '\x1b' in self.current_step: 347 formatted_text = ANSI(self.current_step).__pt_formatted_text__() 348 else: 349 formatted_text = [('', self.current_step)] 350 351 return formatted_text 352 353 @property 354 def done(self) -> bool: 355 return self.flag_done 356 357 @property 358 def started(self) -> bool: 359 return self.flag_started 360 361 def mark_done(self) -> None: 362 self.flag_done = True 363 364 def mark_started(self) -> None: 365 self.flag_started = True 366 367 def set_failed(self) -> None: 368 self.flag_done = True 369 self.return_code = -1 370 371 def set_passed(self) -> None: 372 self.flag_done = True 373 self.return_code = 0 374 375 def passed(self) -> bool: 376 if self.done and self.return_code is not None: 377 return self.return_code == 0 378 return False 379 380 381@dataclass 382class BuildRecipe: 383 """Dataclass to store a list of BuildCommands. 384 385 Example usage: 386 387 .. code-block:: python 388 389 from pw_build.build_recipe import BuildCommand, BuildRecipe 390 391 def should_gen_gn(out: Path) -> bool: 392 return not (out / 'build.ninja').is_file() 393 394 recipe = BuildRecipe( 395 build_dir='out', 396 title='Vanilla Ninja Build', 397 steps=[ 398 BuildCommand(command=['gn', 'gen', '{build_dir}'], 399 run_if=should_gen_gn), 400 BuildCommand(build_system_command='ninja', 401 build_system_extra_args=['-k', '0'], 402 targets=['default']), 403 ], 404 ) 405 406 Args: 407 build_dir: Output directory for this BuildRecipe. On init this out dir 408 is set for all included steps. 409 steps: List of BuildCommands to run. 410 title: Custom title. The build_dir is used if this is ommited. 411 """ 412 413 build_dir: Path 414 steps: List[BuildCommand] = field(default_factory=list) 415 title: Optional[str] = None 416 enabled: bool = True 417 418 def __hash__(self): 419 return hash((self.build_dir, self.title, len(self.steps))) 420 421 def __post_init__(self) -> None: 422 # Update all included steps to use this recipe's build_dir. 423 for step in self.steps: 424 if self.build_dir: 425 step.build_dir = self.build_dir 426 427 # Set logging variables 428 self._logger: Optional[logging.Logger] = None 429 self.error_logger: Optional[logging.Logger] = None 430 self._logfile: Optional[Path] = None 431 self._status: BuildRecipeStatus = BuildRecipeStatus(self) 432 self.project_builder: Optional['ProjectBuilder'] = None 433 434 def toggle_enabled(self) -> None: 435 self.enabled = not self.enabled 436 437 def set_project_builder(self, project_builder) -> None: 438 self.project_builder = project_builder 439 440 def set_targets(self, new_targets: List[str]) -> None: 441 """Reset all build step targets.""" 442 for step in self.steps: 443 step.targets = new_targets 444 445 def set_logger(self, logger: logging.Logger) -> None: 446 self._logger = logger 447 448 def set_error_logger(self, logger: logging.Logger) -> None: 449 self.error_logger = logger 450 451 def set_logfile(self, log_file: Path) -> None: 452 self._logfile = log_file 453 454 def reset_status(self) -> None: 455 self._status = BuildRecipeStatus(self) 456 457 @property 458 def status(self) -> BuildRecipeStatus: 459 return self._status 460 461 @property 462 def log(self) -> logging.Logger: 463 if self._logger: 464 return self._logger 465 return logging.getLogger() 466 467 @property 468 def logfile(self) -> Optional[Path]: 469 return self._logfile 470 471 @property 472 def display_name(self) -> str: 473 if self.title: 474 return self.title 475 return str(self.build_dir) 476 477 def targets(self) -> List[str]: 478 return list( 479 set(target for step in self.steps for target in step.targets) 480 ) 481 482 def __str__(self) -> str: 483 message = self.display_name 484 targets = self.targets() 485 if targets: 486 target_list = ' '.join(self.targets()) 487 message = f'{message} -- {target_list}' 488 return message 489 490 491def create_build_recipes(prefs: 'ProjectBuilderPrefs') -> List[BuildRecipe]: 492 """Create a list of BuildRecipes from ProjectBuilderPrefs.""" 493 build_recipes: List[BuildRecipe] = [] 494 495 if prefs.run_commands: 496 for command_str in prefs.run_commands: 497 build_recipes.append( 498 BuildRecipe( 499 build_dir=Path.cwd(), 500 steps=[BuildCommand(command=shlex.split(command_str))], 501 title=command_str, 502 ) 503 ) 504 505 for build_dir, targets in prefs.build_directories.items(): 506 steps: List[BuildCommand] = [] 507 build_path = Path(build_dir) 508 if not targets: 509 targets = [] 510 511 for ( 512 build_system_command, 513 build_system_extra_args, 514 ) in prefs.build_system_commands(build_dir): 515 steps.append( 516 BuildCommand( 517 build_system_command=build_system_command, 518 build_system_extra_args=build_system_extra_args, 519 targets=targets, 520 ) 521 ) 522 523 build_recipes.append( 524 BuildRecipe( 525 build_dir=build_path, 526 steps=steps, 527 ) 528 ) 529 530 return build_recipes 531