1# Copyright 2022 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Build a Pigweed Project. 15 16Run arbitrary commands or invoke build systems (Ninja, Bazel and make) on one or 17more build directories. 18 19Examples: 20 21 # Build the default target in out/ using ninja. 22 python -m pw_build.project_builder -C out 23 24 # Build pw_run_tests.modules in the out/cmake directory 25 python -m pw_build.project_builder -C out/cmake pw_run_tests.modules 26 27 # Build the default target in out/ and pw_apps in out/cmake 28 python -m pw_build.project_builder -C out -C out/cmake pw_apps 29 30 # Build python.tests in out/ and pw_apps in out/cmake/ 31 python -m pw_build.project_builder python.tests -C out/cmake pw_apps 32 33 # Run 'bazel build' and 'bazel test' on the target '//...' in outbazel/ 34 python -m pw_build.project_builder --run-command 'mkdir -p outbazel' 35 -C outbazel '//...' 36 --build-system-command outbazel 'bazel build' 37 --build-system-command outbazel 'bazel test' 38""" 39 40import argparse 41import concurrent.futures 42import os 43import logging 44from pathlib import Path 45import re 46import shlex 47import sys 48import subprocess 49import time 50from typing import ( 51 Callable, 52 Dict, 53 Generator, 54 List, 55 NoReturn, 56 Optional, 57 Sequence, 58 NamedTuple, 59) 60 61from prompt_toolkit.patch_stdout import StdoutProxy 62 63import pw_cli.env 64import pw_cli.log 65 66from pw_build.build_recipe import BuildRecipe, create_build_recipes 67from pw_build.project_builder_argparse import add_project_builder_arguments 68from pw_build.project_builder_context import get_project_builder_context 69from pw_build.project_builder_prefs import ProjectBuilderPrefs 70 71_COLOR = pw_cli.color.colors() 72_LOG = logging.getLogger('pw_build') 73 74BUILDER_CONTEXT = get_project_builder_context() 75 76PASS_MESSAGE = """ 77 ██████╗ █████╗ ███████╗███████╗██╗ 78 ██╔══██╗██╔══██╗██╔════╝██╔════╝██║ 79 ██████╔╝███████║███████╗███████╗██║ 80 ██╔═══╝ ██╔══██║╚════██║╚════██║╚═╝ 81 ██║ ██║ ██║███████║███████║██╗ 82 ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝ 83""" 84 85# Pick a visually-distinct font from "PASS" to ensure that readers can't 86# possibly mistake the difference between the two states. 87FAIL_MESSAGE = """ 88 ▄██████▒░▄▄▄ ██▓ ░██▓ 89 ▓█▓ ░▒████▄ ▓██▒ ░▓██▒ 90 ▒████▒ ░▒█▀ ▀█▄ ▒██▒ ▒██░ 91 ░▓█▒ ░░██▄▄▄▄██ ░██░ ▒██░ 92 ░▒█░ ▓█ ▓██▒░██░░ ████████▒ 93 ▒█░ ▒▒ ▓▒█░░▓ ░ ▒░▓ ░ 94 ░▒ ▒ ▒▒ ░ ▒ ░░ ░ ▒ ░ 95 ░ ░ ░ ▒ ▒ ░ ░ ░ 96 ░ ░ ░ ░ ░ 97""" 98 99 100class ProjectBuilderCharset(NamedTuple): 101 slug_ok: str 102 slug_fail: str 103 slug_building: str 104 105 106ASCII_CHARSET = ProjectBuilderCharset( 107 _COLOR.green('OK '), 108 _COLOR.red('FAIL'), 109 _COLOR.yellow('... '), 110) 111EMOJI_CHARSET = ProjectBuilderCharset('✔️ ', '❌', '⏱️ ') 112 113 114def _exit(*args) -> NoReturn: 115 _LOG.critical(*args) 116 sys.exit(1) 117 118 119def _exit_due_to_interrupt() -> None: 120 """Abort function called when not using progress bars.""" 121 # To keep the log lines aligned with each other in the presence of 122 # a '^C' from the keyboard interrupt, add a newline before the log. 123 print() 124 _LOG.info('Got Ctrl-C; exiting...') 125 BUILDER_CONTEXT.ctrl_c_interrupt() 126 127 128_NINJA_BUILD_STEP = re.compile( 129 r'^\[(?P<step>[0-9]+)/(?P<total_steps>[0-9]+)\] (?P<action>.*)$' 130) 131 132_NINJA_FAILURE_TEXT = '\033[31mFAILED: ' 133 134 135def execute_command_no_logging( 136 command: List, 137 env: Dict, 138 recipe: BuildRecipe, 139 # pylint: disable=unused-argument 140 logger: logging.Logger = _LOG, 141 line_processed_callback: Optional[Callable[[BuildRecipe], None]] = None, 142 # pylint: enable=unused-argument 143) -> bool: 144 print() 145 proc = subprocess.Popen(command, env=env, errors='replace') 146 BUILDER_CONTEXT.register_process(recipe, proc) 147 returncode = None 148 while returncode is None: 149 if BUILDER_CONTEXT.build_stopping(): 150 proc.terminate() 151 returncode = proc.poll() 152 time.sleep(0.05) 153 print() 154 recipe.status.return_code = returncode 155 156 return proc.returncode == 0 157 158 159def execute_command_with_logging( 160 command: List, 161 env: Dict, 162 recipe: BuildRecipe, 163 logger: logging.Logger = _LOG, 164 line_processed_callback: Optional[Callable[[BuildRecipe], None]] = None, 165) -> bool: 166 """Run a command in a subprocess and log all output.""" 167 current_stdout = '' 168 returncode = None 169 170 with subprocess.Popen( 171 command, 172 env=env, 173 stdout=subprocess.PIPE, 174 stderr=subprocess.STDOUT, 175 errors='replace', 176 ) as proc: 177 BUILDER_CONTEXT.register_process(recipe, proc) 178 # Empty line at the start. 179 logger.info('') 180 181 failure_line = False 182 while returncode is None: 183 output = '' 184 error_output = '' 185 186 if proc.stdout: 187 output = proc.stdout.readline() 188 current_stdout += output 189 if proc.stderr: 190 error_output += proc.stderr.readline() 191 current_stdout += error_output 192 193 if not output and not error_output: 194 returncode = proc.poll() 195 continue 196 197 line_match_result = _NINJA_BUILD_STEP.match(output) 198 if line_match_result: 199 if failure_line and not BUILDER_CONTEXT.build_stopping(): 200 recipe.status.log_last_failure() 201 failure_line = False 202 matches = line_match_result.groupdict() 203 recipe.status.current_step = line_match_result.group(0) 204 step = int(matches.get('step', 0)) 205 total_steps = int(matches.get('total_steps', 1)) 206 recipe.status.percent = float(step / total_steps) 207 208 logger_method = logger.info 209 if output.startswith(_NINJA_FAILURE_TEXT): 210 logger_method = logger.error 211 if failure_line and not BUILDER_CONTEXT.build_stopping(): 212 recipe.status.log_last_failure() 213 recipe.status.increment_error_count() 214 failure_line = True 215 216 # Mypy output mixes character encoding in color coded output 217 # and uses the 'sgr0' (or exit_attribute_mode) capability from the 218 # host machine's terminfo database. 219 # 220 # This can result in this sequence ending up in STDOUT as 221 # b'\x1b(B\x1b[m'. (B tells terminals to interpret text as 222 # USASCII encoding but will appear in prompt_toolkit as a B 223 # character. 224 # 225 # The following replace calls will strip out those 226 # sequences. 227 stripped_output = output.replace('\x1b(B', '').strip() 228 229 if not line_match_result: 230 logger_method(stripped_output) 231 recipe.status.current_step = stripped_output 232 233 if failure_line: 234 recipe.status.append_failure_line(stripped_output) 235 236 BUILDER_CONTEXT.redraw_progress() 237 238 if line_processed_callback: 239 line_processed_callback(recipe) 240 241 if BUILDER_CONTEXT.build_stopping(): 242 proc.terminate() 243 244 recipe.status.return_code = returncode 245 246 # Log the last failure if not done already 247 if failure_line and not BUILDER_CONTEXT.build_stopping(): 248 recipe.status.log_last_failure() 249 250 # Empty line at the end. 251 logger.info('') 252 253 return returncode == 0 254 255 256def log_build_recipe_start( 257 index_message: str, 258 project_builder: 'ProjectBuilder', 259 cfg: BuildRecipe, 260 logger: logging.Logger = _LOG, 261) -> None: 262 """Log recipe start and truncate the build logfile.""" 263 if project_builder.separate_build_file_logging and cfg.logfile: 264 # Truncate the file 265 with open(cfg.logfile, 'w'): 266 pass 267 268 BUILDER_CONTEXT.mark_progress_started(cfg) 269 270 build_start_msg = [ 271 index_message, 272 project_builder.color.cyan('Starting ==>'), 273 project_builder.color.blue('Recipe:'), 274 str(cfg.display_name), 275 project_builder.color.blue('Targets:'), 276 str(' '.join(cfg.targets())), 277 ] 278 279 if cfg.logfile: 280 build_start_msg.extend( 281 [ 282 project_builder.color.blue('Logfile:'), 283 str(cfg.logfile.resolve()), 284 ] 285 ) 286 build_start_str = ' '.join(build_start_msg) 287 288 # Log start to the root log if recipe logs are not sent. 289 if not project_builder.send_recipe_logs_to_root: 290 logger.info(build_start_str) 291 if cfg.logfile: 292 cfg.log.info(build_start_str) 293 294 295def log_build_recipe_finish( 296 index_message: str, 297 project_builder: 'ProjectBuilder', 298 cfg: BuildRecipe, 299 logger: logging.Logger = _LOG, 300) -> None: 301 """Log recipe finish and any build errors.""" 302 303 BUILDER_CONTEXT.mark_progress_done(cfg) 304 305 if BUILDER_CONTEXT.interrupted(): 306 level = logging.WARNING 307 tag = project_builder.color.yellow('(ABORT)') 308 elif cfg.status.failed(): 309 level = logging.ERROR 310 tag = project_builder.color.red('(FAIL)') 311 else: 312 level = logging.INFO 313 tag = project_builder.color.green('(OK)') 314 315 build_finish_msg = [ 316 level, 317 '%s %s %s %s %s', 318 index_message, 319 project_builder.color.cyan('Finished ==>'), 320 project_builder.color.blue('Recipe:'), 321 cfg.display_name, 322 tag, 323 ] 324 325 # Log finish to the root log if recipe logs are not sent. 326 if not project_builder.send_recipe_logs_to_root: 327 logger.log(*build_finish_msg) 328 if cfg.logfile: 329 cfg.log.log(*build_finish_msg) 330 331 if ( 332 not BUILDER_CONTEXT.build_stopping() 333 and cfg.status.failed() 334 and cfg.status.error_count == 0 335 ): 336 cfg.status.log_entire_recipe_logfile() 337 338 339class MissingGlobalLogfile(Exception): 340 """Exception raised if a global logfile is not specifed.""" 341 342 343class DispatchingFormatter(logging.Formatter): 344 """Dispatch log formatting based on the logger name.""" 345 346 def __init__(self, formatters, default_formatter): 347 self._formatters = formatters 348 self._default_formatter = default_formatter 349 super().__init__() 350 351 def format(self, record): 352 logger = logging.getLogger(record.name) 353 formatter = self._formatters.get(logger.name, self._default_formatter) 354 return formatter.format(record) 355 356 357class ProjectBuilder: # pylint: disable=too-many-instance-attributes 358 """Pigweed Project Builder 359 360 Controls how build recipes are executed and logged. 361 362 Example usage: 363 364 .. code-block:: python 365 366 import logging 367 from pathlib import Path 368 369 from pw_build.build_recipe import BuildCommand, BuildRecipe 370 from pw_build.project_builder import ProjectBuilder 371 372 def should_gen_gn(out: Path) -> bool: 373 return not (out / 'build.ninja').is_file() 374 375 recipe = BuildRecipe( 376 build_dir='out', 377 title='Vanilla Ninja Build', 378 steps=[ 379 BuildCommand(command=['gn', 'gen', '{build_dir}'], 380 run_if=should_gen_gn), 381 BuildCommand(build_system_command='ninja', 382 build_system_extra_args=['-k', '0'], 383 targets=['default']), 384 ], 385 ) 386 387 project_builder = ProjectBuilder( 388 build_recipes=[recipe1, ...] 389 banners=True, 390 log_level=logging.INFO 391 separate_build_file_logging=True, 392 root_logger=logging.getLogger(), 393 root_logfile=Path('build_log.txt'), 394 ) 395 396 Args: 397 build_recipes: List of build recipes. 398 jobs: The number of jobs bazel, make, and ninja should use by passing 399 ``-j`` to each. 400 keep_going: If True keep going flags are passed to bazel and ninja with 401 the ``-k`` option. 402 banners: Print the project banner at the start of each build. 403 allow_progress_bars: If False progress bar output will be disabled. 404 colors: Print ANSI colors to stdout and logfiles 405 log_level: Optional log_level, defaults to logging.INFO. 406 root_logfile: Optional root logfile. 407 separate_build_file_logging: If True separate logfiles will be created 408 per build recipe. The location of each file depends on if a 409 ``root_logfile`` is provided. If a root logfile is used each build 410 recipe logfile will be created in the same location. If no 411 root_logfile is specified the per build log files are placed in each 412 build dir as ``log.txt`` 413 send_recipe_logs_to_root: If True will send all build recipie output to 414 the root logger. This only makes sense to use if the builds are run 415 in serial. 416 use_verbatim_error_log_formatting: Use a blank log format when printing 417 errors from sub builds to the root logger. 418 """ 419 420 def __init__( 421 # pylint: disable=too-many-arguments,too-many-locals 422 self, 423 build_recipes: Sequence[BuildRecipe], 424 jobs: Optional[int] = None, 425 banners: bool = True, 426 keep_going: bool = False, 427 abort_callback: Callable = _exit, 428 execute_command: Callable[ 429 [List, Dict, BuildRecipe, logging.Logger, Optional[Callable]], bool 430 ] = execute_command_no_logging, 431 charset: ProjectBuilderCharset = ASCII_CHARSET, 432 colors: bool = True, 433 separate_build_file_logging: bool = False, 434 send_recipe_logs_to_root: bool = False, 435 root_logger: logging.Logger = _LOG, 436 root_logfile: Optional[Path] = None, 437 log_level: int = logging.INFO, 438 allow_progress_bars: bool = True, 439 use_verbatim_error_log_formatting: bool = False, 440 ): 441 self.charset: ProjectBuilderCharset = charset 442 self.abort_callback = abort_callback 443 # Function used to run subprocesses 444 self.execute_command = execute_command 445 self.banners = banners 446 self.build_recipes = build_recipes 447 self.max_name_width = max( 448 [len(str(step.display_name)) for step in self.build_recipes] 449 ) 450 # Set project_builder reference in each recipe. 451 for recipe in self.build_recipes: 452 recipe.set_project_builder(self) 453 454 # Save build system args 455 self.extra_ninja_args: List[str] = [] 456 self.extra_bazel_args: List[str] = [] 457 self.extra_bazel_build_args: List[str] = [] 458 459 # Handle jobs and keep going flags. 460 if jobs: 461 job_args = ['-j', f'{jobs}'] 462 self.extra_ninja_args.extend(job_args) 463 self.extra_bazel_build_args.extend(job_args) 464 if keep_going: 465 self.extra_ninja_args.extend(['-k', '0']) 466 self.extra_bazel_build_args.extend(['-k']) 467 468 self.colors = colors 469 # Reference to pw_cli.color, will return colored text if colors are 470 # enabled. 471 self.color = pw_cli.color.colors(colors) 472 473 # Pass color setting to bazel 474 if colors: 475 self.extra_bazel_args.append('--color=yes') 476 else: 477 self.extra_bazel_args.append('--color=no') 478 479 # Progress bar enable/disable flag 480 self.allow_progress_bars = allow_progress_bars 481 self.stdout_proxy: Optional[StdoutProxy] = None 482 483 # Logger configuration 484 self.root_logger = root_logger 485 self.default_logfile = root_logfile 486 self.default_log_level = log_level 487 # Create separate logs per build 488 self.separate_build_file_logging = separate_build_file_logging 489 # Propagate logs to the root looger 490 self.send_recipe_logs_to_root = send_recipe_logs_to_root 491 492 # Setup the error logger 493 self.use_verbatim_error_log_formatting = ( 494 use_verbatim_error_log_formatting 495 ) 496 self.error_logger = logging.getLogger(f'{root_logger.name}.errors') 497 self.error_logger.setLevel(log_level) 498 self.error_logger.propagate = True 499 for recipe in self.build_recipes: 500 recipe.set_error_logger(self.error_logger) 501 502 # Copy of the standard Pigweed style log formatter, used by default if 503 # no formatter exists on the root logger. 504 timestamp_fmt = self.color.black_on_white('%(asctime)s') + ' ' 505 self.default_log_formatter = logging.Formatter( 506 timestamp_fmt + '%(levelname)s %(message)s', '%Y%m%d %H:%M:%S' 507 ) 508 509 # Empty log formatter (optionally used for error reporting) 510 self.blank_log_formatter = logging.Formatter('%(message)s') 511 512 # Setup the default log handler and inherit user defined formatting on 513 # the root_logger. 514 self.apply_root_log_formatting() 515 516 # Create a root logfile to save what is normally logged to stdout. 517 if root_logfile: 518 # Execute subprocesses and capture logs 519 self.execute_command = execute_command_with_logging 520 521 root_logfile.parent.mkdir(parents=True, exist_ok=True) 522 523 build_log_filehandler = logging.FileHandler( 524 root_logfile, 525 encoding='utf-8', 526 # Truncate the file 527 mode='w', 528 ) 529 build_log_filehandler.setLevel(log_level) 530 build_log_filehandler.setFormatter(self.dispatching_log_formatter) 531 root_logger.addHandler(build_log_filehandler) 532 533 # Set each recipe to use the root logger by default. 534 for recipe in self.build_recipes: 535 recipe.set_logger(root_logger) 536 537 # Create separate logfiles per build 538 if separate_build_file_logging: 539 self._create_per_build_logfiles() 540 541 def _create_per_build_logfiles(self) -> None: 542 """Create separate log files per build. 543 544 If a root logfile is used, create per build log files in the same 545 location. If no root logfile is specified create the per build log files 546 in the build dir as ``log.txt`` 547 """ 548 self.execute_command = execute_command_with_logging 549 550 for recipe in self.build_recipes: 551 sub_logger_name = recipe.display_name.replace('.', '_') 552 new_logger = logging.getLogger( 553 f'{self.root_logger.name}.{sub_logger_name}' 554 ) 555 new_logger.setLevel(self.default_log_level) 556 new_logger.propagate = self.send_recipe_logs_to_root 557 558 new_logfile_dir = recipe.build_dir 559 new_logfile_name = Path('log.txt') 560 new_logfile_postfix = '' 561 if self.default_logfile: 562 new_logfile_dir = self.default_logfile.parent 563 new_logfile_name = self.default_logfile 564 new_logfile_postfix = '_' + recipe.display_name.replace( 565 ' ', '_' 566 ) 567 568 new_logfile = new_logfile_dir / ( 569 new_logfile_name.stem 570 + new_logfile_postfix 571 + new_logfile_name.suffix 572 ) 573 574 new_logfile_dir.mkdir(parents=True, exist_ok=True) 575 new_log_filehandler = logging.FileHandler( 576 new_logfile, 577 encoding='utf-8', 578 # Truncate the file 579 mode='w', 580 ) 581 new_log_filehandler.setLevel(self.default_log_level) 582 new_log_filehandler.setFormatter(self.dispatching_log_formatter) 583 new_logger.addHandler(new_log_filehandler) 584 585 recipe.set_logger(new_logger) 586 recipe.set_logfile(new_logfile) 587 588 def apply_root_log_formatting(self) -> None: 589 """Inherit user defined formatting from the root_logger.""" 590 # Use the the existing root logger formatter if one exists. 591 for handler in logging.getLogger().handlers: 592 if handler.formatter: 593 self.default_log_formatter = handler.formatter 594 break 595 596 formatter_mapping = { 597 self.root_logger.name: self.default_log_formatter, 598 } 599 if self.use_verbatim_error_log_formatting: 600 formatter_mapping[self.error_logger.name] = self.blank_log_formatter 601 602 self.dispatching_log_formatter = DispatchingFormatter( 603 formatter_mapping, 604 self.default_log_formatter, 605 ) 606 607 def should_use_progress_bars(self) -> bool: 608 if not self.allow_progress_bars: 609 return False 610 if self.separate_build_file_logging or self.default_logfile: 611 return True 612 return False 613 614 def use_stdout_proxy(self) -> None: 615 """Setup StdoutProxy for progress bars.""" 616 617 self.stdout_proxy = StdoutProxy(raw=True) 618 root_logger = logging.getLogger() 619 handlers = root_logger.handlers + self.error_logger.handlers 620 621 for handler in handlers: 622 # Must use type() check here since this returns True: 623 # isinstance(logging.FileHandler, logging.StreamHandler) 624 # pylint: disable=unidiomatic-typecheck 625 if type(handler) == logging.StreamHandler: 626 handler.setStream(self.stdout_proxy) # type: ignore 627 handler.setFormatter(self.dispatching_log_formatter) 628 # pylint: enable=unidiomatic-typecheck 629 630 def flush_log_handlers(self) -> None: 631 root_logger = logging.getLogger() 632 handlers = root_logger.handlers + self.error_logger.handlers 633 for cfg in self: 634 handlers.extend(cfg.log.handlers) 635 for handler in handlers: 636 handler.flush() 637 if self.stdout_proxy: 638 self.stdout_proxy.flush() 639 self.stdout_proxy.close() 640 641 def __len__(self) -> int: 642 return len(self.build_recipes) 643 644 def __getitem__(self, index: int) -> BuildRecipe: 645 return self.build_recipes[index] 646 647 def __iter__(self) -> Generator[BuildRecipe, None, None]: 648 return (build_recipe for build_recipe in self.build_recipes) 649 650 def run_build( 651 self, 652 cfg: BuildRecipe, 653 env: Dict, 654 index_message: Optional[str] = '', 655 ) -> bool: 656 """Run a single build config.""" 657 if BUILDER_CONTEXT.build_stopping(): 658 return False 659 660 if self.colors: 661 # Force colors in Pigweed subcommands run through the watcher. 662 env['PW_USE_COLOR'] = '1' 663 # Force Ninja to output ANSI colors 664 env['CLICOLOR_FORCE'] = '1' 665 666 build_succeded = False 667 cfg.reset_status() 668 cfg.status.mark_started() 669 for command_step in cfg.steps: 670 command_args = command_step.get_args( 671 additional_ninja_args=self.extra_ninja_args, 672 additional_bazel_args=self.extra_bazel_args, 673 additional_bazel_build_args=self.extra_bazel_build_args, 674 ) 675 676 # Verify that the build output directories exist. 677 if ( 678 command_step.build_system_command is not None 679 and cfg.build_dir 680 and (not cfg.build_dir.is_dir()) 681 ): 682 self.abort_callback( 683 'Build directory does not exist: %s', cfg.build_dir 684 ) 685 686 quoted_command_args = ' '.join( 687 shlex.quote(arg) for arg in command_args 688 ) 689 build_succeded = True 690 if command_step.should_run(): 691 cfg.log.info( 692 '%s %s %s', 693 index_message, 694 self.color.blue('Run ==>'), 695 quoted_command_args, 696 ) 697 build_succeded = self.execute_command( 698 command_args, env, cfg, cfg.log, None 699 ) 700 else: 701 cfg.log.info( 702 '%s %s %s', 703 index_message, 704 self.color.yellow('Skipped ==>'), 705 quoted_command_args, 706 ) 707 708 BUILDER_CONTEXT.mark_progress_step_complete(cfg) 709 # Don't run further steps if a command fails. 710 if not build_succeded: 711 break 712 713 # If all steps were skipped the return code will not be set. Force 714 # status to passed in this case. 715 if build_succeded and not cfg.status.passed(): 716 cfg.status.set_passed() 717 718 cfg.status.mark_done() 719 720 return build_succeded 721 722 def print_pass_fail_banner( 723 self, 724 cancelled: bool = False, 725 logger: logging.Logger = _LOG, 726 ) -> None: 727 # Check conditions where banners should not be shown: 728 # Banner flag disabled. 729 if not self.banners: 730 return 731 # If restarting or interrupted. 732 if BUILDER_CONTEXT.interrupted(): 733 if BUILDER_CONTEXT.ctrl_c_pressed: 734 _LOG.info( 735 self.color.yellow('Exited due to keyboard interrupt.') 736 ) 737 return 738 # If any build is still pending. 739 if any(recipe.status.pending() for recipe in self): 740 return 741 742 # Show a large color banner for the overall result. 743 if all(recipe.status.passed() for recipe in self) and not cancelled: 744 for line in PASS_MESSAGE.splitlines(): 745 logger.info(self.color.green(line)) 746 else: 747 for line in FAIL_MESSAGE.splitlines(): 748 logger.info(self.color.red(line)) 749 750 def print_build_summary( 751 self, 752 cancelled: bool = False, 753 logger: logging.Logger = _LOG, 754 ) -> None: 755 """Print build status summary table.""" 756 757 build_descriptions = [] 758 build_status = [] 759 760 for cfg in self: 761 description = [str(cfg.display_name).ljust(self.max_name_width)] 762 description.append(' '.join(cfg.targets())) 763 build_descriptions.append(' '.join(description)) 764 765 if cfg.status.passed(): 766 build_status.append(self.charset.slug_ok) 767 elif cfg.status.failed(): 768 build_status.append(self.charset.slug_fail) 769 else: 770 build_status.append(self.charset.slug_building) 771 772 if not cancelled: 773 logger.info(' ╔════════════════════════════════════') 774 logger.info(' ║') 775 776 for slug, cmd in zip(build_status, build_descriptions): 777 logger.info(' ║ %s %s', slug, cmd) 778 779 logger.info(' ║') 780 logger.info(" ╚════════════════════════════════════") 781 else: 782 # Build was interrupted. 783 logger.info('') 784 logger.info(' ╔════════════════════════════════════') 785 logger.info(' ║') 786 logger.info(' ║ %s- interrupted', self.charset.slug_fail) 787 logger.info(' ║') 788 logger.info(" ╚════════════════════════════════════") 789 790 791def run_recipe( 792 index: int, project_builder: ProjectBuilder, cfg: BuildRecipe, env 793) -> bool: 794 if BUILDER_CONTEXT.interrupted(): 795 return False 796 if not cfg.enabled: 797 return False 798 799 num_builds = len(project_builder) 800 index_message = f'[{index}/{num_builds}]' 801 802 result = False 803 804 log_build_recipe_start(index_message, project_builder, cfg) 805 806 result = project_builder.run_build(cfg, env, index_message=index_message) 807 808 log_build_recipe_finish(index_message, project_builder, cfg) 809 810 return result 811 812 813def run_builds(project_builder: ProjectBuilder, workers: int = 1) -> int: 814 """Execute build steps in the ProjectBuilder and print a summary. 815 816 Returns: 1 for a failed build, 0 for success.""" 817 num_builds = len(project_builder) 818 _LOG.info('Starting build with %d directories', num_builds) 819 if project_builder.default_logfile: 820 _LOG.info( 821 '%s %s', 822 project_builder.color.blue('Root logfile:'), 823 project_builder.default_logfile.resolve(), 824 ) 825 826 env = os.environ.copy() 827 828 # Print status before starting 829 if not project_builder.should_use_progress_bars(): 830 project_builder.print_build_summary() 831 project_builder.print_pass_fail_banner() 832 833 if workers > 1 and not project_builder.separate_build_file_logging: 834 _LOG.warning( 835 project_builder.color.yellow( 836 'Running in parallel without --separate-logfiles; All build ' 837 'output will be interleaved.' 838 ) 839 ) 840 841 BUILDER_CONTEXT.set_project_builder(project_builder) 842 BUILDER_CONTEXT.set_building() 843 844 def _cleanup() -> None: 845 if not project_builder.should_use_progress_bars(): 846 project_builder.print_build_summary() 847 project_builder.print_pass_fail_banner() 848 project_builder.flush_log_handlers() 849 BUILDER_CONTEXT.set_idle() 850 BUILDER_CONTEXT.exit_progress() 851 852 if workers == 1: 853 # TODO(tonymd): Try to remove this special case. Using 854 # ThreadPoolExecutor when running in serial (workers==1) currently 855 # breaks Ctrl-C handling. Build processes keep running. 856 try: 857 if project_builder.should_use_progress_bars(): 858 BUILDER_CONTEXT.add_progress_bars() 859 for i, cfg in enumerate(project_builder, start=1): 860 run_recipe(i, project_builder, cfg, env) 861 # Ctrl-C on Unix generates KeyboardInterrupt 862 # Ctrl-Z on Windows generates EOFError 863 except (KeyboardInterrupt, EOFError): 864 _exit_due_to_interrupt() 865 finally: 866 _cleanup() 867 868 else: 869 with concurrent.futures.ThreadPoolExecutor( 870 max_workers=workers 871 ) as executor: 872 futures = [] 873 for i, cfg in enumerate(project_builder, start=1): 874 futures.append( 875 executor.submit(run_recipe, i, project_builder, cfg, env) 876 ) 877 878 try: 879 if project_builder.should_use_progress_bars(): 880 BUILDER_CONTEXT.add_progress_bars() 881 for future in concurrent.futures.as_completed(futures): 882 future.result() 883 # Ctrl-C on Unix generates KeyboardInterrupt 884 # Ctrl-Z on Windows generates EOFError 885 except (KeyboardInterrupt, EOFError): 886 _exit_due_to_interrupt() 887 finally: 888 _cleanup() 889 890 project_builder.flush_log_handlers() 891 return BUILDER_CONTEXT.exit_code() 892 893 894def main() -> int: 895 """Build a Pigweed Project.""" 896 parser = argparse.ArgumentParser( 897 description=__doc__, 898 formatter_class=argparse.RawDescriptionHelpFormatter, 899 ) 900 parser = add_project_builder_arguments(parser) 901 args = parser.parse_args() 902 903 pw_env = pw_cli.env.pigweed_environment() 904 if pw_env.PW_EMOJI: 905 charset = EMOJI_CHARSET 906 else: 907 charset = ASCII_CHARSET 908 909 prefs = ProjectBuilderPrefs( 910 load_argparse_arguments=add_project_builder_arguments 911 ) 912 prefs.apply_command_line_args(args) 913 build_recipes = create_build_recipes(prefs) 914 915 log_level = logging.DEBUG if args.debug_logging else logging.INFO 916 917 pw_cli.log.install( 918 level=log_level, 919 use_color=args.colors, 920 hide_timestamp=False, 921 ) 922 923 project_builder = ProjectBuilder( 924 build_recipes=build_recipes, 925 jobs=args.jobs, 926 banners=args.banners, 927 keep_going=args.keep_going, 928 colors=args.colors, 929 charset=charset, 930 separate_build_file_logging=args.separate_logfiles, 931 root_logfile=args.logfile, 932 root_logger=_LOG, 933 log_level=log_level, 934 ) 935 936 if project_builder.should_use_progress_bars(): 937 project_builder.use_stdout_proxy() 938 939 workers = 1 940 if args.parallel: 941 # If parallel is requested and parallel_workers is set to 0 run all 942 # recipes in parallel. That is, use the number of recipes as the worker 943 # count. 944 if args.parallel_workers == 0: 945 workers = len(project_builder) 946 else: 947 workers = args.parallel_workers 948 949 return run_builds(project_builder, workers) 950 951 952if __name__ == '__main__': 953 sys.exit(main()) 954