1# Copyright 2022 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Build a Pigweed Project. 15 16Run arbitrary commands or invoke build systems (Ninja, Bazel and make) on one or 17more build directories. 18 19Examples: 20 21.. code-block: sh 22 23 # Build the default target in out/ using ninja. 24 python -m pw_build.project_builder -C out 25 26 # Build pw_run_tests.modules in the out/cmake directory 27 python -m pw_build.project_builder -C out/cmake pw_run_tests.modules 28 29 # Build the default target in out/ and pw_apps in out/cmake 30 python -m pw_build.project_builder -C out -C out/cmake pw_apps 31 32 # Build python.tests in out/ and pw_apps in out/cmake/ 33 python -m pw_build.project_builder python.tests -C out/cmake pw_apps 34 35 # Run 'bazel build' and 'bazel test' on the target '//...' in outbazel/ 36 python -m pw_build.project_builder --run-command 'mkdir -p outbazel' 37 -C outbazel '//...' 38 --build-system-command outbazel 'bazel build' 39 --build-system-command outbazel 'bazel test' 40""" 41 42from __future__ import annotations 43 44import argparse 45import concurrent.futures 46import os 47import logging 48from pathlib import Path 49import re 50import shlex 51import sys 52import subprocess 53import time 54from typing import ( 55 Callable, 56 Generator, 57 NoReturn, 58 Sequence, 59 NamedTuple, 60) 61 62from prompt_toolkit.patch_stdout import StdoutProxy 63 64import pw_cli.env 65import pw_cli.log 66 67from pw_build.build_recipe import BuildRecipe, create_build_recipes 68from pw_build.project_builder_argparse import add_project_builder_arguments 69from pw_build.project_builder_context import get_project_builder_context 70from pw_build.project_builder_prefs import ProjectBuilderPrefs 71 72_COLOR = pw_cli.color.colors() 73_LOG = logging.getLogger('pw_build') 74 75BUILDER_CONTEXT = get_project_builder_context() 76 77PASS_MESSAGE = """ 78 ██████╗ █████╗ ███████╗███████╗██╗ 79 ██╔══██╗██╔══██╗██╔════╝██╔════╝██║ 80 ██████╔╝███████║███████╗███████╗██║ 81 ██╔═══╝ ██╔══██║╚════██║╚════██║╚═╝ 82 ██║ ██║ ██║███████║███████║██╗ 83 ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝ 84""" 85 86# Pick a visually-distinct font from "PASS" to ensure that readers can't 87# possibly mistake the difference between the two states. 88FAIL_MESSAGE = """ 89 ▄██████▒░▄▄▄ ██▓ ░██▓ 90 ▓█▓ ░▒████▄ ▓██▒ ░▓██▒ 91 ▒████▒ ░▒█▀ ▀█▄ ▒██▒ ▒██░ 92 ░▓█▒ ░░██▄▄▄▄██ ░██░ ▒██░ 93 ░▒█░ ▓█ ▓██▒░██░░ ████████▒ 94 ▒█░ ▒▒ ▓▒█░░▓ ░ ▒░▓ ░ 95 ░▒ ▒ ▒▒ ░ ▒ ░░ ░ ▒ ░ 96 ░ ░ ░ ▒ ▒ ░ ░ ░ 97 ░ ░ ░ ░ ░ 98""" 99 100 101class ProjectBuilderCharset(NamedTuple): 102 slug_ok: str 103 slug_fail: str 104 slug_building: str 105 106 107ASCII_CHARSET = ProjectBuilderCharset( 108 _COLOR.green('OK '), 109 _COLOR.red('FAIL'), 110 _COLOR.yellow('... '), 111) 112EMOJI_CHARSET = ProjectBuilderCharset('✔️ ', '❌', '⏱️ ') 113 114 115def _exit(*args) -> NoReturn: 116 _LOG.critical(*args) 117 sys.exit(1) 118 119 120def _exit_due_to_interrupt() -> None: 121 """Abort function called when not using progress bars.""" 122 # To keep the log lines aligned with each other in the presence of 123 # a '^C' from the keyboard interrupt, add a newline before the log. 124 print() 125 _LOG.info('Got Ctrl-C; exiting...') 126 BUILDER_CONTEXT.ctrl_c_interrupt() 127 128 129_NINJA_BUILD_STEP = re.compile( 130 # Start of line 131 r'^' 132 # Step count: [1234/5678] 133 r'\[(?P<step>[0-9]+)/(?P<total_steps>[0-9]+)\]' 134 # whitespace 135 r' *' 136 # Build step text 137 r'(?P<action>.*)$' 138) 139 140_BAZEL_BUILD_STEP = re.compile( 141 # Start of line 142 r'^' 143 # Optional starting green color 144 r'(?:\x1b\[32m)?' 145 # Step count: [1,234 / 5,678] 146 r'\[(?P<step>[0-9,]+) */ *(?P<total_steps>[0-9,]+)\]' 147 # Optional ending clear color and space 148 r'(?:\x1b\[0m)? *' 149 # Build step text 150 r'(?P<action>.*)$' 151) 152 153_NINJA_FAILURE = re.compile( 154 # Start of line 155 r'^' 156 # Optional red color 157 r'(?:\x1b\[31m)?' 158 r'FAILED:' 159 r' ' 160 # Optional color reset 161 r'(?:\x1b\[0m)?' 162) 163 164_BAZEL_FAILURE = re.compile( 165 # Start of line 166 r'^' 167 # Optional red color 168 r'(?:\x1b\[31m)?' 169 # Optional bold color 170 r'(?:\x1b\[1m)?' 171 r'FAIL:' 172 # Optional color reset 173 r'(?:\x1b\[0m)?' 174 # whitespace 175 r' *' 176 r'.*bazel-out.*' 177) 178 179_BAZEL_ELAPSED_TIME = re.compile( 180 # Start of line 181 r'^' 182 # Optional green color 183 r'(?:\x1b\[32m)?' 184 r'INFO:' 185 # single space 186 r' ' 187 # Optional color reset 188 r'(?:\x1b\[0m)?' 189 r'Elapsed time:' 190) 191 192 193def execute_command_no_logging( 194 command: list, 195 env: dict, 196 recipe: BuildRecipe, 197 # pylint: disable=unused-argument 198 logger: logging.Logger = _LOG, 199 line_processed_callback: Callable[[BuildRecipe], None] | None = None, 200 # pylint: enable=unused-argument 201) -> bool: 202 print() 203 proc = subprocess.Popen(command, env=env, errors='replace') 204 BUILDER_CONTEXT.register_process(recipe, proc) 205 returncode = None 206 while returncode is None: 207 if BUILDER_CONTEXT.build_stopping(): 208 try: 209 proc.terminate() 210 except ProcessLookupError: 211 # Process already stopped. 212 pass 213 returncode = proc.poll() 214 time.sleep(0.05) 215 print() 216 recipe.status.return_code = returncode 217 218 return proc.returncode == 0 219 220 221def execute_command_with_logging( 222 command: list, 223 env: dict, 224 recipe: BuildRecipe, 225 logger: logging.Logger = _LOG, 226 line_processed_callback: Callable[[BuildRecipe], None] | None = None, 227) -> bool: 228 """Run a command in a subprocess and log all output.""" 229 current_stdout = '' 230 returncode = None 231 232 starting_failure_regex = _NINJA_FAILURE 233 ending_failure_regex = _NINJA_BUILD_STEP 234 build_step_regex = _NINJA_BUILD_STEP 235 236 if command[0].endswith('ninja'): 237 build_step_regex = _NINJA_BUILD_STEP 238 starting_failure_regex = _NINJA_FAILURE 239 ending_failure_regex = _NINJA_BUILD_STEP 240 elif command[0].endswith('bazel'): 241 build_step_regex = _BAZEL_BUILD_STEP 242 starting_failure_regex = _BAZEL_FAILURE 243 ending_failure_regex = _BAZEL_ELAPSED_TIME 244 245 with subprocess.Popen( 246 command, 247 env=env, 248 stdout=subprocess.PIPE, 249 stderr=subprocess.STDOUT, 250 errors='replace', 251 ) as proc: 252 BUILDER_CONTEXT.register_process(recipe, proc) 253 should_log_build_steps = BUILDER_CONTEXT.log_build_steps 254 # Empty line at the start. 255 logger.info('') 256 257 failure_line = False 258 while returncode is None: 259 output = '' 260 error_output = '' 261 262 if proc.stdout: 263 output = proc.stdout.readline() 264 current_stdout += output 265 if proc.stderr: 266 error_output += proc.stderr.readline() 267 current_stdout += error_output 268 269 if not output and not error_output: 270 returncode = proc.poll() 271 continue 272 273 line_match_result = build_step_regex.match(output) 274 if line_match_result: 275 if failure_line and not BUILDER_CONTEXT.build_stopping(): 276 recipe.status.log_last_failure() 277 failure_line = False 278 matches = line_match_result.groupdict() 279 recipe.status.current_step = line_match_result.group(0) 280 recipe.status.percent = 0.0 281 # Remove commas from step and total_steps strings 282 step = int(matches.get('step', '0').replace(',', '')) 283 total_steps = int( 284 matches.get('total_steps', '1').replace(',', '') 285 ) 286 if total_steps > 0: 287 recipe.status.percent = float(step / total_steps) 288 289 logger_method = logger.info 290 if starting_failure_regex.match(output): 291 logger_method = logger.error 292 if failure_line and not BUILDER_CONTEXT.build_stopping(): 293 recipe.status.log_last_failure() 294 recipe.status.increment_error_count() 295 failure_line = True 296 elif ending_failure_regex.match(output): 297 if failure_line and not BUILDER_CONTEXT.build_stopping(): 298 recipe.status.log_last_failure() 299 failure_line = False 300 301 # Mypy output mixes character encoding in color coded output 302 # and uses the 'sgr0' (or exit_attribute_mode) capability from the 303 # host machine's terminfo database. 304 # 305 # This can result in this sequence ending up in STDOUT as 306 # b'\x1b(B\x1b[m'. (B tells terminals to interpret text as 307 # USASCII encoding but will appear in prompt_toolkit as a B 308 # character. 309 # 310 # The following replace calls will strip out those 311 # sequences. 312 stripped_output = output.replace('\x1b(B', '').strip() 313 314 # If this isn't a build step. 315 if not line_match_result or ( 316 # Or if this is a build step and logging build steps is 317 # requested: 318 line_match_result 319 and should_log_build_steps 320 ): 321 # Log this line. 322 logger_method(stripped_output) 323 recipe.status.current_step = stripped_output 324 325 if failure_line: 326 recipe.status.append_failure_line(stripped_output) 327 328 BUILDER_CONTEXT.redraw_progress() 329 330 if line_processed_callback: 331 line_processed_callback(recipe) 332 333 if BUILDER_CONTEXT.build_stopping(): 334 try: 335 proc.terminate() 336 except ProcessLookupError: 337 # Process already stopped. 338 pass 339 340 recipe.status.return_code = returncode 341 342 # Log the last failure if not done already 343 if failure_line and not BUILDER_CONTEXT.build_stopping(): 344 recipe.status.log_last_failure() 345 346 # Empty line at the end. 347 logger.info('') 348 349 return returncode == 0 350 351 352def log_build_recipe_start( 353 index_message: str, 354 project_builder: ProjectBuilder, 355 cfg: BuildRecipe, 356 logger: logging.Logger = _LOG, 357) -> None: 358 """Log recipe start and truncate the build logfile.""" 359 if project_builder.separate_build_file_logging and cfg.logfile: 360 # Truncate the file 361 with open(cfg.logfile, 'w'): 362 pass 363 364 BUILDER_CONTEXT.mark_progress_started(cfg) 365 366 build_start_msg = [ 367 index_message, 368 project_builder.color.cyan('Starting ==>'), 369 project_builder.color.blue('Recipe:'), 370 str(cfg.display_name), 371 project_builder.color.blue('Targets:'), 372 str(' '.join(cfg.targets())), 373 ] 374 375 if cfg.logfile: 376 build_start_msg.extend( 377 [ 378 project_builder.color.blue('Logfile:'), 379 str(cfg.logfile.resolve()), 380 ] 381 ) 382 build_start_str = ' '.join(build_start_msg) 383 384 # Log start to the root log if recipe logs are not sent. 385 if not project_builder.send_recipe_logs_to_root: 386 logger.info(build_start_str) 387 if cfg.logfile: 388 cfg.log.info(build_start_str) 389 390 391def log_build_recipe_finish( 392 index_message: str, 393 project_builder: ProjectBuilder, 394 cfg: BuildRecipe, 395 logger: logging.Logger = _LOG, 396) -> None: 397 """Log recipe finish and any build errors.""" 398 399 BUILDER_CONTEXT.mark_progress_done(cfg) 400 401 if BUILDER_CONTEXT.interrupted(): 402 level = logging.WARNING 403 tag = project_builder.color.yellow('(ABORT)') 404 elif cfg.status.failed(): 405 level = logging.ERROR 406 tag = project_builder.color.red('(FAIL)') 407 else: 408 level = logging.INFO 409 tag = project_builder.color.green('(OK)') 410 411 build_finish_msg = [ 412 level, 413 '%s %s %s %s %s', 414 index_message, 415 project_builder.color.cyan('Finished ==>'), 416 project_builder.color.blue('Recipe:'), 417 cfg.display_name, 418 tag, 419 ] 420 421 # Log finish to the root log if recipe logs are not sent. 422 if not project_builder.send_recipe_logs_to_root: 423 logger.log(*build_finish_msg) 424 if cfg.logfile: 425 cfg.log.log(*build_finish_msg) 426 427 if ( 428 not BUILDER_CONTEXT.build_stopping() 429 and cfg.status.failed() 430 and (cfg.status.error_count == 0 or cfg.status.has_empty_ninja_errors()) 431 ): 432 cfg.status.log_entire_recipe_logfile() 433 434 435class MissingGlobalLogfile(Exception): 436 """Exception raised if a global logfile is not specifed.""" 437 438 439class DispatchingFormatter(logging.Formatter): 440 """Dispatch log formatting based on the logger name.""" 441 442 def __init__(self, formatters, default_formatter): 443 self._formatters = formatters 444 self._default_formatter = default_formatter 445 super().__init__() 446 447 def format(self, record): 448 logger = logging.getLogger(record.name) 449 formatter = self._formatters.get(logger.name, self._default_formatter) 450 return formatter.format(record) 451 452 453class ProjectBuilder: # pylint: disable=too-many-instance-attributes 454 """Pigweed Project Builder 455 456 Controls how build recipes are executed and logged. 457 458 Example usage: 459 460 .. code-block:: python 461 462 import logging 463 from pathlib import Path 464 465 from pw_build.build_recipe import BuildCommand, BuildRecipe 466 from pw_build.project_builder import ProjectBuilder 467 468 def should_gen_gn(out: Path) -> bool: 469 return not (out / 'build.ninja').is_file() 470 471 recipe = BuildRecipe( 472 build_dir='out', 473 title='Vanilla Ninja Build', 474 steps=[ 475 BuildCommand(command=['gn', 'gen', '{build_dir}'], 476 run_if=should_gen_gn), 477 BuildCommand(build_system_command='ninja', 478 build_system_extra_args=['-k', '0'], 479 targets=['default']), 480 ], 481 ) 482 483 project_builder = ProjectBuilder( 484 build_recipes=[recipe1, ...] 485 banners=True, 486 log_level=logging.INFO 487 separate_build_file_logging=True, 488 root_logger=logging.getLogger(), 489 root_logfile=Path('build_log.txt'), 490 ) 491 492 Args: 493 build_recipes: List of build recipes. 494 jobs: The number of jobs bazel, make, and ninja should use by passing 495 ``-j`` to each. 496 keep_going: If True keep going flags are passed to bazel and ninja with 497 the ``-k`` option. 498 banners: Print the project banner at the start of each build. 499 allow_progress_bars: If False progress bar output will be disabled. 500 log_build_steps: If True all build step lines will be logged to the 501 screen and logfiles. Default: False. 502 colors: Print ANSI colors to stdout and logfiles 503 log_level: Optional log_level, defaults to logging.INFO. 504 root_logfile: Optional root logfile. 505 separate_build_file_logging: If True separate logfiles will be created 506 per build recipe. The location of each file depends on if a 507 ``root_logfile`` is provided. If a root logfile is used each build 508 recipe logfile will be created in the same location. If no 509 root_logfile is specified the per build log files are placed in each 510 build dir as ``log.txt`` 511 send_recipe_logs_to_root: If True will send all build recipie output to 512 the root logger. This only makes sense to use if the builds are run 513 in serial. 514 use_verbatim_error_log_formatting: Use a blank log format when printing 515 errors from sub builds to the root logger. 516 """ 517 518 def __init__( 519 # pylint: disable=too-many-arguments,too-many-locals 520 self, 521 build_recipes: Sequence[BuildRecipe], 522 jobs: int | None = None, 523 banners: bool = True, 524 keep_going: bool = False, 525 abort_callback: Callable = _exit, 526 execute_command: Callable[ 527 [list, dict, BuildRecipe, logging.Logger, Callable | None], bool 528 ] = execute_command_no_logging, 529 charset: ProjectBuilderCharset = ASCII_CHARSET, 530 colors: bool = True, 531 separate_build_file_logging: bool = False, 532 send_recipe_logs_to_root: bool = False, 533 root_logger: logging.Logger = _LOG, 534 root_logfile: Path | None = None, 535 log_level: int = logging.INFO, 536 allow_progress_bars: bool = True, 537 use_verbatim_error_log_formatting: bool = False, 538 log_build_steps: bool = False, 539 ): 540 self.charset: ProjectBuilderCharset = charset 541 self.abort_callback = abort_callback 542 # Function used to run subprocesses 543 self.execute_command = execute_command 544 self.banners = banners 545 self.build_recipes = build_recipes 546 self.max_name_width = max( 547 [len(str(step.display_name)) for step in self.build_recipes] 548 ) 549 # Set project_builder reference in each recipe. 550 for recipe in self.build_recipes: 551 recipe.set_project_builder(self) 552 553 # Save build system args 554 self.extra_ninja_args: list[str] = [] 555 self.extra_bazel_args: list[str] = [] 556 self.extra_bazel_build_args: list[str] = [] 557 558 # Handle jobs and keep going flags. 559 if jobs: 560 job_args = ['-j', f'{jobs}'] 561 self.extra_ninja_args.extend(job_args) 562 self.extra_bazel_build_args.extend(job_args) 563 if keep_going: 564 self.extra_ninja_args.extend(['-k', '0']) 565 self.extra_bazel_build_args.extend(['-k']) 566 567 self.colors = colors 568 # Reference to pw_cli.color, will return colored text if colors are 569 # enabled. 570 self.color = pw_cli.color.colors(colors) 571 572 # Pass color setting to bazel 573 if colors: 574 self.extra_bazel_args.append('--color=yes') 575 else: 576 self.extra_bazel_args.append('--color=no') 577 578 # Progress bar enable/disable flag 579 self.allow_progress_bars = allow_progress_bars 580 self.log_build_steps = log_build_steps 581 self.stdout_proxy: StdoutProxy | None = None 582 583 # Logger configuration 584 self.root_logger = root_logger 585 self.default_logfile = root_logfile 586 self.default_log_level = log_level 587 # Create separate logs per build 588 self.separate_build_file_logging = separate_build_file_logging 589 # Propagate logs to the root looger 590 self.send_recipe_logs_to_root = send_recipe_logs_to_root 591 592 # Setup the error logger 593 self.use_verbatim_error_log_formatting = ( 594 use_verbatim_error_log_formatting 595 ) 596 self.error_logger = logging.getLogger(f'{root_logger.name}.errors') 597 self.error_logger.setLevel(log_level) 598 self.error_logger.propagate = True 599 for recipe in self.build_recipes: 600 recipe.set_error_logger(self.error_logger) 601 602 # Copy of the standard Pigweed style log formatter, used by default if 603 # no formatter exists on the root logger. 604 timestamp_fmt = self.color.black_on_white('%(asctime)s') + ' ' 605 self.default_log_formatter = logging.Formatter( 606 timestamp_fmt + '%(levelname)s %(message)s', '%Y%m%d %H:%M:%S' 607 ) 608 609 # Empty log formatter (optionally used for error reporting) 610 self.blank_log_formatter = logging.Formatter('%(message)s') 611 612 # Setup the default log handler and inherit user defined formatting on 613 # the root_logger. 614 self.apply_root_log_formatting() 615 616 # Create a root logfile to save what is normally logged to stdout. 617 if root_logfile: 618 # Execute subprocesses and capture logs 619 self.execute_command = execute_command_with_logging 620 621 root_logfile.parent.mkdir(parents=True, exist_ok=True) 622 623 build_log_filehandler = logging.FileHandler( 624 root_logfile, 625 encoding='utf-8', 626 # Truncate the file 627 mode='w', 628 ) 629 build_log_filehandler.setLevel(log_level) 630 build_log_filehandler.setFormatter(self.dispatching_log_formatter) 631 root_logger.addHandler(build_log_filehandler) 632 633 # Set each recipe to use the root logger by default. 634 for recipe in self.build_recipes: 635 recipe.set_logger(root_logger) 636 637 # Create separate logfiles per build 638 if separate_build_file_logging: 639 self._create_per_build_logfiles() 640 641 def _create_per_build_logfiles(self) -> None: 642 """Create separate log files per build. 643 644 If a root logfile is used, create per build log files in the same 645 location. If no root logfile is specified create the per build log files 646 in the build dir as ``log.txt`` 647 """ 648 self.execute_command = execute_command_with_logging 649 650 for recipe in self.build_recipes: 651 sub_logger_name = recipe.display_name.replace('.', '_') 652 new_logger = logging.getLogger( 653 f'{self.root_logger.name}.{sub_logger_name}' 654 ) 655 new_logger.setLevel(self.default_log_level) 656 new_logger.propagate = self.send_recipe_logs_to_root 657 658 new_logfile_dir = recipe.build_dir 659 new_logfile_name = Path('log.txt') 660 new_logfile_postfix = '' 661 if self.default_logfile: 662 new_logfile_dir = self.default_logfile.parent 663 new_logfile_name = self.default_logfile 664 new_logfile_postfix = '_' + recipe.display_name.replace( 665 ' ', '_' 666 ) 667 668 new_logfile = new_logfile_dir / ( 669 new_logfile_name.stem 670 + new_logfile_postfix 671 + new_logfile_name.suffix 672 ) 673 674 new_logfile_dir.mkdir(parents=True, exist_ok=True) 675 new_log_filehandler = logging.FileHandler( 676 new_logfile, 677 encoding='utf-8', 678 # Truncate the file 679 mode='w', 680 ) 681 new_log_filehandler.setLevel(self.default_log_level) 682 new_log_filehandler.setFormatter(self.dispatching_log_formatter) 683 new_logger.addHandler(new_log_filehandler) 684 685 recipe.set_logger(new_logger) 686 recipe.set_logfile(new_logfile) 687 688 def apply_root_log_formatting(self) -> None: 689 """Inherit user defined formatting from the root_logger.""" 690 # Use the the existing root logger formatter if one exists. 691 for handler in logging.getLogger().handlers: 692 if handler.formatter: 693 self.default_log_formatter = handler.formatter 694 break 695 696 formatter_mapping = { 697 self.root_logger.name: self.default_log_formatter, 698 } 699 if self.use_verbatim_error_log_formatting: 700 formatter_mapping[self.error_logger.name] = self.blank_log_formatter 701 702 self.dispatching_log_formatter = DispatchingFormatter( 703 formatter_mapping, 704 self.default_log_formatter, 705 ) 706 707 def should_use_progress_bars(self) -> bool: 708 if not self.allow_progress_bars: 709 return False 710 if self.separate_build_file_logging or self.default_logfile: 711 return True 712 return False 713 714 def use_stdout_proxy(self) -> None: 715 """Setup StdoutProxy for progress bars.""" 716 717 self.stdout_proxy = StdoutProxy(raw=True) 718 root_logger = logging.getLogger() 719 handlers = root_logger.handlers + self.error_logger.handlers 720 721 for handler in handlers: 722 # Must use type() check here since this returns True: 723 # isinstance(logging.FileHandler, logging.StreamHandler) 724 # pylint: disable=unidiomatic-typecheck 725 if type(handler) == logging.StreamHandler: 726 handler.setStream(self.stdout_proxy) # type: ignore 727 handler.setFormatter(self.dispatching_log_formatter) 728 # pylint: enable=unidiomatic-typecheck 729 730 def flush_log_handlers(self) -> None: 731 root_logger = logging.getLogger() 732 handlers = root_logger.handlers + self.error_logger.handlers 733 for cfg in self: 734 handlers.extend(cfg.log.handlers) 735 for handler in handlers: 736 handler.flush() 737 if self.stdout_proxy: 738 self.stdout_proxy.flush() 739 self.stdout_proxy.close() 740 741 def __len__(self) -> int: 742 return len(self.build_recipes) 743 744 def __getitem__(self, index: int) -> BuildRecipe: 745 return self.build_recipes[index] 746 747 def __iter__(self) -> Generator[BuildRecipe, None, None]: 748 return (build_recipe for build_recipe in self.build_recipes) 749 750 def run_build( 751 self, 752 cfg: BuildRecipe, 753 env: dict, 754 index_message: str | None = '', 755 ) -> bool: 756 """Run a single build config.""" 757 if BUILDER_CONTEXT.build_stopping(): 758 return False 759 760 if self.colors: 761 # Force colors in Pigweed subcommands run through the watcher. 762 env['PW_USE_COLOR'] = '1' 763 # Force Ninja to output ANSI colors 764 env['CLICOLOR_FORCE'] = '1' 765 766 build_succeeded = False 767 cfg.reset_status() 768 cfg.status.mark_started() 769 770 if cfg.auto_create_build_dir: 771 cfg.build_dir.mkdir(parents=True, exist_ok=True) 772 773 for command_step in cfg.steps: 774 command_args = command_step.get_args( 775 additional_ninja_args=self.extra_ninja_args, 776 additional_bazel_args=self.extra_bazel_args, 777 additional_bazel_build_args=self.extra_bazel_build_args, 778 ) 779 780 quoted_command_args = ' '.join( 781 shlex.quote(arg) for arg in command_args 782 ) 783 build_succeeded = True 784 if command_step.should_run(): 785 cfg.log.info( 786 '%s %s %s', 787 index_message, 788 self.color.blue('Run ==>'), 789 quoted_command_args, 790 ) 791 792 # Verify that the build output directories exist. 793 if ( 794 command_step.build_system_command is not None 795 and cfg.build_dir 796 and (not cfg.build_dir.is_dir()) 797 ): 798 self.abort_callback( 799 'Build directory does not exist: %s', cfg.build_dir 800 ) 801 802 build_succeeded = self.execute_command( 803 command_args, env, cfg, cfg.log, None 804 ) 805 else: 806 cfg.log.info( 807 '%s %s %s', 808 index_message, 809 self.color.yellow('Skipped ==>'), 810 quoted_command_args, 811 ) 812 813 BUILDER_CONTEXT.mark_progress_step_complete(cfg) 814 # Don't run further steps if a command fails. 815 if not build_succeeded: 816 break 817 818 # If all steps were skipped the return code will not be set. Force 819 # status to passed in this case. 820 if build_succeeded and not cfg.status.passed(): 821 cfg.status.set_passed() 822 823 cfg.status.mark_done() 824 825 return build_succeeded 826 827 def print_pass_fail_banner( 828 self, 829 cancelled: bool = False, 830 logger: logging.Logger = _LOG, 831 ) -> None: 832 # Check conditions where banners should not be shown: 833 # Banner flag disabled. 834 if not self.banners: 835 return 836 # If restarting or interrupted. 837 if BUILDER_CONTEXT.interrupted(): 838 if BUILDER_CONTEXT.ctrl_c_pressed: 839 _LOG.info( 840 self.color.yellow('Exited due to keyboard interrupt.') 841 ) 842 return 843 # If any build is still pending. 844 if any(recipe.status.pending() for recipe in self): 845 return 846 847 # Show a large color banner for the overall result. 848 if all(recipe.status.passed() for recipe in self) and not cancelled: 849 for line in PASS_MESSAGE.splitlines(): 850 logger.info(self.color.green(line)) 851 else: 852 for line in FAIL_MESSAGE.splitlines(): 853 logger.info(self.color.red(line)) 854 855 def print_build_summary( 856 self, 857 cancelled: bool = False, 858 logger: logging.Logger = _LOG, 859 ) -> None: 860 """Print build status summary table.""" 861 862 build_descriptions = [] 863 build_status = [] 864 865 for cfg in self: 866 description = [str(cfg.display_name).ljust(self.max_name_width)] 867 description.append(' '.join(cfg.targets())) 868 build_descriptions.append(' '.join(description)) 869 870 if cfg.status.passed(): 871 build_status.append(self.charset.slug_ok) 872 elif cfg.status.failed(): 873 build_status.append(self.charset.slug_fail) 874 else: 875 build_status.append(self.charset.slug_building) 876 877 if not cancelled: 878 logger.info(' ╔════════════════════════════════════') 879 logger.info(' ║') 880 881 for slug, cmd in zip(build_status, build_descriptions): 882 logger.info(' ║ %s %s', slug, cmd) 883 884 logger.info(' ║') 885 logger.info(" ╚════════════════════════════════════") 886 else: 887 # Build was interrupted. 888 logger.info('') 889 logger.info(' ╔════════════════════════════════════') 890 logger.info(' ║') 891 logger.info(' ║ %s- interrupted', self.charset.slug_fail) 892 logger.info(' ║') 893 logger.info(" ╚════════════════════════════════════") 894 895 896def run_recipe( 897 index: int, project_builder: ProjectBuilder, cfg: BuildRecipe, env 898) -> bool: 899 if BUILDER_CONTEXT.interrupted(): 900 return False 901 if not cfg.enabled: 902 return False 903 904 num_builds = len(project_builder) 905 index_message = f'[{index}/{num_builds}]' 906 907 result = False 908 909 log_build_recipe_start(index_message, project_builder, cfg) 910 911 result = project_builder.run_build(cfg, env, index_message=index_message) 912 913 log_build_recipe_finish(index_message, project_builder, cfg) 914 915 return result 916 917 918def run_builds(project_builder: ProjectBuilder, workers: int = 1) -> int: 919 """Execute all build recipe steps. 920 921 Args: 922 project_builder: A ProjectBuilder instance 923 workers: The number of build recipes that should be run in 924 parallel. Defaults to 1 or no parallel execution. 925 926 Returns: 927 1 for a failed build, 0 for success. 928 """ 929 num_builds = len(project_builder) 930 _LOG.info('Starting build with %d directories', num_builds) 931 if project_builder.default_logfile: 932 _LOG.info( 933 '%s %s', 934 project_builder.color.blue('Root logfile:'), 935 project_builder.default_logfile.resolve(), 936 ) 937 938 env = os.environ.copy() 939 940 # Print status before starting 941 if not project_builder.should_use_progress_bars(): 942 project_builder.print_build_summary() 943 project_builder.print_pass_fail_banner() 944 945 if workers > 1 and not project_builder.separate_build_file_logging: 946 _LOG.warning( 947 project_builder.color.yellow( 948 'Running in parallel without --separate-logfiles; All build ' 949 'output will be interleaved.' 950 ) 951 ) 952 953 BUILDER_CONTEXT.set_project_builder(project_builder) 954 BUILDER_CONTEXT.set_building() 955 956 def _cleanup() -> None: 957 if not project_builder.should_use_progress_bars(): 958 project_builder.print_build_summary() 959 project_builder.print_pass_fail_banner() 960 project_builder.flush_log_handlers() 961 BUILDER_CONTEXT.set_idle() 962 BUILDER_CONTEXT.exit_progress() 963 964 if workers == 1: 965 # TODO(tonymd): Try to remove this special case. Using 966 # ThreadPoolExecutor when running in serial (workers==1) currently 967 # breaks Ctrl-C handling. Build processes keep running. 968 try: 969 if project_builder.should_use_progress_bars(): 970 BUILDER_CONTEXT.add_progress_bars() 971 for i, cfg in enumerate(project_builder, start=1): 972 run_recipe(i, project_builder, cfg, env) 973 # Ctrl-C on Unix generates KeyboardInterrupt 974 # Ctrl-Z on Windows generates EOFError 975 except (KeyboardInterrupt, EOFError): 976 _exit_due_to_interrupt() 977 finally: 978 _cleanup() 979 980 else: 981 with concurrent.futures.ThreadPoolExecutor( 982 max_workers=workers 983 ) as executor: 984 futures = [] 985 for i, cfg in enumerate(project_builder, start=1): 986 futures.append( 987 executor.submit(run_recipe, i, project_builder, cfg, env) 988 ) 989 990 try: 991 if project_builder.should_use_progress_bars(): 992 BUILDER_CONTEXT.add_progress_bars() 993 for future in concurrent.futures.as_completed(futures): 994 future.result() 995 # Ctrl-C on Unix generates KeyboardInterrupt 996 # Ctrl-Z on Windows generates EOFError 997 except (KeyboardInterrupt, EOFError): 998 _exit_due_to_interrupt() 999 finally: 1000 _cleanup() 1001 1002 project_builder.flush_log_handlers() 1003 return BUILDER_CONTEXT.exit_code() 1004 1005 1006def main() -> int: 1007 """Build a Pigweed Project.""" 1008 parser = argparse.ArgumentParser( 1009 description=__doc__, 1010 formatter_class=argparse.RawDescriptionHelpFormatter, 1011 ) 1012 parser = add_project_builder_arguments(parser) 1013 args = parser.parse_args() 1014 1015 pw_env = pw_cli.env.pigweed_environment() 1016 if pw_env.PW_EMOJI: 1017 charset = EMOJI_CHARSET 1018 else: 1019 charset = ASCII_CHARSET 1020 1021 prefs = ProjectBuilderPrefs( 1022 load_argparse_arguments=add_project_builder_arguments 1023 ) 1024 prefs.apply_command_line_args(args) 1025 build_recipes = create_build_recipes(prefs) 1026 1027 log_level = logging.DEBUG if args.debug_logging else logging.INFO 1028 1029 pw_cli.log.install( 1030 level=log_level, 1031 use_color=args.colors, 1032 hide_timestamp=False, 1033 ) 1034 1035 project_builder = ProjectBuilder( 1036 build_recipes=build_recipes, 1037 jobs=args.jobs, 1038 banners=args.banners, 1039 keep_going=args.keep_going, 1040 colors=args.colors, 1041 charset=charset, 1042 separate_build_file_logging=args.separate_logfiles, 1043 root_logfile=args.logfile, 1044 root_logger=_LOG, 1045 log_level=log_level, 1046 ) 1047 1048 if project_builder.should_use_progress_bars(): 1049 project_builder.use_stdout_proxy() 1050 1051 workers = 1 1052 if args.parallel: 1053 # If parallel is requested and parallel_workers is set to 0 run all 1054 # recipes in parallel. That is, use the number of recipes as the worker 1055 # count. 1056 if args.parallel_workers == 0: 1057 workers = len(project_builder) 1058 else: 1059 workers = args.parallel_workers 1060 1061 return run_builds(project_builder, workers) 1062 1063 1064if __name__ == '__main__': 1065 sys.exit(main()) 1066