1#!/usr/bin/env python 2# Copyright 2020 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15"""Watch files for changes and rebuild. 16 17Run arbitrary commands or invoke build systems (Ninja, Bazel and make) on one or 18more build directories whenever source files change. 19 20Examples: 21 22 # Build the default target in out/ using ninja. 23 pw watch -C out 24 25 # Build python.lint and stm32f429i targets in out/ using ninja. 26 pw watch python.lint stm32f429i 27 28 # Build pw_run_tests.modules in the out/cmake directory 29 pw watch -C out/cmake pw_run_tests.modules 30 31 # Build the default target in out/ and pw_apps in out/cmake 32 pw watch -C out -C out/cmake pw_apps 33 34 # Build python.tests in out/ and pw_apps in out/cmake/ 35 pw watch python.tests -C out/cmake pw_apps 36 37 # Run 'bazel build' and 'bazel test' on the target '//...' in outbazel/ 38 pw watch --run-command 'mkdir -p outbazel' 39 -C outbazel '//...' 40 --build-system-command outbazel 'bazel build' 41 --build-system-command outbazel 'bazel test' 42""" 43 44import argparse 45import concurrent.futures 46import errno 47import http.server 48import logging 49import os 50from pathlib import Path 51import re 52import subprocess 53import socketserver 54import sys 55import threading 56from threading import Thread 57from typing import ( 58 Callable, 59 Iterable, 60 List, 61 NoReturn, 62 Optional, 63 Sequence, 64 Tuple, 65) 66 67try: 68 import httpwatcher # type: ignore[import] 69except ImportError: 70 httpwatcher = None 71 72from watchdog.events import FileSystemEventHandler # type: ignore[import] 73from watchdog.observers import Observer # type: ignore[import] 74 75from prompt_toolkit import prompt 76 77from pw_build.build_recipe import BuildRecipe, create_build_recipes 78from pw_build.project_builder import ( 79 ProjectBuilder, 80 execute_command_no_logging, 81 execute_command_with_logging, 82 log_build_recipe_start, 83 log_build_recipe_finish, 84 ASCII_CHARSET, 85 EMOJI_CHARSET, 86) 87from pw_build.project_builder_context import get_project_builder_context 88import pw_cli.branding 89import pw_cli.color 90import pw_cli.env 91import pw_cli.log 92import pw_cli.plugins 93import pw_console.python_logging 94 95from pw_watch.argparser import ( 96 WATCH_PATTERN_DELIMITER, 97 WATCH_PATTERNS, 98 add_parser_arguments, 99) 100from pw_watch.debounce import DebouncedFunction, Debouncer 101from pw_watch.watch_app import WatchAppPrefs, WatchApp 102 103_COLOR = pw_cli.color.colors() 104_LOG = logging.getLogger('pw_build.watch') 105_ERRNO_INOTIFY_LIMIT_REACHED = 28 106 107# Suppress events under 'fsevents', generated by watchdog on every file 108# event on MacOS. 109# TODO(b/182281481): Fix file ignoring, rather than just suppressing logs 110_FSEVENTS_LOG = logging.getLogger('fsevents') 111_FSEVENTS_LOG.setLevel(logging.WARNING) 112 113_FULLSCREEN_STATUS_COLUMN_WIDTH = 10 114 115BUILDER_CONTEXT = get_project_builder_context() 116 117 118def git_ignored(file: Path) -> bool: 119 """Returns true if this file is in a Git repo and ignored by that repo. 120 121 Returns true for ignored files that were manually added to a repo. 122 """ 123 file = file.resolve() 124 directory = file.parent 125 126 # Run the Git command from file's parent so that the correct repo is used. 127 while True: 128 try: 129 returncode = subprocess.run( 130 ['git', 'check-ignore', '--quiet', '--no-index', file], 131 stdout=subprocess.DEVNULL, 132 stderr=subprocess.DEVNULL, 133 cwd=directory, 134 ).returncode 135 return returncode in (0, 128) 136 except FileNotFoundError: 137 # If the directory no longer exists, try parent directories until 138 # an existing directory is found or all directories have been 139 # checked. This approach makes it possible to check if a deleted 140 # path is ignored in the repo it was originally created in. 141 if directory == directory.parent: 142 return False 143 144 directory = directory.parent 145 146 147class PigweedBuildWatcher(FileSystemEventHandler, DebouncedFunction): 148 """Process filesystem events and launch builds if necessary.""" 149 150 # pylint: disable=too-many-instance-attributes 151 NINJA_BUILD_STEP = re.compile( 152 r'^\[(?P<step>[0-9]+)/(?P<total_steps>[0-9]+)\] (?P<action>.*)$' 153 ) 154 155 def __init__( # pylint: disable=too-many-arguments 156 self, 157 project_builder: ProjectBuilder, 158 patterns: Sequence[str] = (), 159 ignore_patterns: Sequence[str] = (), 160 restart: bool = True, 161 fullscreen: bool = False, 162 banners: bool = True, 163 use_logfile: bool = False, 164 separate_logfiles: bool = False, 165 parallel_workers: int = 1, 166 ): 167 super().__init__() 168 169 self.banners = banners 170 self.current_build_step = '' 171 self.current_build_percent = 0.0 172 self.current_build_errors = 0 173 self.patterns = patterns 174 self.ignore_patterns = ignore_patterns 175 self.project_builder = project_builder 176 self.parallel_workers = parallel_workers 177 178 self.restart_on_changes = restart 179 self.fullscreen_enabled = fullscreen 180 self.watch_app: Optional[WatchApp] = None 181 182 self.use_logfile = use_logfile 183 self.separate_logfiles = separate_logfiles 184 if self.parallel_workers > 1: 185 self.separate_logfiles = True 186 187 self.debouncer = Debouncer(self) 188 189 # Track state of a build. These need to be members instead of locals 190 # due to the split between dispatch(), run(), and on_complete(). 191 self.matching_path: Optional[Path] = None 192 193 if ( 194 not self.fullscreen_enabled 195 and not self.project_builder.should_use_progress_bars() 196 ): 197 self.wait_for_keypress_thread = threading.Thread( 198 None, self._wait_for_enter 199 ) 200 self.wait_for_keypress_thread.start() 201 202 if self.fullscreen_enabled: 203 BUILDER_CONTEXT.using_fullscreen = True 204 205 def rebuild(self): 206 """Rebuild command triggered from watch app.""" 207 self.debouncer.press('Manual build requested') 208 209 def _wait_for_enter(self) -> None: 210 try: 211 while True: 212 _ = prompt('') 213 self.rebuild() 214 # Ctrl-C on Unix generates KeyboardInterrupt 215 # Ctrl-Z on Windows generates EOFError 216 except (KeyboardInterrupt, EOFError): 217 # Force stop any running ninja builds. 218 _exit_due_to_interrupt() 219 220 def _path_matches(self, path: Path) -> bool: 221 """Returns true if path matches according to the watcher patterns""" 222 return not any(path.match(x) for x in self.ignore_patterns) and any( 223 path.match(x) for x in self.patterns 224 ) 225 226 def dispatch(self, event) -> None: 227 # There isn't any point in triggering builds on new directory creation. 228 # It's the creation or modification of files that indicate something 229 # meaningful enough changed for a build. 230 if event.is_directory: 231 return 232 233 # Collect paths of interest from the event. 234 paths: List[str] = [] 235 if hasattr(event, 'dest_path'): 236 paths.append(os.fsdecode(event.dest_path)) 237 if event.src_path: 238 paths.append(os.fsdecode(event.src_path)) 239 for raw_path in paths: 240 _LOG.debug('File event: %s', raw_path) 241 242 # Check whether Git cares about any of these paths. 243 for path in (Path(p).resolve() for p in paths): 244 if not git_ignored(path) and self._path_matches(path): 245 self._handle_matched_event(path) 246 return 247 248 def _handle_matched_event(self, matching_path: Path) -> None: 249 if self.matching_path is None: 250 self.matching_path = matching_path 251 252 log_message = f'File change detected: {os.path.relpath(matching_path)}' 253 if self.restart_on_changes: 254 if self.fullscreen_enabled and self.watch_app: 255 self.watch_app.clear_log_panes() 256 self.debouncer.press(f'{log_message} Triggering build...') 257 else: 258 _LOG.info('%s ; not rebuilding', log_message) 259 260 def _clear_screen(self) -> None: 261 if self.fullscreen_enabled: 262 return 263 if self.project_builder.should_use_progress_bars(): 264 BUILDER_CONTEXT.clear_progress_scrollback() 265 return 266 print('\033c', end='') # TODO(pwbug/38): Not Windows compatible. 267 sys.stdout.flush() 268 269 # Implementation of DebouncedFunction.run() 270 # 271 # Note: This will run on the timer thread created by the Debouncer, rather 272 # than on the main thread that's watching file events. This enables the 273 # watcher to continue receiving file change events during a build. 274 def run(self) -> None: 275 """Run all the builds and capture pass/fail for each.""" 276 277 # Clear the screen and show a banner indicating the build is starting. 278 self._clear_screen() 279 280 if self.banners: 281 for line in pw_cli.branding.banner().splitlines(): 282 _LOG.info(line) 283 if self.fullscreen_enabled: 284 _LOG.info( 285 self.project_builder.color.green( 286 'Watching for changes. Ctrl-d to exit; enter to rebuild' 287 ) 288 ) 289 else: 290 _LOG.info( 291 self.project_builder.color.green( 292 'Watching for changes. Ctrl-C to exit; enter to rebuild' 293 ) 294 ) 295 if self.matching_path: 296 _LOG.info('') 297 _LOG.info('Change detected: %s', self.matching_path) 298 299 num_builds = len(self.project_builder) 300 _LOG.info('Starting build with %d directories', num_builds) 301 302 if self.project_builder.default_logfile: 303 _LOG.info( 304 '%s %s', 305 self.project_builder.color.blue('Root logfile:'), 306 self.project_builder.default_logfile.resolve(), 307 ) 308 309 env = os.environ.copy() 310 if self.project_builder.colors: 311 # Force colors in Pigweed subcommands run through the watcher. 312 env['PW_USE_COLOR'] = '1' 313 # Force Ninja to output ANSI colors 314 env['CLICOLOR_FORCE'] = '1' 315 316 # Reset status 317 BUILDER_CONTEXT.set_project_builder(self.project_builder) 318 BUILDER_CONTEXT.set_enter_callback(self.rebuild) 319 BUILDER_CONTEXT.set_building() 320 321 for cfg in self.project_builder: 322 cfg.reset_status() 323 324 with concurrent.futures.ThreadPoolExecutor( 325 max_workers=self.parallel_workers 326 ) as executor: 327 futures = [] 328 if ( 329 not self.fullscreen_enabled 330 and self.project_builder.should_use_progress_bars() 331 ): 332 BUILDER_CONTEXT.add_progress_bars() 333 334 for i, cfg in enumerate(self.project_builder, start=1): 335 futures.append(executor.submit(self.run_recipe, i, cfg, env)) 336 337 for future in concurrent.futures.as_completed(futures): 338 future.result() 339 340 BUILDER_CONTEXT.set_idle() 341 342 def run_recipe(self, index: int, cfg: BuildRecipe, env) -> None: 343 if BUILDER_CONTEXT.interrupted(): 344 return 345 if not cfg.enabled: 346 return 347 348 num_builds = len(self.project_builder) 349 index_message = f'[{index}/{num_builds}]' 350 351 log_build_recipe_start( 352 index_message, self.project_builder, cfg, logger=_LOG 353 ) 354 355 self.project_builder.run_build( 356 cfg, 357 env, 358 index_message=index_message, 359 ) 360 361 log_build_recipe_finish( 362 index_message, 363 self.project_builder, 364 cfg, 365 logger=_LOG, 366 ) 367 368 def execute_command( 369 self, 370 command: list, 371 env: dict, 372 recipe: BuildRecipe, 373 # pylint: disable=unused-argument 374 *args, 375 **kwargs, 376 # pylint: enable=unused-argument 377 ) -> bool: 378 """Runs a command with a blank before/after for visual separation.""" 379 if self.fullscreen_enabled: 380 return self._execute_command_watch_app(command, env, recipe) 381 382 if self.separate_logfiles: 383 return execute_command_with_logging( 384 command, env, recipe, logger=recipe.log 385 ) 386 387 if self.use_logfile: 388 return execute_command_with_logging( 389 command, env, recipe, logger=_LOG 390 ) 391 392 return execute_command_no_logging(command, env, recipe) 393 394 def _execute_command_watch_app( 395 self, 396 command: list, 397 env: dict, 398 recipe: BuildRecipe, 399 ) -> bool: 400 """Runs a command with and outputs the logs.""" 401 if not self.watch_app: 402 return False 403 404 self.watch_app.redraw_ui() 405 406 def new_line_callback(recipe: BuildRecipe) -> None: 407 self.current_build_step = recipe.status.current_step 408 self.current_build_percent = recipe.status.percent 409 self.current_build_errors = recipe.status.error_count 410 411 if self.watch_app: 412 self.watch_app.redraw_ui() 413 414 desired_logger = _LOG 415 if self.separate_logfiles: 416 desired_logger = recipe.log 417 418 result = execute_command_with_logging( 419 command, 420 env, 421 recipe, 422 logger=desired_logger, 423 line_processed_callback=new_line_callback, 424 ) 425 426 self.watch_app.redraw_ui() 427 428 return result 429 430 # Implementation of DebouncedFunction.cancel() 431 def cancel(self) -> bool: 432 if self.restart_on_changes: 433 BUILDER_CONTEXT.restart_flag = True 434 BUILDER_CONTEXT.terminate_and_wait() 435 return True 436 437 return False 438 439 # Implementation of DebouncedFunction.on_complete() 440 def on_complete(self, cancelled: bool = False) -> None: 441 # First, use the standard logging facilities to report build status. 442 if cancelled: 443 _LOG.info('Build stopped.') 444 elif BUILDER_CONTEXT.interrupted(): 445 pass # Don't print anything. 446 elif all( 447 recipe.status.passed() 448 for recipe in self.project_builder 449 if recipe.enabled 450 ): 451 _LOG.info('Finished; all successful') 452 else: 453 _LOG.info('Finished; some builds failed') 454 455 # For non-fullscreen pw watch 456 if ( 457 not self.fullscreen_enabled 458 and not self.project_builder.should_use_progress_bars() 459 ): 460 # Show a more distinct colored banner. 461 self.project_builder.print_build_summary( 462 cancelled=cancelled, logger=_LOG 463 ) 464 self.project_builder.print_pass_fail_banner( 465 cancelled=cancelled, logger=_LOG 466 ) 467 468 if self.watch_app: 469 self.watch_app.redraw_ui() 470 self.matching_path = None 471 472 # Implementation of DebouncedFunction.on_keyboard_interrupt() 473 def on_keyboard_interrupt(self) -> None: 474 _exit_due_to_interrupt() 475 476 477def _exit(code: int) -> NoReturn: 478 # Flush all log handlers 479 logging.shutdown() 480 # Note: The "proper" way to exit is via observer.stop(), then 481 # running a join. However it's slower, so just exit immediately. 482 # 483 # Additionally, since there are several threads in the watcher, the usual 484 # sys.exit approach doesn't work. Instead, run the low level exit which 485 # kills all threads. 486 os._exit(code) # pylint: disable=protected-access 487 488 489def _exit_due_to_interrupt() -> None: 490 # To keep the log lines aligned with each other in the presence of 491 # a '^C' from the keyboard interrupt, add a newline before the log. 492 print('') 493 _LOG.info('Got Ctrl-C; exiting...') 494 BUILDER_CONTEXT.ctrl_c_interrupt() 495 496 497def _log_inotify_watch_limit_reached(): 498 # Show information and suggested commands in OSError: inotify limit reached. 499 _LOG.error( 500 'Inotify watch limit reached: run this in your terminal if ' 501 'you are in Linux to temporarily increase inotify limit.' 502 ) 503 _LOG.info('') 504 _LOG.info( 505 _COLOR.green( 506 ' sudo sysctl fs.inotify.max_user_watches=' '$NEW_LIMIT$' 507 ) 508 ) 509 _LOG.info('') 510 _LOG.info( 511 ' Change $NEW_LIMIT$ with an integer number, ' 512 'e.g., 20000 should be enough.' 513 ) 514 515 516def _exit_due_to_inotify_watch_limit(): 517 _log_inotify_watch_limit_reached() 518 _exit(1) 519 520 521def _log_inotify_instance_limit_reached(): 522 # Show information and suggested commands in OSError: inotify limit reached. 523 _LOG.error( 524 'Inotify instance limit reached: run this in your terminal if ' 525 'you are in Linux to temporarily increase inotify limit.' 526 ) 527 _LOG.info('') 528 _LOG.info( 529 _COLOR.green( 530 ' sudo sysctl fs.inotify.max_user_instances=' '$NEW_LIMIT$' 531 ) 532 ) 533 _LOG.info('') 534 _LOG.info( 535 ' Change $NEW_LIMIT$ with an integer number, ' 536 'e.g., 20000 should be enough.' 537 ) 538 539 540def _exit_due_to_inotify_instance_limit(): 541 _log_inotify_instance_limit_reached() 542 _exit(1) 543 544 545def _exit_due_to_pigweed_not_installed(): 546 # Show information and suggested commands when pigweed environment variable 547 # not found. 548 _LOG.error( 549 'Environment variable $PW_ROOT not defined or is defined ' 550 'outside the current directory.' 551 ) 552 _LOG.error( 553 'Did you forget to activate the Pigweed environment? ' 554 'Try source ./activate.sh' 555 ) 556 _LOG.error( 557 'Did you forget to install the Pigweed environment? ' 558 'Try source ./bootstrap.sh' 559 ) 560 _exit(1) 561 562 563# Go over each directory inside of the current directory. 564# If it is not on the path of elements in directories_to_exclude, add 565# (directory, True) to subdirectories_to_watch and later recursively call 566# Observer() on them. 567# Otherwise add (directory, False) to subdirectories_to_watch and later call 568# Observer() with recursion=False. 569def minimal_watch_directories(to_watch: Path, to_exclude: Iterable[Path]): 570 """Determine which subdirectory to watch recursively""" 571 try: 572 to_watch = Path(to_watch) 573 except TypeError: 574 assert False, "Please watch one directory at a time." 575 576 # Reformat to_exclude. 577 directories_to_exclude: List[Path] = [ 578 to_watch.joinpath(directory_to_exclude) 579 for directory_to_exclude in to_exclude 580 if to_watch.joinpath(directory_to_exclude).is_dir() 581 ] 582 583 # Split the relative path of directories_to_exclude (compared to to_watch), 584 # and generate all parent paths needed to be watched without recursion. 585 exclude_dir_parents = {to_watch} 586 for directory_to_exclude in directories_to_exclude: 587 parts = list(Path(directory_to_exclude).relative_to(to_watch).parts)[ 588 :-1 589 ] 590 dir_tmp = to_watch 591 for part in parts: 592 dir_tmp = Path(dir_tmp, part) 593 exclude_dir_parents.add(dir_tmp) 594 595 # Go over all layers of directory. Append those that are the parents of 596 # directories_to_exclude to the list with recursion==False, and others 597 # with recursion==True. 598 for directory in exclude_dir_parents: 599 dir_path = Path(directory) 600 yield dir_path, False 601 for item in Path(directory).iterdir(): 602 if ( 603 item.is_dir() 604 and item not in exclude_dir_parents 605 and item not in directories_to_exclude 606 ): 607 yield item, True 608 609 610def get_common_excludes() -> List[Path]: 611 """Find commonly excluded directories, and return them as a [Path]""" 612 exclude_list: List[Path] = [] 613 614 typical_ignored_directories: List[str] = [ 615 '.environment', # Legacy bootstrap-created CIPD and Python venv. 616 '.presubmit', # Presubmit-created CIPD and Python venv. 617 '.git', # Pigweed's git repo. 618 '.mypy_cache', # Python static analyzer. 619 '.cargo', # Rust package manager. 620 'environment', # Bootstrap-created CIPD and Python venv. 621 'out', # Typical build directory. 622 ] 623 624 # Preset exclude list for Pigweed's upstream directories. 625 pw_root_dir = Path(os.environ['PW_ROOT']) 626 exclude_list.extend( 627 pw_root_dir / ignored_directory 628 for ignored_directory in typical_ignored_directories 629 ) 630 631 # Preset exclude for common downstream project structures. 632 # 633 # If watch is invoked outside of the Pigweed root, exclude common 634 # directories. 635 pw_project_root_dir = Path(os.environ['PW_PROJECT_ROOT']) 636 if pw_project_root_dir != pw_root_dir: 637 exclude_list.extend( 638 pw_project_root_dir / ignored_directory 639 for ignored_directory in typical_ignored_directories 640 ) 641 642 # Check for and warn about legacy directories. 643 legacy_directories = [ 644 '.cipd', # Legacy CIPD location. 645 '.python3-venv', # Legacy Python venv location. 646 ] 647 found_legacy = False 648 for legacy_directory in legacy_directories: 649 full_legacy_directory = pw_root_dir / legacy_directory 650 if full_legacy_directory.is_dir(): 651 _LOG.warning( 652 'Legacy environment directory found: %s', 653 str(full_legacy_directory), 654 ) 655 exclude_list.append(full_legacy_directory) 656 found_legacy = True 657 if found_legacy: 658 _LOG.warning( 659 'Found legacy environment directory(s); these ' 'should be deleted' 660 ) 661 662 return exclude_list 663 664 665def _simple_docs_server( 666 address: str, port: int, path: Path 667) -> Callable[[], None]: 668 class Handler(http.server.SimpleHTTPRequestHandler): 669 def __init__(self, *args, **kwargs): 670 super().__init__(*args, directory=path, **kwargs) 671 672 # Disable logs to stdout 673 def log_message( 674 self, format: str, *args # pylint: disable=redefined-builtin 675 ) -> None: 676 return 677 678 def simple_http_server_thread(): 679 with socketserver.TCPServer((address, port), Handler) as httpd: 680 httpd.serve_forever() 681 682 return simple_http_server_thread 683 684 685def _httpwatcher_docs_server( 686 address: str, port: int, path: Path 687) -> Callable[[], None]: 688 def httpwatcher_thread(): 689 # Disable logs from httpwatcher and deps 690 logging.getLogger('httpwatcher').setLevel(logging.CRITICAL) 691 logging.getLogger('tornado').setLevel(logging.CRITICAL) 692 693 httpwatcher.watch(path, host=address, port=port) 694 695 return httpwatcher_thread 696 697 698def _serve_docs( 699 build_dir: Path, 700 docs_path: Path, 701 address: str = '127.0.0.1', 702 port: int = 8000, 703) -> None: 704 address = '127.0.0.1' 705 docs_path = build_dir.joinpath(docs_path.joinpath('html')) 706 707 if httpwatcher is not None: 708 _LOG.info('Using httpwatcher. Docs will reload when changed.') 709 server_thread = _httpwatcher_docs_server(address, port, docs_path) 710 else: 711 _LOG.info( 712 'Using simple HTTP server. Docs will not reload when changed.' 713 ) 714 _LOG.info('Install httpwatcher and restart for automatic docs reload.') 715 server_thread = _simple_docs_server(address, port, docs_path) 716 717 # Spin up server in a new thread since it blocks 718 threading.Thread(None, server_thread, 'pw_docs_server').start() 719 720 721def watch_logging_init(log_level: int, fullscreen: bool, colors: bool) -> None: 722 # Logging setup 723 if not fullscreen: 724 pw_cli.log.install( 725 level=log_level, 726 use_color=colors, 727 hide_timestamp=False, 728 ) 729 return 730 731 watch_logfile = pw_console.python_logging.create_temp_log_file( 732 prefix=__package__ 733 ) 734 735 pw_cli.log.install( 736 level=logging.DEBUG, 737 use_color=colors, 738 hide_timestamp=False, 739 log_file=watch_logfile, 740 ) 741 742 743def watch_setup( # pylint: disable=too-many-locals 744 project_builder: ProjectBuilder, 745 # NOTE: The following args should have defaults matching argparse. This 746 # allows use of watch_setup by other project build scripts. 747 patterns: str = WATCH_PATTERN_DELIMITER.join(WATCH_PATTERNS), 748 ignore_patterns_string: str = '', 749 exclude_list: Optional[List[Path]] = None, 750 restart: bool = True, 751 serve_docs: bool = False, 752 serve_docs_port: int = 8000, 753 serve_docs_path: Path = Path('docs/gen/docs'), 754 fullscreen: bool = False, 755 banners: bool = True, 756 logfile: Optional[Path] = None, 757 separate_logfiles: bool = False, 758 parallel: bool = False, 759 parallel_workers: int = 0, 760 # pylint: disable=unused-argument 761 default_build_targets: Optional[List[str]] = None, 762 build_directories: Optional[List[str]] = None, 763 build_system_commands: Optional[List[str]] = None, 764 run_command: Optional[List[str]] = None, 765 jobs: Optional[int] = None, 766 keep_going: bool = False, 767 colors: bool = True, 768 debug_logging: bool = False, 769 # pylint: enable=unused-argument 770 # pylint: disable=too-many-arguments 771) -> Tuple[PigweedBuildWatcher, List[Path]]: 772 """Watches files and runs Ninja commands when they change.""" 773 watch_logging_init( 774 log_level=project_builder.default_log_level, 775 fullscreen=fullscreen, 776 colors=colors, 777 ) 778 779 # Update the project_builder log formatters since pw_cli.log.install may 780 # have changed it. 781 project_builder.apply_root_log_formatting() 782 783 if project_builder.should_use_progress_bars(): 784 project_builder.use_stdout_proxy() 785 786 _LOG.info('Starting Pigweed build watcher') 787 788 # Get pigweed directory information from environment variable PW_ROOT. 789 if os.environ['PW_ROOT'] is None: 790 _exit_due_to_pigweed_not_installed() 791 pw_root = Path(os.environ['PW_ROOT']).resolve() 792 if Path.cwd().resolve() not in [pw_root, *pw_root.parents]: 793 _exit_due_to_pigweed_not_installed() 794 795 build_recipes = project_builder.build_recipes 796 797 # Preset exclude list for pigweed directory. 798 if not exclude_list: 799 exclude_list = [] 800 exclude_list += get_common_excludes() 801 # Add build directories to the exclude list. 802 exclude_list.extend( 803 cfg.build_dir.resolve() 804 for cfg in build_recipes 805 if isinstance(cfg.build_dir, Path) 806 ) 807 808 for i, build_recipe in enumerate(build_recipes, start=1): 809 _LOG.info('Will build [%d/%d]: %s', i, len(build_recipes), build_recipe) 810 811 _LOG.debug('Patterns: %s', patterns) 812 813 if serve_docs: 814 _serve_docs( 815 build_recipes[0].build_dir, serve_docs_path, port=serve_docs_port 816 ) 817 818 # Ignore the user-specified patterns. 819 ignore_patterns = ( 820 ignore_patterns_string.split(WATCH_PATTERN_DELIMITER) 821 if ignore_patterns_string 822 else [] 823 ) 824 825 # Add project_builder logfiles to ignore_patterns 826 if project_builder.default_logfile: 827 ignore_patterns.append(str(project_builder.default_logfile)) 828 if project_builder.separate_build_file_logging: 829 for recipe in project_builder: 830 if recipe.logfile: 831 ignore_patterns.append(str(recipe.logfile)) 832 833 workers = 1 834 if parallel: 835 # If parallel is requested and parallel_workers is set to 0 run all 836 # recipes in parallel. That is, use the number of recipes as the worker 837 # count. 838 if parallel_workers == 0: 839 workers = len(project_builder) 840 else: 841 workers = parallel_workers 842 843 event_handler = PigweedBuildWatcher( 844 project_builder=project_builder, 845 patterns=patterns.split(WATCH_PATTERN_DELIMITER), 846 ignore_patterns=ignore_patterns, 847 restart=restart, 848 fullscreen=fullscreen, 849 banners=banners, 850 use_logfile=bool(logfile), 851 separate_logfiles=separate_logfiles, 852 parallel_workers=workers, 853 ) 854 855 project_builder.execute_command = event_handler.execute_command 856 857 return event_handler, exclude_list 858 859 860def watch( 861 event_handler: PigweedBuildWatcher, 862 exclude_list: List[Path], 863): 864 """Watches files and runs Ninja commands when they change.""" 865 # Try to make a short display path for the watched directory that has 866 # "$HOME" instead of the full home directory. This is nice for users 867 # who have deeply nested home directories. 868 path_to_log = str(Path().resolve()).replace(str(Path.home()), '$HOME') 869 870 try: 871 # It can take awhile to configure the filesystem watcher, so have the 872 # message reflect that with the "...". Run inside the try: to 873 # gracefully handle the user Ctrl-C'ing out during startup. 874 875 _LOG.info('Attaching filesystem watcher to %s/...', path_to_log) 876 877 # Observe changes for all files in the root directory. Whether the 878 # directory should be observed recursively or not is determined by the 879 # second element in subdirectories_to_watch. 880 observers = [] 881 for path, rec in minimal_watch_directories(Path.cwd(), exclude_list): 882 observer = Observer() 883 observer.schedule( 884 event_handler, 885 str(path), 886 recursive=rec, 887 ) 888 observer.start() 889 observers.append(observer) 890 891 event_handler.debouncer.press('Triggering initial build...') 892 for observer in observers: 893 while observer.is_alive(): 894 observer.join(1) 895 _LOG.error('observers joined') 896 897 # Ctrl-C on Unix generates KeyboardInterrupt 898 # Ctrl-Z on Windows generates EOFError 899 except (KeyboardInterrupt, EOFError): 900 _exit_due_to_interrupt() 901 except OSError as err: 902 if err.args[0] == _ERRNO_INOTIFY_LIMIT_REACHED: 903 if event_handler.watch_app: 904 event_handler.watch_app.exit( 905 log_after_shutdown=_log_inotify_watch_limit_reached 906 ) 907 elif event_handler.project_builder.should_use_progress_bars(): 908 BUILDER_CONTEXT.exit( 909 log_after_shutdown=_log_inotify_watch_limit_reached, 910 ) 911 else: 912 _exit_due_to_inotify_watch_limit() 913 if err.errno == errno.EMFILE: 914 if event_handler.watch_app: 915 event_handler.watch_app.exit( 916 log_after_shutdown=_log_inotify_instance_limit_reached 917 ) 918 elif event_handler.project_builder.should_use_progress_bars(): 919 BUILDER_CONTEXT.exit( 920 log_after_shutdown=_log_inotify_instance_limit_reached 921 ) 922 else: 923 _exit_due_to_inotify_instance_limit() 924 raise err 925 926 927def run_watch( 928 event_handler: PigweedBuildWatcher, 929 exclude_list: List[Path], 930 prefs: Optional[WatchAppPrefs] = None, 931 fullscreen: bool = False, 932) -> None: 933 """Start pw_watch.""" 934 if not prefs: 935 prefs = WatchAppPrefs(load_argparse_arguments=add_parser_arguments) 936 937 if fullscreen: 938 watch_thread = Thread( 939 target=watch, 940 args=(event_handler, exclude_list), 941 daemon=True, 942 ) 943 watch_thread.start() 944 watch_app = WatchApp( 945 event_handler=event_handler, 946 prefs=prefs, 947 ) 948 949 event_handler.watch_app = watch_app 950 watch_app.run() 951 952 else: 953 watch(event_handler, exclude_list) 954 955 956def get_parser() -> argparse.ArgumentParser: 957 parser = argparse.ArgumentParser( 958 description=__doc__, 959 formatter_class=argparse.RawDescriptionHelpFormatter, 960 ) 961 parser = add_parser_arguments(parser) 962 return parser 963 964 965def main() -> None: 966 """Watch files for changes and rebuild.""" 967 parser = get_parser() 968 args = parser.parse_args() 969 970 prefs = WatchAppPrefs(load_argparse_arguments=add_parser_arguments) 971 prefs.apply_command_line_args(args) 972 build_recipes = create_build_recipes(prefs) 973 974 env = pw_cli.env.pigweed_environment() 975 if env.PW_EMOJI: 976 charset = EMOJI_CHARSET 977 else: 978 charset = ASCII_CHARSET 979 980 # Force separate-logfiles for split window panes if running in parallel. 981 separate_logfiles = args.separate_logfiles 982 if args.parallel: 983 separate_logfiles = True 984 985 def _recipe_abort(*args) -> None: 986 _LOG.critical(*args) 987 988 project_builder = ProjectBuilder( 989 build_recipes=build_recipes, 990 jobs=args.jobs, 991 banners=args.banners, 992 keep_going=args.keep_going, 993 colors=args.colors, 994 charset=charset, 995 separate_build_file_logging=separate_logfiles, 996 root_logfile=args.logfile, 997 root_logger=_LOG, 998 log_level=logging.DEBUG if args.debug_logging else logging.INFO, 999 abort_callback=_recipe_abort, 1000 ) 1001 1002 event_handler, exclude_list = watch_setup(project_builder, **vars(args)) 1003 1004 run_watch( 1005 event_handler, 1006 exclude_list, 1007 prefs=prefs, 1008 fullscreen=args.fullscreen, 1009 ) 1010 1011 1012if __name__ == '__main__': 1013 main() 1014