1#!/usr/bin/env python 2# Copyright 2020 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15"""Watch files for changes and rebuild. 16 17pw watch runs Ninja in a build directory when source files change. It works with 18any Ninja project (GN or CMake). 19 20Usage examples: 21 22 # Find a build directory and build the default target 23 pw watch 24 25 # Find a build directory and build the stm32f429i target 26 pw watch python.lint stm32f429i 27 28 # Build pw_run_tests.modules in the out/cmake directory 29 pw watch -C out/cmake pw_run_tests.modules 30 31 # Build the default target in out/ and pw_apps in out/cmake 32 pw watch -C out -C out/cmake pw_apps 33 34 # Find a directory and build python.tests, and build pw_apps in out/cmake 35 pw watch python.tests -C out/cmake pw_apps 36""" 37 38import argparse 39from dataclasses import dataclass 40import errno 41from itertools import zip_longest 42import logging 43import os 44from pathlib import Path 45import re 46import shlex 47import subprocess 48import sys 49import threading 50from threading import Thread 51from typing import ( 52 Iterable, 53 List, 54 NamedTuple, 55 NoReturn, 56 Optional, 57 Sequence, 58 Tuple, 59) 60 61import httpwatcher # type: ignore 62 63from watchdog.events import FileSystemEventHandler # type: ignore[import] 64from watchdog.observers import Observer # type: ignore[import] 65 66from prompt_toolkit.formatted_text.base import OneStyleAndTextTuple 67from prompt_toolkit.formatted_text import StyleAndTextTuples 68 69import pw_cli.branding 70import pw_cli.color 71import pw_cli.env 72import pw_cli.log 73import pw_cli.plugins 74import pw_console.python_logging 75 76from pw_watch.watch_app import WatchApp 77from pw_watch.debounce import DebouncedFunction, Debouncer 78 79_COLOR = pw_cli.color.colors() 80_LOG = logging.getLogger('pw_watch') 81_NINJA_LOG = logging.getLogger('pw_watch_ninja_output') 82_ERRNO_INOTIFY_LIMIT_REACHED = 28 83 84# Suppress events under 'fsevents', generated by watchdog on every file 85# event on MacOS. 86# TODO(b/182281481): Fix file ignoring, rather than just suppressing logs 87_FSEVENTS_LOG = logging.getLogger('fsevents') 88_FSEVENTS_LOG.setLevel(logging.WARNING) 89 90_PASS_MESSAGE = """ 91 ██████╗ █████╗ ███████╗███████╗██╗ 92 ██╔══██╗██╔══██╗██╔════╝██╔════╝██║ 93 ██████╔╝███████║███████╗███████╗██║ 94 ██╔═══╝ ██╔══██║╚════██║╚════██║╚═╝ 95 ██║ ██║ ██║███████║███████║██╗ 96 ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝ 97""" 98 99# Pick a visually-distinct font from "PASS" to ensure that readers can't 100# possibly mistake the difference between the two states. 101_FAIL_MESSAGE = """ 102 ▄██████▒░▄▄▄ ██▓ ░██▓ 103 ▓█▓ ░▒████▄ ▓██▒ ░▓██▒ 104 ▒████▒ ░▒█▀ ▀█▄ ▒██▒ ▒██░ 105 ░▓█▒ ░░██▄▄▄▄██ ░██░ ▒██░ 106 ░▒█░ ▓█ ▓██▒░██░░ ████████▒ 107 ▒█░ ▒▒ ▓▒█░░▓ ░ ▒░▓ ░ 108 ░▒ ▒ ▒▒ ░ ▒ ░░ ░ ▒ ░ 109 ░ ░ ░ ▒ ▒ ░ ░ ░ 110 ░ ░ ░ ░ ░ 111""" 112 113_FULLSCREEN_STATUS_COLUMN_WIDTH = 10 114 115 116# TODO(keir): Figure out a better strategy for exiting. The problem with the 117# watcher is that doing a "clean exit" is slow. However, by directly exiting, 118# we remove the possibility of the wrapper script doing anything on exit. 119def _die(*args) -> NoReturn: 120 _LOG.critical(*args) 121 sys.exit(1) 122 123 124class WatchCharset(NamedTuple): 125 slug_ok: str 126 slug_fail: str 127 128 129_ASCII_CHARSET = WatchCharset(_COLOR.green('OK '), _COLOR.red('FAIL')) 130_EMOJI_CHARSET = WatchCharset('✔️ ', '��') 131 132 133@dataclass(frozen=True) 134class BuildCommand: 135 build_dir: Path 136 targets: Tuple[str, ...] = () 137 138 def args(self) -> Tuple[str, ...]: 139 return (str(self.build_dir), *self.targets) 140 141 def __str__(self) -> str: 142 return ' '.join(shlex.quote(arg) for arg in self.args()) 143 144 145def git_ignored(file: Path) -> bool: 146 """Returns true if this file is in a Git repo and ignored by that repo. 147 148 Returns true for ignored files that were manually added to a repo. 149 """ 150 file = file.resolve() 151 directory = file.parent 152 153 # Run the Git command from file's parent so that the correct repo is used. 154 while True: 155 try: 156 returncode = subprocess.run( 157 ['git', 'check-ignore', '--quiet', '--no-index', file], 158 stdout=subprocess.DEVNULL, 159 stderr=subprocess.DEVNULL, 160 cwd=directory).returncode 161 return returncode in (0, 128) 162 except FileNotFoundError: 163 # If the directory no longer exists, try parent directories until 164 # an existing directory is found or all directories have been 165 # checked. This approach makes it possible to check if a deleted 166 # path is ignored in the repo it was originally created in. 167 if directory == directory.parent: 168 return False 169 170 directory = directory.parent 171 172 173class PigweedBuildWatcher(FileSystemEventHandler, DebouncedFunction): 174 """Process filesystem events and launch builds if necessary.""" 175 # pylint: disable=too-many-instance-attributes 176 NINJA_BUILD_STEP = re.compile( 177 r'^\[(?P<step>[0-9]+)/(?P<total_steps>[0-9]+)\] (?P<action>.*)$') 178 179 def __init__( 180 self, 181 build_commands: Sequence[BuildCommand], 182 patterns: Sequence[str] = (), 183 ignore_patterns: Sequence[str] = (), 184 charset: WatchCharset = _ASCII_CHARSET, 185 restart: bool = True, 186 jobs: int = None, 187 fullscreen: bool = False, 188 banners: bool = True, 189 ): 190 super().__init__() 191 192 self.banners = banners 193 self.status_message: Optional[OneStyleAndTextTuple] = None 194 self.result_message: Optional[StyleAndTextTuples] = None 195 self.current_stdout = '' 196 self.current_build_step = '' 197 self.current_build_percent = 0.0 198 self.current_build_errors = 0 199 self.patterns = patterns 200 self.ignore_patterns = ignore_patterns 201 self.build_commands = build_commands 202 self.charset: WatchCharset = charset 203 204 self.restart_on_changes = restart 205 self.fullscreen_enabled = fullscreen 206 self.watch_app: Optional[WatchApp] = None 207 self._current_build: subprocess.Popen 208 209 self._extra_ninja_args = [] if jobs is None else [f'-j{jobs}'] 210 211 self.debouncer = Debouncer(self) 212 213 # Track state of a build. These need to be members instead of locals 214 # due to the split between dispatch(), run(), and on_complete(). 215 self.matching_path: Optional[Path] = None 216 self.builds_succeeded: List[bool] = [] 217 218 if not self.fullscreen_enabled: 219 self.wait_for_keypress_thread = threading.Thread( 220 None, self._wait_for_enter) 221 self.wait_for_keypress_thread.start() 222 223 def rebuild(self): 224 """ Rebuild command triggered from watch app.""" 225 self._current_build.terminate() 226 self._current_build.wait() 227 self.debouncer.press('Manual build requested') 228 229 def _wait_for_enter(self) -> NoReturn: 230 try: 231 while True: 232 _ = input() 233 self._current_build.terminate() 234 self._current_build.wait() 235 236 self.debouncer.press('Manual build requested...') 237 # Ctrl-C on Unix generates KeyboardInterrupt 238 # Ctrl-Z on Windows generates EOFError 239 except (KeyboardInterrupt, EOFError): 240 _exit_due_to_interrupt() 241 242 def _path_matches(self, path: Path) -> bool: 243 """Returns true if path matches according to the watcher patterns""" 244 return (not any(path.match(x) for x in self.ignore_patterns) 245 and any(path.match(x) for x in self.patterns)) 246 247 def dispatch(self, event) -> None: 248 # There isn't any point in triggering builds on new directory creation. 249 # It's the creation or modification of files that indicate something 250 # meaningful enough changed for a build. 251 if event.is_directory: 252 return 253 254 # Collect paths of interest from the event. 255 paths: List[str] = [] 256 if hasattr(event, 'dest_path'): 257 paths.append(os.fsdecode(event.dest_path)) 258 if event.src_path: 259 paths.append(os.fsdecode(event.src_path)) 260 for raw_path in paths: 261 _LOG.debug('File event: %s', raw_path) 262 263 # Check whether Git cares about any of these paths. 264 for path in (Path(p).resolve() for p in paths): 265 if not git_ignored(path) and self._path_matches(path): 266 self._handle_matched_event(path) 267 return 268 269 def _handle_matched_event(self, matching_path: Path) -> None: 270 if self.matching_path is None: 271 self.matching_path = matching_path 272 273 log_message = f'File change detected: {os.path.relpath(matching_path)}' 274 if self.restart_on_changes: 275 if self.fullscreen_enabled and self.watch_app: 276 self.watch_app.rebuild_on_filechange() 277 self.debouncer.press(f'{log_message} Triggering build...') 278 else: 279 _LOG.info('%s ; not rebuilding', log_message) 280 281 def _clear_screen(self) -> None: 282 if not self.fullscreen_enabled: 283 print('\033c', end='') # TODO(pwbug/38): Not Windows compatible. 284 sys.stdout.flush() 285 286 # Implementation of DebouncedFunction.run() 287 # 288 # Note: This will run on the timer thread created by the Debouncer, rather 289 # than on the main thread that's watching file events. This enables the 290 # watcher to continue receiving file change events during a build. 291 def run(self) -> None: 292 """Run all the builds in serial and capture pass/fail for each.""" 293 294 # Clear the screen and show a banner indicating the build is starting. 295 self._clear_screen() 296 297 if self.fullscreen_enabled: 298 self.create_result_message() 299 _LOG.info( 300 _COLOR.green( 301 'Watching for changes. Ctrl-d to exit; enter to rebuild')) 302 else: 303 for line in pw_cli.branding.banner().splitlines(): 304 _LOG.info(line) 305 _LOG.info( 306 _COLOR.green( 307 ' Watching for changes. Ctrl-C to exit; enter to rebuild') 308 ) 309 _LOG.info('') 310 _LOG.info('Change detected: %s', self.matching_path) 311 312 self._clear_screen() 313 314 self.builds_succeeded = [] 315 num_builds = len(self.build_commands) 316 _LOG.info('Starting build with %d directories', num_builds) 317 318 env = os.environ.copy() 319 # Force colors in Pigweed subcommands run through the watcher. 320 env['PW_USE_COLOR'] = '1' 321 # Force Ninja to output ANSI colors 322 env['CLICOLOR_FORCE'] = '1' 323 324 for i, cmd in enumerate(self.build_commands, 1): 325 index = f'[{i}/{num_builds}]' 326 self.builds_succeeded.append(self._run_build(index, cmd, env)) 327 if self.builds_succeeded[-1]: 328 level = logging.INFO 329 tag = '(OK)' 330 else: 331 level = logging.ERROR 332 tag = '(FAIL)' 333 334 _LOG.log(level, '%s Finished build: %s %s', index, cmd, tag) 335 self.create_result_message() 336 337 def create_result_message(self): 338 if not self.fullscreen_enabled: 339 return 340 341 self.result_message = [] 342 first_building_target_found = False 343 for (succeeded, command) in zip_longest(self.builds_succeeded, 344 self.build_commands): 345 if succeeded: 346 self.result_message.append( 347 ('class:theme-fg-green', 348 'OK'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH))) 349 elif succeeded is None and not first_building_target_found: 350 first_building_target_found = True 351 self.result_message.append( 352 ('class:theme-fg-yellow', 353 'Building'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH))) 354 elif first_building_target_found: 355 self.result_message.append( 356 ('', ''.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH))) 357 else: 358 self.result_message.append( 359 ('class:theme-fg-red', 360 'Failed'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH))) 361 self.result_message.append(('', f' {command}\n')) 362 363 def _run_build(self, index: str, cmd: BuildCommand, env: dict) -> bool: 364 # Make sure there is a build.ninja file for Ninja to use. 365 build_ninja = cmd.build_dir / 'build.ninja' 366 if not build_ninja.exists(): 367 # If this is a CMake directory, prompt the user to re-run CMake. 368 if cmd.build_dir.joinpath('CMakeCache.txt').exists(): 369 _LOG.error('%s %s does not exist; re-run CMake to generate it', 370 index, build_ninja) 371 return False 372 373 _LOG.warning('%s %s does not exist; running gn gen %s', index, 374 build_ninja, cmd.build_dir) 375 if not self._execute_command(['gn', 'gen', cmd.build_dir], env): 376 return False 377 378 command = ['ninja', *self._extra_ninja_args, '-C', *cmd.args()] 379 _LOG.info('%s Starting build: %s', index, 380 ' '.join(shlex.quote(arg) for arg in command)) 381 382 return self._execute_command(command, env) 383 384 def _execute_command(self, command: list, env: dict) -> bool: 385 """Runs a command with a blank before/after for visual separation.""" 386 self.current_build_errors = 0 387 self.status_message = ( 388 'class:theme-fg-yellow', 389 'Building'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH)) 390 if self.fullscreen_enabled: 391 return self._execute_command_watch_app(command, env) 392 print() 393 self._current_build = subprocess.Popen(command, env=env) 394 returncode = self._current_build.wait() 395 print() 396 return returncode == 0 397 398 def _execute_command_watch_app(self, command: list, env: dict) -> bool: 399 """Runs a command with and outputs the logs.""" 400 if not self.watch_app: 401 return False 402 self.current_stdout = '' 403 returncode = None 404 with subprocess.Popen(command, 405 env=env, 406 stdout=subprocess.PIPE, 407 stderr=subprocess.STDOUT, 408 errors='replace') as proc: 409 self._current_build = proc 410 411 # Empty line at the start. 412 _NINJA_LOG.info('') 413 while returncode is None: 414 if not proc.stdout: 415 continue 416 417 output = proc.stdout.readline() 418 self.current_stdout += output 419 420 line_match_result = self.NINJA_BUILD_STEP.match(output) 421 if line_match_result: 422 matches = line_match_result.groupdict() 423 self.current_build_step = line_match_result.group(0) 424 self.current_build_percent = float( 425 int(matches.get('step', 0)) / 426 int(matches.get('total_steps', 1))) 427 428 elif output.startswith(WatchApp.NINJA_FAILURE_TEXT): 429 _NINJA_LOG.critical(output.strip()) 430 self.current_build_errors += 1 431 432 else: 433 # Mypy output mixes character encoding in its colored output 434 # due to it's use of the curses module retrieving the 'sgr0' 435 # (or exit_attribute_mode) capability from the host 436 # machine's terminfo database. 437 # 438 # This can result in this sequence ending up in STDOUT as 439 # b'\x1b(B\x1b[m'. (B tells terminals to interpret text as 440 # USASCII encoding but will appear in prompt_toolkit as a B 441 # character. 442 # 443 # The following replace calls will strip out those 444 # instances. 445 _NINJA_LOG.info( 446 output.replace('\x1b(B\x1b[m', 447 '').replace('\x1b[1m', '').strip()) 448 self.watch_app.redraw_ui() 449 450 returncode = proc.poll() 451 # Empty line at the end. 452 _NINJA_LOG.info('') 453 454 return returncode == 0 455 456 # Implementation of DebouncedFunction.cancel() 457 def cancel(self) -> bool: 458 if self.restart_on_changes: 459 self._current_build.terminate() 460 self._current_build.wait() 461 return True 462 463 return False 464 465 # Implementation of DebouncedFunction.run() 466 def on_complete(self, cancelled: bool = False) -> None: 467 # First, use the standard logging facilities to report build status. 468 if cancelled: 469 self.status_message = ( 470 '', 'Cancelled'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH)) 471 _LOG.error('Finished; build was interrupted') 472 elif all(self.builds_succeeded): 473 self.status_message = ( 474 'class:theme-fg-green', 475 'Succeeded'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH)) 476 _LOG.info('Finished; all successful') 477 else: 478 self.status_message = ( 479 'class:theme-fg-red', 480 'Failed'.rjust(_FULLSCREEN_STATUS_COLUMN_WIDTH)) 481 _LOG.info('Finished; some builds failed') 482 483 # Show individual build results for fullscreen app 484 if self.fullscreen_enabled: 485 self.create_result_message() 486 # For non-fullscreen pw watch 487 else: 488 # Show a more distinct colored banner. 489 if not cancelled: 490 # Write out build summary table so you can tell which builds 491 # passed and which builds failed. 492 _LOG.info('') 493 _LOG.info(' .------------------------------------') 494 _LOG.info(' |') 495 for (succeeded, cmd) in zip(self.builds_succeeded, 496 self.build_commands): 497 slug = (self.charset.slug_ok 498 if succeeded else self.charset.slug_fail) 499 _LOG.info(' | %s %s', slug, cmd) 500 _LOG.info(' |') 501 _LOG.info(" '------------------------------------") 502 else: 503 # Build was interrupted. 504 _LOG.info('') 505 _LOG.info(' .------------------------------------') 506 _LOG.info(' |') 507 _LOG.info(' | %s- interrupted', self.charset.slug_fail) 508 _LOG.info(' |') 509 _LOG.info(" '------------------------------------") 510 511 # Show a large color banner for the overall result. 512 if self.banners: 513 if all(self.builds_succeeded) and not cancelled: 514 for line in _PASS_MESSAGE.splitlines(): 515 _LOG.info(_COLOR.green(line)) 516 else: 517 for line in _FAIL_MESSAGE.splitlines(): 518 _LOG.info(_COLOR.red(line)) 519 520 if self.watch_app: 521 self.watch_app.redraw_ui() 522 self.matching_path = None 523 524 # Implementation of DebouncedFunction.on_keyboard_interrupt() 525 def on_keyboard_interrupt(self) -> NoReturn: 526 _exit_due_to_interrupt() 527 528 529_WATCH_PATTERN_DELIMITER = ',' 530_WATCH_PATTERNS = ( 531 '*.bloaty', 532 '*.c', 533 '*.cc', 534 '*.css', 535 '*.cpp', 536 '*.cmake', 537 'CMakeLists.txt', 538 '*.gn', 539 '*.gni', 540 '*.go', 541 '*.h', 542 '*.hpp', 543 '*.ld', 544 '*.md', 545 '*.options', 546 '*.proto', 547 '*.py', 548 '*.rst', 549 '*.s', 550 '*.S', 551) 552 553 554def add_parser_arguments(parser: argparse.ArgumentParser) -> None: 555 """Sets up an argument parser for pw watch.""" 556 parser.add_argument('--patterns', 557 help=(_WATCH_PATTERN_DELIMITER + 558 '-delimited list of globs to ' 559 'watch to trigger recompile'), 560 default=_WATCH_PATTERN_DELIMITER.join(_WATCH_PATTERNS)) 561 parser.add_argument('--ignore_patterns', 562 dest='ignore_patterns_string', 563 help=(_WATCH_PATTERN_DELIMITER + 564 '-delimited list of globs to ' 565 'ignore events from')) 566 567 parser.add_argument('--exclude_list', 568 nargs='+', 569 type=Path, 570 help='directories to ignore during pw watch', 571 default=[]) 572 parser.add_argument('--no-restart', 573 dest='restart', 574 action='store_false', 575 help='do not restart ongoing builds if files change') 576 parser.add_argument( 577 'default_build_targets', 578 nargs='*', 579 metavar='target', 580 default=[], 581 help=('Automatically locate a build directory and build these ' 582 'targets. For example, `host docs` searches for a Ninja ' 583 'build directory at out/ and builds the `host` and `docs` ' 584 'targets. To specify one or more directories, ust the ' 585 '-C / --build_directory option.')) 586 parser.add_argument( 587 '-C', 588 '--build_directory', 589 dest='build_directories', 590 nargs='+', 591 action='append', 592 default=[], 593 metavar=('directory', 'target'), 594 help=('Specify a build directory and optionally targets to ' 595 'build. `pw watch -C out tgt` is equivalent to `ninja ' 596 '-C out tgt`')) 597 parser.add_argument( 598 '--serve-docs', 599 dest='serve_docs', 600 action='store_true', 601 default=False, 602 help='Start a webserver for docs on localhost. The port for this ' 603 ' webserver can be set with the --serve-docs-port option. ' 604 ' Defaults to http://127.0.0.1:8000') 605 parser.add_argument( 606 '--serve-docs-port', 607 dest='serve_docs_port', 608 type=int, 609 default=8000, 610 help='Set the port for the docs webserver. Default to 8000.') 611 612 parser.add_argument( 613 '--serve-docs-path', 614 dest='serve_docs_path', 615 type=Path, 616 default="docs/gen/docs", 617 help='Set the path for the docs to serve. Default to docs/gen/docs' 618 ' in the build directory.') 619 parser.add_argument( 620 '-j', 621 '--jobs', 622 type=int, 623 help="Number of cores to use; defaults to Ninja's default") 624 parser.add_argument('-f', 625 '--fullscreen', 626 action='store_true', 627 default=False, 628 help='Use a fullscreen interface.') 629 parser.add_argument('--debug-logging', 630 action='store_true', 631 help='Enable debug logging.') 632 parser.add_argument('--no-banners', 633 dest='banners', 634 action='store_false', 635 help='Hide pass/fail banners.') 636 637 638def _exit(code: int) -> NoReturn: 639 # Note: The "proper" way to exit is via observer.stop(), then 640 # running a join. However it's slower, so just exit immediately. 641 # 642 # Additionally, since there are several threads in the watcher, the usual 643 # sys.exit approach doesn't work. Instead, run the low level exit which 644 # kills all threads. 645 os._exit(code) # pylint: disable=protected-access 646 647 648def _exit_due_to_interrupt() -> NoReturn: 649 # To keep the log lines aligned with each other in the presence of 650 # a '^C' from the keyboard interrupt, add a newline before the log. 651 _LOG.info('Got Ctrl-C; exiting...') 652 _exit(0) 653 654 655def _exit_due_to_inotify_watch_limit(): 656 # Show information and suggested commands in OSError: inotify limit reached. 657 _LOG.error('Inotify watch limit reached: run this in your terminal if ' 658 'you are in Linux to temporarily increase inotify limit. \n') 659 _LOG.info( 660 _COLOR.green(' sudo sysctl fs.inotify.max_user_watches=' 661 '$NEW_LIMIT$\n')) 662 _LOG.info(' Change $NEW_LIMIT$ with an integer number, ' 663 'e.g., 20000 should be enough.') 664 _exit(0) 665 666 667def _exit_due_to_inotify_instance_limit(): 668 # Show information and suggested commands in OSError: inotify limit reached. 669 _LOG.error('Inotify instance limit reached: run this in your terminal if ' 670 'you are in Linux to temporarily increase inotify limit. \n') 671 _LOG.info( 672 _COLOR.green(' sudo sysctl fs.inotify.max_user_instances=' 673 '$NEW_LIMIT$\n')) 674 _LOG.info(' Change $NEW_LIMIT$ with an integer number, ' 675 'e.g., 20000 should be enough.') 676 _exit(0) 677 678 679def _exit_due_to_pigweed_not_installed(): 680 # Show information and suggested commands when pigweed environment variable 681 # not found. 682 _LOG.error('Environment variable $PW_ROOT not defined or is defined ' 683 'outside the current directory.') 684 _LOG.error('Did you forget to activate the Pigweed environment? ' 685 'Try source ./activate.sh') 686 _LOG.error('Did you forget to install the Pigweed environment? ' 687 'Try source ./bootstrap.sh') 688 _exit(1) 689 690 691# Go over each directory inside of the current directory. 692# If it is not on the path of elements in directories_to_exclude, add 693# (directory, True) to subdirectories_to_watch and later recursively call 694# Observer() on them. 695# Otherwise add (directory, False) to subdirectories_to_watch and later call 696# Observer() with recursion=False. 697def minimal_watch_directories(to_watch: Path, to_exclude: Iterable[Path]): 698 """Determine which subdirectory to watch recursively""" 699 try: 700 to_watch = Path(to_watch) 701 except TypeError: 702 assert False, "Please watch one directory at a time." 703 704 # Reformat to_exclude. 705 directories_to_exclude: List[Path] = [ 706 to_watch.joinpath(directory_to_exclude) 707 for directory_to_exclude in to_exclude 708 if to_watch.joinpath(directory_to_exclude).is_dir() 709 ] 710 711 # Split the relative path of directories_to_exclude (compared to to_watch), 712 # and generate all parent paths needed to be watched without recursion. 713 exclude_dir_parents = {to_watch} 714 for directory_to_exclude in directories_to_exclude: 715 parts = list( 716 Path(directory_to_exclude).relative_to(to_watch).parts)[:-1] 717 dir_tmp = to_watch 718 for part in parts: 719 dir_tmp = Path(dir_tmp, part) 720 exclude_dir_parents.add(dir_tmp) 721 722 # Go over all layers of directory. Append those that are the parents of 723 # directories_to_exclude to the list with recursion==False, and others 724 # with recursion==True. 725 for directory in exclude_dir_parents: 726 dir_path = Path(directory) 727 yield dir_path, False 728 for item in Path(directory).iterdir(): 729 if (item.is_dir() and item not in exclude_dir_parents 730 and item not in directories_to_exclude): 731 yield item, True 732 733 734def get_common_excludes() -> List[Path]: 735 """Find commonly excluded directories, and return them as a [Path]""" 736 exclude_list: List[Path] = [] 737 738 typical_ignored_directories: List[str] = [ 739 '.environment', # Legacy bootstrap-created CIPD and Python venv. 740 '.presubmit', # Presubmit-created CIPD and Python venv. 741 '.git', # Pigweed's git repo. 742 '.mypy_cache', # Python static analyzer. 743 '.cargo', # Rust package manager. 744 'environment', # Bootstrap-created CIPD and Python venv. 745 'out', # Typical build directory. 746 ] 747 748 # Preset exclude list for Pigweed's upstream directories. 749 pw_root_dir = Path(os.environ['PW_ROOT']) 750 exclude_list.extend(pw_root_dir / ignored_directory 751 for ignored_directory in typical_ignored_directories) 752 753 # Preset exclude for common downstream project structures. 754 # 755 # If watch is invoked outside of the Pigweed root, exclude common 756 # directories. 757 pw_project_root_dir = Path(os.environ['PW_PROJECT_ROOT']) 758 if pw_project_root_dir != pw_root_dir: 759 exclude_list.extend( 760 pw_project_root_dir / ignored_directory 761 for ignored_directory in typical_ignored_directories) 762 763 # Check for and warn about legacy directories. 764 legacy_directories = [ 765 '.cipd', # Legacy CIPD location. 766 '.python3-venv', # Legacy Python venv location. 767 ] 768 found_legacy = False 769 for legacy_directory in legacy_directories: 770 full_legacy_directory = pw_root_dir / legacy_directory 771 if full_legacy_directory.is_dir(): 772 _LOG.warning('Legacy environment directory found: %s', 773 str(full_legacy_directory)) 774 exclude_list.append(full_legacy_directory) 775 found_legacy = True 776 if found_legacy: 777 _LOG.warning('Found legacy environment directory(s); these ' 778 'should be deleted') 779 780 return exclude_list 781 782 783def watch_setup( 784 default_build_targets: List[str], 785 build_directories: List[str], 786 patterns: str, 787 ignore_patterns_string: str, 788 exclude_list: List[Path], 789 restart: bool, 790 jobs: Optional[int], 791 serve_docs: bool, 792 serve_docs_port: int, 793 serve_docs_path: Path, 794 fullscreen: bool, 795 banners: bool, 796 # pylint: disable=too-many-arguments 797) -> Tuple[str, PigweedBuildWatcher, List[Path]]: 798 """Watches files and runs Ninja commands when they change.""" 799 _LOG.info('Starting Pigweed build watcher') 800 801 # Get pigweed directory information from environment variable PW_ROOT. 802 if os.environ['PW_ROOT'] is None: 803 _exit_due_to_pigweed_not_installed() 804 pw_root = Path(os.environ['PW_ROOT']).resolve() 805 if Path.cwd().resolve() not in [pw_root, *pw_root.parents]: 806 _exit_due_to_pigweed_not_installed() 807 808 # Preset exclude list for pigweed directory. 809 exclude_list += get_common_excludes() 810 # Add build directories to the exclude list. 811 exclude_list.extend( 812 Path(build_dir[0]).resolve() for build_dir in build_directories) 813 814 build_commands = [ 815 BuildCommand(Path(build_dir[0]), tuple(build_dir[1:])) 816 for build_dir in build_directories 817 ] 818 819 # If no build directory was specified, check for out/build.ninja. 820 if default_build_targets or not build_directories: 821 # Make sure we found something; if not, bail. 822 if not Path('out').exists(): 823 _die("No build dirs found. Did you forget to run 'gn gen out'?") 824 825 build_commands.append( 826 BuildCommand(Path('out'), tuple(default_build_targets))) 827 828 # Verify that the build output directories exist. 829 for i, build_target in enumerate(build_commands, 1): 830 if not build_target.build_dir.is_dir(): 831 _die("Build directory doesn't exist: %s", build_target) 832 else: 833 _LOG.info('Will build [%d/%d]: %s', i, len(build_commands), 834 build_target) 835 836 _LOG.debug('Patterns: %s', patterns) 837 838 if serve_docs: 839 840 def _serve_docs(): 841 # Disable logs from httpwatcher and deps 842 logging.getLogger('httpwatcher').setLevel(logging.CRITICAL) 843 logging.getLogger('tornado').setLevel(logging.CRITICAL) 844 845 docs_path = build_commands[0].build_dir.joinpath( 846 serve_docs_path.joinpath('html')) 847 httpwatcher.watch(docs_path, 848 host="127.0.0.1", 849 port=serve_docs_port) 850 851 # Spin up an httpwatcher in a new thread since it blocks 852 threading.Thread(None, _serve_docs, "httpwatcher").start() 853 854 # Try to make a short display path for the watched directory that has 855 # "$HOME" instead of the full home directory. This is nice for users 856 # who have deeply nested home directories. 857 path_to_log = str(Path().resolve()).replace(str(Path.home()), '$HOME') 858 859 # Ignore the user-specified patterns. 860 ignore_patterns = (ignore_patterns_string.split(_WATCH_PATTERN_DELIMITER) 861 if ignore_patterns_string else []) 862 863 env = pw_cli.env.pigweed_environment() 864 if env.PW_EMOJI: 865 charset = _EMOJI_CHARSET 866 else: 867 charset = _ASCII_CHARSET 868 869 event_handler = PigweedBuildWatcher( 870 build_commands=build_commands, 871 patterns=patterns.split(_WATCH_PATTERN_DELIMITER), 872 ignore_patterns=ignore_patterns, 873 charset=charset, 874 restart=restart, 875 jobs=jobs, 876 fullscreen=fullscreen, 877 banners=banners, 878 ) 879 return path_to_log, event_handler, exclude_list 880 881 882def watch(path_to_log: Path, event_handler: PigweedBuildWatcher, 883 exclude_list: List[Path]): 884 """Watches files and runs Ninja commands when they change.""" 885 try: 886 # It can take awhile to configure the filesystem watcher, so have the 887 # message reflect that with the "...". Run inside the try: to 888 # gracefully handle the user Ctrl-C'ing out during startup. 889 890 _LOG.info('Attaching filesystem watcher to %s/...', path_to_log) 891 892 # Observe changes for all files in the root directory. Whether the 893 # directory should be observed recursively or not is determined by the 894 # second element in subdirectories_to_watch. 895 observers = [] 896 for path, rec in minimal_watch_directories(Path.cwd(), exclude_list): 897 observer = Observer() 898 observer.schedule( 899 event_handler, 900 str(path), 901 recursive=rec, 902 ) 903 observer.start() 904 observers.append(observer) 905 906 event_handler.debouncer.press('Triggering initial build...') 907 for observer in observers: 908 while observer.is_alive(): 909 observer.join(1) 910 911 # Ctrl-C on Unix generates KeyboardInterrupt 912 # Ctrl-Z on Windows generates EOFError 913 except (KeyboardInterrupt, EOFError): 914 _exit_due_to_interrupt() 915 except OSError as err: 916 if err.args[0] == _ERRNO_INOTIFY_LIMIT_REACHED: 917 _exit_due_to_inotify_watch_limit() 918 if err.errno == errno.EMFILE: 919 _exit_due_to_inotify_instance_limit() 920 raise err 921 922 _LOG.critical('Should never get here') 923 observer.join() 924 925 926def main() -> None: 927 """Watch files for changes and rebuild.""" 928 parser = argparse.ArgumentParser( 929 description=__doc__, 930 formatter_class=argparse.RawDescriptionHelpFormatter) 931 add_parser_arguments(parser) 932 args = parser.parse_args() 933 934 path_to_log, event_handler, exclude_list = watch_setup( 935 default_build_targets=args.default_build_targets, 936 build_directories=args.build_directories, 937 patterns=args.patterns, 938 ignore_patterns_string=args.ignore_patterns_string, 939 exclude_list=args.exclude_list, 940 restart=args.restart, 941 jobs=args.jobs, 942 serve_docs=args.serve_docs, 943 serve_docs_port=args.serve_docs_port, 944 serve_docs_path=args.serve_docs_path, 945 fullscreen=args.fullscreen, 946 banners=args.banners, 947 ) 948 949 if args.fullscreen: 950 watch_logfile = (pw_console.python_logging.create_temp_log_file( 951 prefix=__package__)) 952 pw_cli.log.install( 953 level=logging.DEBUG, 954 use_color=True, 955 hide_timestamp=False, 956 log_file=watch_logfile, 957 ) 958 pw_console.python_logging.setup_python_logging( 959 last_resort_filename=watch_logfile) 960 961 watch_thread = Thread(target=watch, 962 args=(path_to_log, event_handler, exclude_list), 963 daemon=True) 964 watch_thread.start() 965 watch_app = WatchApp(event_handler=event_handler, 966 debug_logging=args.debug_logging, 967 log_file_name=watch_logfile) 968 969 event_handler.watch_app = watch_app 970 watch_app.run() 971 else: 972 pw_cli.log.install( 973 level=logging.DEBUG if args.debug_logging else logging.INFO, 974 use_color=True, 975 hide_timestamp=False, 976 ) 977 watch(Path(path_to_log), event_handler, exclude_list) 978 979 980if __name__ == '__main__': 981 main() 982