1#!/usr/bin/env python3 2 3"""Ninja File Parser. 4""" 5 6from __future__ import print_function 7 8import argparse 9import collections 10import os 11import re 12import struct 13import sys 14 15try: 16 import cPickle as pickle # Python 2 17except ImportError: 18 import pickle # Python 3 19 20try: 21 from cStringIO import StringIO # Python 2 22except ImportError: 23 from io import StringIO # Python 3 24 25try: 26 from sys import intern 27except ImportError: 28 pass # In Python 2, intern() is a built-in function. 29 30if sys.version_info < (3,): 31 # Wrap built-in open() function to ignore encoding in Python 2. 32 _builtin_open = open 33 def open(path, mode, encoding=None): 34 return _builtin_open(path, mode) 35 36 # Replace built-in zip() function with itertools.izip 37 from itertools import izip as zip 38 39 40class EvalEnv(dict): 41 __slots__ = ('parent') 42 43 44 def __init__(self, *args, **kwargs): 45 super(EvalEnv, self).__init__(*args, **kwargs) 46 self.parent = None 47 48 49 def get_recursive(self, key, default=None): 50 try: 51 return self[key] 52 except KeyError: 53 if self.parent: 54 return self.parent.get_recursive(key, default) 55 return default 56 57 58class BuildEvalEnv(EvalEnv): 59 __slots__ = ('_build_env', '_rule_env') 60 61 62 def __init__(self, build_env, rule_env): 63 self._build_env = build_env 64 self._rule_env = rule_env 65 66 67 def get_recursive(self, key, default=None): 68 try: 69 return self._build_env[key] 70 except KeyError: 71 pass 72 73 if self._rule_env: 74 try: 75 return self._rule_env[key] 76 except KeyError: 77 pass 78 79 if self._build_env.parent: 80 return self._build_env.parent.get_recursive(key, default) 81 return default 82 83 84class EvalError(ValueError): 85 """Exceptions for ``EvalString`` evalution errors.""" 86 pass 87 88 89class EvalCircularError(EvalError): 90 """Exception for circular substitution in ``EvalString``.""" 91 92 93 def __init__(self, expanded_vars): 94 super(EvalCircularError, self).__init__( 95 'circular evaluation: ' + ' -> '.join(expanded_vars)) 96 97 98class EvalString(tuple): 99 """Strings with variables to be substituted.""" 100 101 102 def __bool__(self): 103 """Check whether this is an empty string.""" 104 return len(self) > 1 105 106 107 def __nonzero__(self): 108 """Check whether this is an empty string (Python2).""" 109 return self.__bool__() 110 111 112 def create_iters(self): 113 """Create descriptors and segments iterators.""" 114 curr_iter = iter(self) 115 descs = next(curr_iter) 116 return zip(descs, curr_iter) 117 118 119def _eval_string(s, env, expanded_vars, result_buf): 120 """Evaluate each segments in ``EvalString`` and write result to the 121 given StringIO buffer. 122 123 Args: 124 env: A ``dict`` that maps a name to ``EvalString`` object. 125 expanded_vars: A ``list`` that keeps the variable under evaluation. 126 result_buf: Output buffer. 127 """ 128 if type(s) is str: 129 result_buf.write(s) 130 return 131 132 for desc, seg in s.create_iters(): 133 if desc == 't': 134 # Append raw text 135 result_buf.write(seg) 136 else: 137 # Substitute variables 138 varname = seg 139 if varname in expanded_vars: 140 raise EvalCircularError(expanded_vars + [varname]) 141 expanded_vars.append(varname) 142 try: 143 next_es = env.get_recursive(varname) 144 if next_es: 145 _eval_string(next_es, env, expanded_vars, result_buf) 146 finally: 147 expanded_vars.pop() 148 149 150def eval_string(s, env): 151 """Evaluate a ``str`` or ``EvalString`` in an environment. 152 153 Args: 154 env: A ``dict`` that maps a name to an ``EvalString`` object. 155 156 Returns: 157 str: The result of evaluation. 158 159 Raises: 160 EvalNameError: Unknown variable name occurs. 161 EvalCircularError: Circular variable substitution occurs. 162 """ 163 expanded_vars = [] 164 result_buf = StringIO() 165 _eval_string(s, env, expanded_vars, result_buf) 166 return result_buf.getvalue() 167 168 169def eval_path_strings(strs, env): 170 """Evalute a list of ``EvalString`` in an environment and normalize paths. 171 172 Args: 173 strs: A list of ``EvalString`` which should be treated as paths. 174 env: A ``dict`` that maps a name to an ``EvalString`` object. 175 176 Returns: 177 The list of evaluated strings. 178 """ 179 return [intern(os.path.normpath(eval_string(s, env))) for s in strs] 180 181 182class EvalStringBuilder(object): 183 def __init__(self): 184 self._segs = [''] 185 186 187 def append_raw(self, text): 188 descs = self._segs[0] 189 if descs and descs[-1] == 't': 190 self._segs[-1] += text 191 else: 192 self._segs[0] += 't' 193 self._segs.append(text) 194 return self 195 196 197 def append_var(self, varname): 198 self._segs[0] += 'v' 199 self._segs.append(varname) 200 return self 201 202 203 def getvalue(self): 204 return EvalString(intern(seg) for seg in self._segs) 205 206 207class Build(object): 208 __slots__ = ('explicit_outs', 'implicit_outs', 'rule', 'explicit_ins', 209 'implicit_ins', 'prerequisites', 'bindings', 210 'depfile_implicit_ins') 211 212 213class Rule(object): 214 __slots__ = ('name', 'bindings') 215 216 217class Pool(object): 218 __slots__ = ('name', 'bindings') 219 220 221class Default(object): 222 __slots__ = ('outs') 223 224 225Token = collections.namedtuple('Token', 'kind line column value') 226 227 228class TK(object): 229 """Token ID enumerations.""" 230 231 # Trivial tokens 232 EOF = 0 233 COMMENT = 1 234 NEWLINE = 2 235 SPACE = 3 236 ESC_NEWLINE = 4 237 IDENT = 5 238 PIPE2 = 6 239 PIPE = 7 240 COLON = 8 241 ASSIGN = 9 242 243 # Non-trivial tokens 244 PATH = 10 245 STRING = 11 246 247 248class TokenMatcher(object): 249 def __init__(self, patterns): 250 self._matcher = re.compile('|'.join('(' + p + ')' for k, p in patterns)) 251 self._kinds = [k for k, p in patterns] 252 253 254 def match(self, buf, pos): 255 match = self._matcher.match(buf, pos) 256 if not match: 257 return None 258 return (self._kinds[match.lastindex - 1], match.start(), match.end()) 259 260 261class ParseError(ValueError): 262 def __init__(self, path, line, column, reason=None): 263 self.path = path 264 self.line = line 265 self.column = column 266 self.reason = reason 267 268 269 def __repr__(self): 270 s = 'ParseError: {}:{}:{}'.format(self.path, self.line, self.column) 271 if self.reason: 272 s += ': ' + self.reason 273 return s 274 275 276class Lexer(object): 277 def __init__(self, lines_iterable, path='<stdin>', encoding='utf-8'): 278 self.encoding = encoding 279 self.path = path 280 281 self._line_iter = iter(lines_iterable) 282 self._line_buf = None 283 self._line = 0 284 self._line_pos = 0 285 self._line_end = 0 286 287 self._line_start = True 288 289 self._next_token = None 290 self._next_pos = None 291 292 293 def raise_error(self, reason=None): 294 raise ParseError(self.path, self._line, self._line_pos + 1, reason) 295 296 297 def _read_next_line(self): 298 try: 299 self._line_buf = next(self._line_iter) 300 self._line_pos = 0 301 self._line_end = len(self._line_buf) 302 self._line += 1 303 return True 304 except StopIteration: 305 self._line_buf = None 306 return False 307 308 309 def _ensure_line(self): 310 if self._line_buf and self._line_pos < self._line_end: 311 return True 312 return self._read_next_line() 313 314 _COMMENT_MATCHER = re.compile(r'[ \t]*(?:#[^\n]*)?(?=\n)') 315 316 317 def _ensure_non_comment_line(self): 318 if not self._ensure_line(): 319 return False 320 # Match comments or spaces 321 match = self._COMMENT_MATCHER.match(self._line_buf) 322 if not match: 323 return True 324 # Move the cursor to the newline character 325 self._line_pos = match.end() 326 return True 327 328 _SPACE_MATCHER = re.compile(r'[ \t]+') 329 330 331 def _skip_space(self): 332 match = self._SPACE_MATCHER.match(self._line_buf, self._line_pos) 333 if match: 334 self._line_pos = match.end() 335 336 _SIMPLE_TOKEN_MATCHER = TokenMatcher([ 337 (TK.COMMENT, r'#[^\n]*'), 338 (TK.NEWLINE, r'[\r\n]'), 339 (TK.SPACE, r'[ \t]+'), 340 (TK.ESC_NEWLINE, r'\$[\r\n]'), 341 (TK.IDENT, r'[\w_.-]+'), 342 (TK.PIPE2, r'\|\|'), 343 (TK.PIPE, r'\|'), 344 (TK.COLON, r':'), 345 (TK.ASSIGN, r'='), 346 ]) 347 348 349 def peek(self): 350 if self._next_token is not None: 351 return self._next_token 352 while True: 353 if not self._ensure_non_comment_line(): 354 return Token(TK.EOF, self._line, self._line_pos + 1, '') 355 356 match = self._SIMPLE_TOKEN_MATCHER.match( 357 self._line_buf, self._line_pos) 358 if not match: 359 return None 360 kind, start, end = match 361 362 # Skip comments and spaces 363 if ((kind == TK.SPACE and not self._line_start) or 364 (kind == TK.ESC_NEWLINE) or 365 (kind == TK.COMMENT)): 366 self._line_pos = end 367 continue 368 369 # Save the peaked token 370 token = Token(kind, self._line, self._line_pos + 1, 371 self._line_buf[start:end]) 372 self._next_token = token 373 self._next_pos = end 374 return token 375 376 377 def lex(self): 378 token = self.peek() 379 if not token: 380 self.raise_error() 381 self._line_start = token.kind == TK.NEWLINE 382 self._line_pos = self._next_pos 383 self._next_token = None 384 self._next_pos = None 385 return token 386 387 388 def lex_match(self, match_set): 389 token = self.lex() 390 if token.kind not in match_set: 391 self.raise_error() 392 return token 393 394 395 class STR_TK(object): 396 END = 0 397 CHARS = 1 398 ESC_CHAR = 2 399 ESC_NEWLINE = 3 400 VAR = 4 401 CURVE_VAR = 5 402 403 404 _PATH_TOKEN_MATCHER = TokenMatcher([ 405 (STR_TK.END, r'[ \t\n|:]'), 406 (STR_TK.CHARS, r'[^ \t\n|:$]+'), 407 (STR_TK.ESC_CHAR, r'\$[^\n{\w_-]'), 408 (STR_TK.ESC_NEWLINE, r'\$\n[ \t]*'), 409 (STR_TK.VAR, r'\$[\w_-]+'), 410 (STR_TK.CURVE_VAR, r'\$\{[\w_.-]+\}'), 411 ]) 412 413 414 _STR_TOKEN_MATCHER = TokenMatcher([ 415 (STR_TK.END, r'\n+'), 416 (STR_TK.CHARS, r'[^\n$]+'), 417 (STR_TK.ESC_CHAR, r'\$[^\n{\w_-]'), 418 (STR_TK.ESC_NEWLINE, r'\$\n[ \t]*'), 419 (STR_TK.VAR, r'\$[\w_-]+'), 420 (STR_TK.CURVE_VAR, r'\$\{[\w_.-]+\}'), 421 ]) 422 423 424 def _lex_string_or_path(self, matcher, result_kind): 425 self._ensure_line() 426 self._skip_space() 427 428 start_line = self._line 429 start_column = self._line_pos + 1 430 431 builder = EvalStringBuilder() 432 433 while True: 434 if not self._ensure_line(): 435 break 436 437 match = matcher.match(self._line_buf, self._line_pos) 438 if not match: 439 self.raise_error('unknown character sequence') 440 441 kind, start, end = match 442 if kind == self.STR_TK.END: 443 break 444 445 self._line_pos = end 446 447 if kind == self.STR_TK.CHARS: 448 builder.append_raw(self._line_buf[start:end]) 449 elif kind == self.STR_TK.ESC_CHAR: 450 ch = self._line_buf[start + 1] 451 if ch in ' \t:$': 452 builder.append_raw(ch) 453 else: 454 self.raise_error('bad escape sequence') 455 elif kind == self.STR_TK.ESC_NEWLINE: 456 if not self._read_next_line(): 457 break 458 self._skip_space() 459 elif kind == self.STR_TK.VAR: 460 builder.append_var(self._line_buf[start + 1 : end]) 461 else: 462 assert kind == self.STR_TK.CURVE_VAR 463 builder.append_var(self._line_buf[start + 2 : end - 1]) 464 465 self._next_token = None 466 return Token(result_kind, start_line, start_column, builder.getvalue()) 467 468 469 def lex_path(self): 470 return self._lex_string_or_path(self._PATH_TOKEN_MATCHER, TK.PATH) 471 472 473 def lex_string(self): 474 return self._lex_string_or_path(self._STR_TOKEN_MATCHER, TK.STRING) 475 476 477Manifest = collections.namedtuple('Manifest', 'builds rules pools defaults') 478 479 480class Parser(object): 481 """Ninja Manifest Parser 482 483 This parser parses ninja-build manifest files, such as:: 484 485 cflags = -Wall 486 487 pool cc_pool 488 depth = 1 489 490 rule cc 491 command = gcc -c -o $out $in $cflags $extra_cflags 492 pool = cc_pool 493 494 build test.o : cc test.c 495 extra_cflags = -Werror 496 497 default test.o 498 499 Example: 500 >>> manifest = Parser().parse('build.ninja', 'utf-8') 501 >>> print(manifest.builds) 502 503 """ 504 505 506 def __init__(self, base_dir=None): 507 if base_dir is None: 508 self._base_dir = os.getcwd() 509 else: 510 self._base_dir = base_dir 511 512 # File context 513 self._context = [] 514 self._lexer = None 515 self._env = None 516 517 # Intermediate results 518 self._builds = [] 519 self._rules = [] 520 self._pools = [] 521 self._defaults = [] 522 523 self._rules_dict = {} 524 525 526 def _push_context(self, lexer, env): 527 """Push a parsing file context. 528 529 Args: 530 lexer: Lexer for the associated file. 531 env: Environment for global variable bindings. 532 """ 533 534 self._context.append((self._lexer, self._env)) 535 self._lexer = lexer 536 self._env = env 537 538 539 def _pop_context(self): 540 """Push a parsing file context.""" 541 542 current_context = (self._lexer, self._env) 543 self._lexer, self._env = self._context.pop() 544 return current_context 545 546 547 def parse(self, path, encoding, depfile=None): 548 """Parse a ninja-build manifest file. 549 550 Args: 551 path (str): Input file path to be parsed. 552 encoding (str): Input file encoding. 553 554 Returns: 555 Manifest: Parsed manifest for the given ninja-build manifest file. 556 """ 557 558 self._parse_internal(path, encoding, EvalEnv()) 559 if depfile: 560 self.parse_dep_file(depfile, encoding) 561 return Manifest(self._builds, self._rules, self._pools, self._defaults) 562 563 564 def _parse_internal(self, path, encoding, env): 565 path = os.path.join(self._base_dir, path) 566 with open(path, 'r', encoding=encoding) as fp: 567 self._push_context(Lexer(fp, path, encoding), env) 568 try: 569 self._parse_all_top_level_stmts() 570 finally: 571 self._pop_context() 572 573 574 def _parse_all_top_level_stmts(self): 575 """Parse all top-level statements in a file.""" 576 while self._parse_top_level_stmt(): 577 pass 578 579 580 def _parse_top_level_stmt(self): 581 """Parse a top level statement.""" 582 583 token = self._lexer.peek() 584 if not token: 585 # An unexpected non-trivial token occurs. Raise an error. 586 self._lexer.raise_error() 587 588 if token.kind == TK.EOF: 589 return False 590 elif token.kind == TK.NEWLINE: 591 self._lexer.lex() 592 elif token.kind == TK.IDENT: 593 ident = token.value 594 if ident == 'rule': 595 self._parse_rule_stmt() 596 elif ident == 'build': 597 self._parse_build_stmt() 598 elif ident == 'default': 599 self._parse_default_stmt() 600 elif ident == 'pool': 601 self._parse_pool_stmt() 602 elif ident in {'subninja', 'include'}: 603 self._parse_include_stmt() 604 else: 605 self._parse_global_binding_stmt() 606 else: 607 # An unexpected trivial token occurs. Raise an error. 608 self._lexer.raise_error() 609 return True 610 611 612 def _parse_path_list(self, end_set): 613 """Parse a list of paths.""" 614 615 result = [] 616 while True: 617 token = self._lexer.peek() 618 if token: 619 if token.kind in end_set: 620 break 621 elif token.kind != TK.IDENT: 622 self._lexer.raise_error() 623 624 token = self._lexer.lex_path() 625 result.append(token.value) 626 return result 627 628 629 def _parse_binding_stmt(self): 630 """Parse a variable binding statement. 631 632 Example: 633 IDENT = STRING 634 """ 635 key = self._lexer.lex_match({TK.IDENT}).value 636 self._lexer.lex_match({TK.ASSIGN}) 637 token = self._lexer.lex_string() 638 value = token.value 639 self._lexer.lex_match({TK.NEWLINE, TK.EOF}) 640 return (key, value) 641 642 643 def _parse_global_binding_stmt(self): 644 """Parse a global variable binding statement. 645 646 Example: 647 IDENT = STRING 648 """ 649 650 key, value = self._parse_binding_stmt() 651 value = eval_string(value, self._env) 652 self._env[key] = value 653 654 655 def _parse_local_binding_block(self): 656 """Parse several local variable bindings. 657 658 Example: 659 SPACE IDENT1 = STRING1 660 SPACE IDENT2 = STRING2 661 """ 662 result = EvalEnv() 663 while True: 664 token = self._lexer.peek() 665 if not token or token.kind != TK.SPACE: 666 break 667 self._lexer.lex() 668 key, value = self._parse_binding_stmt() 669 result[key] = value 670 return result 671 672 673 def _parse_build_stmt(self): 674 """Parse `build` statement. 675 676 Example: 677 build PATH1 PATH2 | PATH3 PATH4 : IDENT PATH5 PATH6 | $ 678 PATH7 PATH8 || PATH9 PATH10 679 SPACE IDENT1 = STRING1 680 SPACE IDENT2 = STRING2 681 """ 682 683 token = self._lexer.lex_match({TK.IDENT}) 684 assert token.value == 'build' 685 686 build = Build() 687 688 # Parse explicit outs 689 explicit_outs = self._parse_path_list({TK.PIPE, TK.COLON}) 690 691 # Parse implicit outs 692 token = self._lexer.peek() 693 if token.kind == TK.PIPE: 694 self._lexer.lex() 695 implicit_outs = self._parse_path_list({TK.COLON}) 696 else: 697 implicit_outs = tuple() 698 699 self._lexer.lex_match({TK.COLON}) 700 701 # Parse rule name for this build statement 702 build.rule = self._lexer.lex_match({TK.IDENT}).value 703 try: 704 rule_env = self._rules_dict[build.rule].bindings 705 except KeyError: 706 if build.rule != 'phony': 707 self._lexer.raise_error('undeclared rule name') 708 rule_env = self._env 709 710 # Parse explicit ins 711 explicit_ins = self._parse_path_list( 712 {TK.PIPE, TK.PIPE2, TK.NEWLINE, TK.EOF}) 713 714 # Parse implicit ins 715 token = self._lexer.peek() 716 if token.kind == TK.PIPE: 717 self._lexer.lex() 718 implicit_ins = self._parse_path_list({TK.PIPE2, TK.NEWLINE, TK.EOF}) 719 else: 720 implicit_ins = tuple() 721 722 # Parse order-only prerequisites 723 token = self._lexer.peek() 724 if token.kind == TK.PIPE2: 725 self._lexer.lex() 726 prerequisites = self._parse_path_list({TK.NEWLINE, TK.EOF}) 727 else: 728 prerequisites = tuple() 729 730 self._lexer.lex_match({TK.NEWLINE, TK.EOF}) 731 732 # Parse local bindings 733 bindings = self._parse_local_binding_block() 734 bindings.parent = self._env 735 if bindings: 736 build.bindings = bindings 737 else: 738 # Don't keep the empty ``dict`` object if there are no bindings 739 build.bindings = None 740 741 # Evaluate all paths 742 env = BuildEvalEnv(bindings, rule_env) 743 744 build.explicit_outs = eval_path_strings(explicit_outs, env) 745 build.implicit_outs = eval_path_strings(implicit_outs, env) 746 build.explicit_ins = eval_path_strings(explicit_ins, env) 747 build.implicit_ins = eval_path_strings(implicit_ins, env) 748 build.prerequisites = eval_path_strings(prerequisites, env) 749 build.depfile_implicit_ins = tuple() 750 751 self._builds.append(build) 752 753 754 def _parse_rule_stmt(self): 755 """Parse a `rule` statement. 756 757 Example: 758 rule IDENT 759 SPACE IDENT1 = STRING1 760 SPACE IDENT2 = STRING2 761 """ 762 763 token = self._lexer.lex_match({TK.IDENT}) 764 assert token.value == 'rule' 765 766 rule = Rule() 767 rule.name = self._lexer.lex_match({TK.IDENT}).value 768 self._lexer.lex_match({TK.NEWLINE, TK.EOF}) 769 rule.bindings = self._parse_local_binding_block() 770 771 self._rules.append(rule) 772 self._rules_dict[rule.name] = rule 773 774 775 def _parse_default_stmt(self): 776 """Parse a `default` statement. 777 778 Example: 779 default PATH1 PATH2 PATH3 780 """ 781 782 token = self._lexer.lex_match({TK.IDENT}) 783 assert token.value == 'default' 784 785 default = Default() 786 outs = self._parse_path_list({TK.NEWLINE, TK.EOF}) 787 default.outs = eval_path_strings(outs, self._env) 788 789 self._lexer.lex_match({TK.NEWLINE, TK.EOF}) 790 791 self._defaults.append(default) 792 793 794 def _parse_pool_stmt(self): 795 """Parse a `pool` statement. 796 797 Example: 798 pool IDENT 799 SPACE IDENT1 = STRING1 800 SPACE IDENT2 = STRING2 801 """ 802 token = self._lexer.lex_match({TK.IDENT}) 803 assert token.value == 'pool' 804 805 pool = Pool() 806 807 token = self._lexer.lex() 808 assert token.kind == TK.IDENT 809 pool.name = token.value 810 811 self._lexer.lex_match({TK.NEWLINE, TK.EOF}) 812 813 pool.bindings = self._parse_local_binding_block() 814 815 self._pools.append(pool) 816 817 818 def _parse_include_stmt(self): 819 """Parse an `include` or `subninja` statement. 820 821 Example: 822 include PATH 823 subninja PATH 824 """ 825 826 token = self._lexer.lex_match({TK.IDENT}) 827 assert token.value in {'include', 'subninja'} 828 wrap_env = token.value == 'subninja' 829 830 token = self._lexer.lex_path() 831 path = eval_string(token.value, self._env) # XXX: Check lookup order 832 self._lexer.lex_match({TK.NEWLINE, TK.EOF}) 833 834 if wrap_env: 835 env = EvalEnv() 836 env.parent = self._env 837 else: 838 env = self._env 839 self._parse_internal(path, self._lexer.encoding, env) 840 841 842 def parse_dep_file(self, path, encoding): 843 depfile = DepFileParser().parse(path, encoding) 844 for build in self._builds: 845 depfile_implicit_ins = set() 846 for explicit_out in build.explicit_outs: 847 deps = depfile.get(explicit_out) 848 if deps: 849 depfile_implicit_ins.update(deps.implicit_ins) 850 build.depfile_implicit_ins = tuple(sorted(depfile_implicit_ins)) 851 852 853class DepFileError(ValueError): 854 pass 855 856 857class DepFileRecord(object): 858 __slots__ = ('id', 'explicit_out', 'mtime', 'implicit_ins') 859 860 861 def __init__(self, id, explicit_out, mtime, implicit_ins): 862 self.id = id 863 self.explicit_out = explicit_out 864 self.mtime = mtime 865 self.implicit_ins = implicit_ins 866 867 868class DepFileParser(object): 869 """Ninja deps log parser which parses ``.ninja_deps`` file. 870 """ 871 872 873 def __init__(self): 874 self._deps = [] 875 self._paths = [] 876 self._path_deps = {} 877 878 879 def parse(self, path, encoding): 880 with open(path, 'rb') as fp: 881 return self._parse(fp, encoding) 882 883 884 @staticmethod 885 def _unpack_uint32(buf): 886 return struct.unpack('<I', buf)[0] 887 888 889 @staticmethod 890 def _unpack_uint32_iter(buf): 891 for p in struct.iter_unpack('<I', buf): 892 yield p[0] 893 894 895 if sys.version_info < (3,): 896 @staticmethod 897 def _extract_path(s, encoding): 898 pos = len(s) 899 count = 3 900 while count > 0 and pos > 0 and s[pos - 1] == b'\0': 901 pos -= 1 902 count -= 1 903 return intern(s[0:pos]) 904 else: 905 @staticmethod 906 def _extract_path(s, encoding): 907 pos = len(s) 908 count = 3 909 while count > 0 and pos > 0 and s[pos - 1] == 0: 910 pos -= 1 911 count -= 1 912 return intern(s[0:pos].decode(encoding)) 913 914 915 def _get_path(self, index): 916 try: 917 return self._paths[index] 918 except IndexError: 919 raise DepFileError('path index overflow') 920 921 922 def _parse(self, fp, encoding): 923 # Check the magic word 924 if fp.readline() != b'# ninjadeps\n': 925 raise DepFileError('bad magic word') 926 927 # Check the file format version 928 version = self._unpack_uint32(fp.read(4)) 929 if version != 3: 930 raise DepFileError('unsupported deps log version: ' + str(version)) 931 932 # Read the records 933 MAX_RECORD_SIZE = (1 << 19) - 1 934 while True: 935 buf = fp.read(4) 936 if not buf: 937 break 938 939 record_size = self._unpack_uint32(buf) 940 is_dep = bool(record_size >> 31) 941 record_size &= (1 << 31) - 1 942 943 if record_size > MAX_RECORD_SIZE: 944 raise DepFileError('record size overflow') 945 946 if is_dep: 947 if record_size % 4 != 0 or record_size < 8: 948 raise DepFileError('corrupted deps record') 949 950 buf = fp.read(record_size) 951 952 dep_iter = self._unpack_uint32_iter(buf) 953 954 idx = len(self._deps) 955 explicit_out = self._get_path(next(dep_iter)) 956 mtime = next(dep_iter) 957 implicit_ins = [self._get_path(p) for p in dep_iter] 958 959 deps = DepFileRecord(idx, explicit_out, mtime, implicit_ins) 960 961 old_deps = self._path_deps.get(explicit_out) 962 if not old_deps: 963 self._deps.append(deps) 964 self._path_deps[explicit_out] = deps 965 elif old_deps.mtime > deps.mtime: 966 self._deps.append(None) 967 else: 968 self._deps[old_deps.id] = None 969 self._deps.append(deps) 970 self._path_deps[explicit_out] = deps 971 else: 972 if record_size < 4: 973 raise DepFileError('corrupted path record') 974 buf = fp.read(record_size - 4) 975 path = self._extract_path(buf, encoding) 976 buf = fp.read(4) 977 checksum = 0xffffffff ^ self._unpack_uint32(buf) 978 if len(self._paths) != checksum: 979 raise DepFileError('bad path record checksum') 980 self._paths.append(path) 981 982 return self._path_deps 983 984 985def _parse_args(): 986 """Parse command line options.""" 987 988 parser = argparse.ArgumentParser() 989 subparsers = parser.add_subparsers(dest='command') 990 991 def _register_input_file_args(parser): 992 parser.add_argument('input_file', help='input ninja file') 993 parser.add_argument('--ninja-deps', help='.ninja_deps file') 994 parser.add_argument('--cwd', help='working directory for ninja') 995 parser.add_argument('--encoding', default='utf-8', 996 help='ninja file encoding') 997 998 # dump sub-command 999 parser_dump = subparsers.add_parser('dump', help='dump dependency graph') 1000 _register_input_file_args(parser_dump) 1001 parser_dump.add_argument('-o', '--output', help='output file') 1002 1003 # pickle sub-command 1004 parser_pickle = subparsers.add_parser( 1005 'pickle', help='serialize dependency graph with pickle') 1006 _register_input_file_args(parser_pickle) 1007 parser_pickle.add_argument('-o', '--output', required=True, 1008 help='output file') 1009 1010 # Parse arguments and check sub-command 1011 args = parser.parse_args() 1012 if args.command is None: 1013 parser.print_help() 1014 sys.exit(1) 1015 1016 return args 1017 1018 1019def load_manifest_from_args(args): 1020 """Load the input manifest specified by command line options.""" 1021 1022 input_file = args.input_file 1023 1024 # If the input file name ends with `.pickle`, load it with pickle.load(). 1025 if input_file.endswith('.pickle'): 1026 with open(input_file, 'rb') as pickle_file: 1027 return pickle.load(pickle_file) 1028 1029 # Parse the ninja file 1030 return Parser(args.cwd).parse(args.input_file, args.encoding, 1031 args.ninja_deps) 1032 1033 1034def dump_manifest(manifest, file): 1035 """Dump a manifest to a text file.""" 1036 1037 for rule in manifest.rules: 1038 print('rule', rule.name, file=file) 1039 1040 for build in manifest.builds: 1041 print('build', file=file) 1042 for path in build.explicit_outs: 1043 print(' explicit_out:', path, file=file) 1044 for path in build.implicit_outs: 1045 print(' implicit_out:', path, file=file) 1046 for path in build.explicit_ins: 1047 print(' explicit_in:', path, file=file) 1048 for path in build.implicit_ins: 1049 print(' implicit_in:', path, file=file) 1050 for path in build.prerequisites: 1051 print(' prerequisites:', path, file=file) 1052 for path in build.depfile_implicit_ins: 1053 print(' depfile_implicit_in:', path, file=file) 1054 1055 for pool in manifest.pools: 1056 print('pool', pool.name, file=file) 1057 1058 for default in manifest.defaults: 1059 print('default', file=file) 1060 for path in default.outs: 1061 print(' out:', path, file=file) 1062 1063 1064def command_dump_main(args): 1065 """Main function for the dump sub-command""" 1066 if args.output is None: 1067 dump_manifest(load_manifest_from_args(args), sys.stdout) 1068 else: 1069 with open(args.output, 'w') as output_file: 1070 dump_manifest(load_manifest_from_args(args), output_file) 1071 1072 1073def command_pickle_main(args): 1074 """Main function for the pickle sub-command""" 1075 with open(args.output, 'wb') as output_file: 1076 pickle.dump(load_manifest_from_args(args), output_file) 1077 1078 1079def main(): 1080 """Main function for the executable""" 1081 args = _parse_args() 1082 if args.command == 'dump': 1083 command_dump_main(args) 1084 elif args.command == 'pickle': 1085 command_pickle_main(args) 1086 else: 1087 raise KeyError('unknown command ' + args.command) 1088 1089 1090if __name__ == '__main__': 1091 import ninja 1092 ninja.main() 1093