1#! /usr/bin/python 2# 3# Protocol Buffers - Google's data interchange format 4# Copyright 2015 Google Inc. All rights reserved. 5# https://developers.google.com/protocol-buffers/ 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions are 9# met: 10# 11# * Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# * Redistributions in binary form must reproduce the above 14# copyright notice, this list of conditions and the following disclaimer 15# in the documentation and/or other materials provided with the 16# distribution. 17# * Neither the name of Google Inc. nor the names of its 18# contributors may be used to endorse or promote products derived from 19# this software without specific prior written permission. 20# 21# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 33"""PDDM - Poor Developers' Debug-able Macros 34 35A simple markup that can be added in comments of source so they can then be 36expanded out into code. Most of this could be done with CPP macros, but then 37developers can't really step through them in the debugger, this way they are 38expanded to the same code, but you can debug them. 39 40Any file can be processed, but the syntax is designed around a C based compiler. 41Processed lines start with "//%". There are three types of sections you can 42create: Text (left alone), Macro Definitions, and Macro Expansions. There is 43no order required between definitions and expansions, all definitions are read 44before any expansions are processed (thus, if desired, definitions can be put 45at the end of the file to keep them out of the way of the code). 46 47Macro Definitions are started with "//%PDDM-DEFINE Name(args)" and all lines 48afterwards that start with "//%" are included in the definition. Multiple 49macros can be defined in one block by just using a another "//%PDDM-DEFINE" 50line to start the next macro. Optionally, a macro can be ended with 51"//%PDDM-DEFINE-END", this can be useful when you want to make it clear that 52trailing blank lines are included in the macro. You can also end a definition 53with an expansion. 54 55Macro Expansions are started by single lines containing 56"//%PDDM-EXPAND Name(args)" and then with "//%PDDM-EXPAND-END" or another 57expansions. All lines in-between are replaced by the result of the expansion. 58The first line of the expansion is always a blank like just for readability. 59 60Expansion itself is pretty simple, one macro can invoke another macro, but 61you cannot nest the invoke of a macro in another macro (i.e. - can't do 62"foo(bar(a))", but you can define foo(a) and bar(b) where bar invokes foo() 63within its expansion. 64 65When macros are expanded, the arg references can also add "$O" suffix to the 66name (i.e. - "NAME$O") to specify an option to be applied. The options are: 67 68 $S - Replace each character in the value with a space. 69 $l - Lowercase the first letter of the value. 70 $L - Lowercase the whole value. 71 $u - Uppercase the first letter of the value. 72 $U - Uppercase the whole value. 73 74Within a macro you can use ## to cause things to get joined together after 75expansion (i.e. - "a##b" within a macro will become "ab"). 76 77Example: 78 79 int foo(MyEnum x) { 80 switch (x) { 81 //%PDDM-EXPAND case(Enum_Left, 1) 82 //%PDDM-EXPAND case(Enum_Center, 2) 83 //%PDDM-EXPAND case(Enum_Right, 3) 84 //%PDDM-EXPAND-END 85 } 86 87 //%PDDM-DEFINE case(_A, _B) 88 //% case _A: 89 //% return _B; 90 91 A macro ends at the start of the next one, or an optional %PDDM-DEFINE-END 92 can be used to avoid adding extra blank lines/returns (or make it clear when 93 it is desired). 94 95 One macro can invoke another by simply using its name NAME(ARGS). You cannot 96 nest an invoke inside another (i.e. - NAME1(NAME2(ARGS)) isn't supported). 97 98 Within a macro you can use ## to cause things to get joined together after 99 processing (i.e. - "a##b" within a macro will become "ab"). 100 101 102""" 103 104import optparse 105import os 106import re 107import sys 108 109 110# Regex for macro definition. 111_MACRO_RE = re.compile(r'(?P<name>\w+)\((?P<args>.*?)\)') 112# Regex for macro's argument definition. 113_MACRO_ARG_NAME_RE = re.compile(r'^\w+$') 114 115# Line inserted after each EXPAND. 116_GENERATED_CODE_LINE = ( 117 '// This block of code is generated, do not edit it directly.' 118) 119 120 121def _MacroRefRe(macro_names): 122 # Takes in a list of macro names and makes a regex that will match invokes 123 # of those macros. 124 return re.compile(r'\b(?P<macro_ref>(?P<name>(%s))\((?P<args>.*?)\))' % 125 '|'.join(macro_names)) 126 127 128def _MacroArgRefRe(macro_arg_names): 129 # Takes in a list of macro arg names and makes a regex that will match 130 # uses of those args. 131 return re.compile(r'\b(?P<name>(%s))(\$(?P<option>.))?\b' % 132 '|'.join(macro_arg_names)) 133 134 135class PDDMError(Exception): 136 """Error thrown by pddm.""" 137 pass 138 139 140class MacroCollection(object): 141 """Hold a set of macros and can resolve/expand them.""" 142 143 def __init__(self, a_file=None): 144 """Initializes the collection. 145 146 Args: 147 a_file: The file like stream to parse. 148 149 Raises: 150 PDDMError if there are any issues. 151 """ 152 self._macros = dict() 153 if a_file: 154 self.ParseInput(a_file) 155 156 class MacroDefinition(object): 157 """Holds a macro definition.""" 158 159 def __init__(self, name, arg_names): 160 self._name = name 161 self._args = tuple(arg_names) 162 self._body = '' 163 self._needNewLine = False 164 165 def AppendLine(self, line): 166 if self._needNewLine: 167 self._body += '\n' 168 self._body += line 169 self._needNewLine = not line.endswith('\n') 170 171 @property 172 def name(self): 173 return self._name 174 175 @property 176 def args(self): 177 return self._args 178 179 @property 180 def body(self): 181 return self._body 182 183 def ParseInput(self, a_file): 184 """Consumes input extracting definitions. 185 186 Args: 187 a_file: The file like stream to parse. 188 189 Raises: 190 PDDMError if there are any issues. 191 """ 192 input_lines = a_file.read().splitlines() 193 self.ParseLines(input_lines) 194 195 def ParseLines(self, input_lines): 196 """Parses list of lines. 197 198 Args: 199 input_lines: A list of strings of input to parse (no newlines on the 200 strings). 201 202 Raises: 203 PDDMError if there are any issues. 204 """ 205 current_macro = None 206 for line in input_lines: 207 if line.startswith('PDDM-'): 208 directive = line.split(' ', 1)[0] 209 if directive == 'PDDM-DEFINE': 210 name, args = self._ParseDefineLine(line) 211 if self._macros.get(name): 212 raise PDDMError('Attempt to redefine macro: "%s"' % line) 213 current_macro = self.MacroDefinition(name, args) 214 self._macros[name] = current_macro 215 continue 216 if directive == 'PDDM-DEFINE-END': 217 if not current_macro: 218 raise PDDMError('Got DEFINE-END directive without an active macro:' 219 ' "%s"' % line) 220 current_macro = None 221 continue 222 raise PDDMError('Hit a line with an unknown directive: "%s"' % line) 223 224 if current_macro: 225 current_macro.AppendLine(line) 226 continue 227 228 # Allow blank lines between macro definitions. 229 if line.strip() == '': 230 continue 231 232 raise PDDMError('Hit a line that wasn\'t a directive and no open macro' 233 ' definition: "%s"' % line) 234 235 def _ParseDefineLine(self, input_line): 236 assert input_line.startswith('PDDM-DEFINE') 237 line = input_line[12:].strip() 238 match = _MACRO_RE.match(line) 239 # Must match full line 240 if match is None or match.group(0) != line: 241 raise PDDMError('Failed to parse macro definition: "%s"' % input_line) 242 name = match.group('name') 243 args_str = match.group('args').strip() 244 args = [] 245 if args_str: 246 for part in args_str.split(','): 247 arg = part.strip() 248 if arg == '': 249 raise PDDMError('Empty arg name in macro definition: "%s"' 250 % input_line) 251 if not _MACRO_ARG_NAME_RE.match(arg): 252 raise PDDMError('Invalid arg name "%s" in macro definition: "%s"' 253 % (arg, input_line)) 254 if arg in args: 255 raise PDDMError('Arg name "%s" used more than once in macro' 256 ' definition: "%s"' % (arg, input_line)) 257 args.append(arg) 258 return (name, tuple(args)) 259 260 def Expand(self, macro_ref_str): 261 """Expands the macro reference. 262 263 Args: 264 macro_ref_str: String of a macro reference (i.e. foo(a, b)). 265 266 Returns: 267 The text from the expansion. 268 269 Raises: 270 PDDMError if there are any issues. 271 """ 272 match = _MACRO_RE.match(macro_ref_str) 273 if match is None or match.group(0) != macro_ref_str: 274 raise PDDMError('Failed to parse macro reference: "%s"' % macro_ref_str) 275 if match.group('name') not in self._macros: 276 raise PDDMError('No macro named "%s".' % match.group('name')) 277 return self._Expand(match, [], macro_ref_str) 278 279 def _FormatStack(self, macro_ref_stack): 280 result = '' 281 for _, macro_ref in reversed(macro_ref_stack): 282 result += '\n...while expanding "%s".' % macro_ref 283 return result 284 285 def _Expand(self, macro_ref_match, macro_stack, macro_ref_str=None): 286 if macro_ref_str is None: 287 macro_ref_str = macro_ref_match.group('macro_ref') 288 name = macro_ref_match.group('name') 289 for prev_name, prev_macro_ref in macro_stack: 290 if name == prev_name: 291 raise PDDMError('Found macro recursion, invoking "%s":%s' % 292 (macro_ref_str, self._FormatStack(macro_stack))) 293 macro = self._macros[name] 294 args_str = macro_ref_match.group('args').strip() 295 args = [] 296 if args_str or len(macro.args): 297 args = [x.strip() for x in args_str.split(',')] 298 if len(args) != len(macro.args): 299 raise PDDMError('Expected %d args, got: "%s".%s' % 300 (len(macro.args), macro_ref_str, 301 self._FormatStack(macro_stack))) 302 # Replace args usages. 303 result = self._ReplaceArgValues(macro, args, macro_ref_str, macro_stack) 304 # Expand any macro invokes. 305 new_macro_stack = macro_stack + [(name, macro_ref_str)] 306 while True: 307 eval_result = self._EvalMacrosRefs(result, new_macro_stack) 308 # Consume all ## directives to glue things together. 309 eval_result = eval_result.replace('##', '') 310 if eval_result == result: 311 break 312 result = eval_result 313 return result 314 315 def _ReplaceArgValues(self, 316 macro, arg_values, macro_ref_to_report, macro_stack): 317 if len(arg_values) == 0: 318 # Nothing to do 319 return macro.body 320 assert len(arg_values) == len(macro.args) 321 args = dict(zip(macro.args, arg_values)) 322 323 def _lookupArg(match): 324 val = args[match.group('name')] 325 opt = match.group('option') 326 if opt: 327 if opt == 'S': # Spaces for the length 328 return ' ' * len(val) 329 elif opt == 'l': # Lowercase first character 330 if val: 331 return val[0].lower() + val[1:] 332 else: 333 return val 334 elif opt == 'L': # All Lowercase 335 return val.lower() 336 elif opt == 'u': # Uppercase first character 337 if val: 338 return val[0].upper() + val[1:] 339 else: 340 return val 341 elif opt == 'U': # All Uppercase 342 return val.upper() 343 else: 344 raise PDDMError('Unknown arg option "%s$%s" while expanding "%s".%s' 345 % (match.group('name'), match.group('option'), 346 macro_ref_to_report, 347 self._FormatStack(macro_stack))) 348 return val 349 # Let the regex do the work! 350 macro_arg_ref_re = _MacroArgRefRe(macro.args) 351 return macro_arg_ref_re.sub(_lookupArg, macro.body) 352 353 def _EvalMacrosRefs(self, text, macro_stack): 354 macro_ref_re = _MacroRefRe(self._macros.keys()) 355 356 def _resolveMacro(match): 357 return self._Expand(match, macro_stack) 358 return macro_ref_re.sub(_resolveMacro, text) 359 360 361class SourceFile(object): 362 """Represents a source file with PDDM directives in it.""" 363 364 def __init__(self, a_file, import_resolver=None): 365 """Initializes the file reading in the file. 366 367 Args: 368 a_file: The file to read in. 369 import_resolver: a function that given a path will return a stream for 370 the contents. 371 372 Raises: 373 PDDMError if there are any issues. 374 """ 375 self._sections = [] 376 self._original_content = a_file.read() 377 self._import_resolver = import_resolver 378 self._processed_content = None 379 380 class SectionBase(object): 381 382 def __init__(self, first_line_num): 383 self._lines = [] 384 self._first_line_num = first_line_num 385 386 def TryAppend(self, line, line_num): 387 """Try appending a line. 388 389 Args: 390 line: The line to append. 391 line_num: The number of the line. 392 393 Returns: 394 A tuple of (SUCCESS, CAN_ADD_MORE). If SUCCESS if False, the line 395 wasn't append. If SUCCESS is True, then CAN_ADD_MORE is True/False to 396 indicate if more lines can be added after this one. 397 """ 398 assert False, "subclass should have overridden" 399 return (False, False) 400 401 def HitEOF(self): 402 """Called when the EOF was reached for for a given section.""" 403 pass 404 405 def BindMacroCollection(self, macro_collection): 406 """Binds the chunk to a macro collection. 407 408 Args: 409 macro_collection: The collection to bind too. 410 """ 411 pass 412 413 def Append(self, line): 414 self._lines.append(line) 415 416 @property 417 def lines(self): 418 return self._lines 419 420 @property 421 def num_lines_captured(self): 422 return len(self._lines) 423 424 @property 425 def first_line_num(self): 426 return self._first_line_num 427 428 @property 429 def first_line(self): 430 if not self._lines: 431 return '' 432 return self._lines[0] 433 434 @property 435 def text(self): 436 return '\n'.join(self.lines) + '\n' 437 438 class TextSection(SectionBase): 439 """Text section that is echoed out as is.""" 440 441 def TryAppend(self, line, line_num): 442 if line.startswith('//%PDDM'): 443 return (False, False) 444 self.Append(line) 445 return (True, True) 446 447 class ExpansionSection(SectionBase): 448 """Section that is the result of an macro expansion.""" 449 450 def __init__(self, first_line_num): 451 SourceFile.SectionBase.__init__(self, first_line_num) 452 self._macro_collection = None 453 454 def TryAppend(self, line, line_num): 455 if line.startswith('//%PDDM'): 456 directive = line.split(' ', 1)[0] 457 if directive == '//%PDDM-EXPAND': 458 self.Append(line) 459 return (True, True) 460 if directive == '//%PDDM-EXPAND-END': 461 assert self.num_lines_captured > 0 462 return (True, False) 463 raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % 464 (directive, line_num, self.first_line)) 465 # Eat other lines. 466 return (True, True) 467 468 def HitEOF(self): 469 raise PDDMError('Hit the end of the file while in "%s".' % 470 self.first_line) 471 472 def BindMacroCollection(self, macro_collection): 473 self._macro_collection = macro_collection 474 475 @property 476 def lines(self): 477 captured_lines = SourceFile.SectionBase.lines.fget(self) 478 directive_len = len('//%PDDM-EXPAND') 479 result = [] 480 for line in captured_lines: 481 result.append(line) 482 if self._macro_collection: 483 # Always add a blank line, seems to read better. (If need be, add an 484 # option to the EXPAND to indicate if this should be done.) 485 result.extend([_GENERATED_CODE_LINE, '// clang-format off', '']) 486 macro = line[directive_len:].strip() 487 try: 488 expand_result = self._macro_collection.Expand(macro) 489 # Since expansions are line oriented, strip trailing whitespace 490 # from the lines. 491 lines = [x.rstrip() for x in expand_result.split('\n')] 492 lines.append('// clang-format on') 493 result.append('\n'.join(lines)) 494 except PDDMError as e: 495 raise PDDMError('%s\n...while expanding "%s" from the section' 496 ' that started:\n Line %d: %s' % 497 (e.message, macro, 498 self.first_line_num, self.first_line)) 499 500 # Add the ending marker. 501 if len(captured_lines) == 1: 502 result.append('//%%PDDM-EXPAND-END %s' % 503 captured_lines[0][directive_len:].strip()) 504 else: 505 result.append('//%%PDDM-EXPAND-END (%s expansions)' % 506 len(captured_lines)) 507 return result 508 509 class DefinitionSection(SectionBase): 510 """Section containing macro definitions""" 511 512 def TryAppend(self, line, line_num): 513 if not line.startswith('//%'): 514 return (False, False) 515 if line.startswith('//%PDDM'): 516 directive = line.split(' ', 1)[0] 517 if directive == "//%PDDM-EXPAND": 518 return False, False 519 if directive not in ('//%PDDM-DEFINE', '//%PDDM-DEFINE-END'): 520 raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % 521 (directive, line_num, self.first_line)) 522 self.Append(line) 523 return (True, True) 524 525 def BindMacroCollection(self, macro_collection): 526 if macro_collection: 527 try: 528 # Parse the lines after stripping the prefix. 529 macro_collection.ParseLines([x[3:] for x in self.lines]) 530 except PDDMError as e: 531 raise PDDMError('%s\n...while parsing section that started:\n' 532 ' Line %d: %s' % 533 (e.message, self.first_line_num, self.first_line)) 534 535 class ImportDefinesSection(SectionBase): 536 """Section containing an import of PDDM-DEFINES from an external file.""" 537 538 def __init__(self, first_line_num, import_resolver): 539 SourceFile.SectionBase.__init__(self, first_line_num) 540 self._import_resolver = import_resolver 541 542 def TryAppend(self, line, line_num): 543 if not line.startswith('//%PDDM-IMPORT-DEFINES '): 544 return (False, False) 545 assert self.num_lines_captured == 0 546 self.Append(line) 547 return (True, False) 548 549 def BindMacroCollection(self, macro_colletion): 550 if not macro_colletion: 551 return 552 if self._import_resolver is None: 553 raise PDDMError('Got an IMPORT-DEFINES without a resolver (line %d):' 554 ' "%s".' % (self.first_line_num, self.first_line)) 555 import_name = self.first_line.split(' ', 1)[1].strip() 556 imported_file = self._import_resolver(import_name) 557 if imported_file is None: 558 raise PDDMError('Resolver failed to find "%s" (line %d):' 559 ' "%s".' % 560 (import_name, self.first_line_num, self.first_line)) 561 try: 562 imported_src_file = SourceFile(imported_file, self._import_resolver) 563 imported_src_file._ParseFile() 564 for section in imported_src_file._sections: 565 section.BindMacroCollection(macro_colletion) 566 except PDDMError as e: 567 raise PDDMError('%s\n...while importing defines:\n' 568 ' Line %d: %s' % 569 (e.message, self.first_line_num, self.first_line)) 570 571 def _ParseFile(self): 572 self._sections = [] 573 lines = self._original_content.splitlines() 574 cur_section = None 575 for line_num, line in enumerate(lines, 1): 576 if not cur_section: 577 cur_section = self._MakeSection(line, line_num) 578 was_added, accept_more = cur_section.TryAppend(line, line_num) 579 if not was_added: 580 cur_section = self._MakeSection(line, line_num) 581 was_added, accept_more = cur_section.TryAppend(line, line_num) 582 assert was_added 583 if not accept_more: 584 cur_section = None 585 586 if cur_section: 587 cur_section.HitEOF() 588 589 def _MakeSection(self, line, line_num): 590 if not line.startswith('//%PDDM'): 591 section = self.TextSection(line_num) 592 else: 593 directive = line.split(' ', 1)[0] 594 if directive == '//%PDDM-EXPAND': 595 section = self.ExpansionSection(line_num) 596 elif directive == '//%PDDM-DEFINE': 597 section = self.DefinitionSection(line_num) 598 elif directive == '//%PDDM-IMPORT-DEFINES': 599 section = self.ImportDefinesSection(line_num, self._import_resolver) 600 else: 601 raise PDDMError('Unexpected line %d: "%s".' % (line_num, line)) 602 self._sections.append(section) 603 return section 604 605 def ProcessContent(self, strip_expansion=False): 606 """Processes the file contents.""" 607 self._ParseFile() 608 if strip_expansion: 609 # Without a collection the expansions become blank, removing them. 610 collection = None 611 else: 612 collection = MacroCollection() 613 for section in self._sections: 614 section.BindMacroCollection(collection) 615 result = '' 616 for section in self._sections: 617 result += section.text 618 self._processed_content = result 619 620 @property 621 def original_content(self): 622 return self._original_content 623 624 @property 625 def processed_content(self): 626 return self._processed_content 627 628 629def main(args): 630 usage = '%prog [OPTIONS] PATH ...' 631 description = ( 632 'Processes PDDM directives in the given paths and write them back out.' 633 ) 634 parser = optparse.OptionParser(usage=usage, description=description) 635 parser.add_option('--dry-run', 636 default=False, action='store_true', 637 help='Don\'t write back to the file(s), just report if the' 638 ' contents needs an update and exit with a value of 1.') 639 parser.add_option('--verbose', 640 default=False, action='store_true', 641 help='Reports is a file is already current.') 642 parser.add_option('--collapse', 643 default=False, action='store_true', 644 help='Removes all the generated code.') 645 opts, extra_args = parser.parse_args(args) 646 647 if not extra_args: 648 parser.error('Need atleast one file to process') 649 650 result = 0 651 for a_path in extra_args: 652 if not os.path.exists(a_path): 653 sys.stderr.write('ERROR: File not found: %s\n' % a_path) 654 return 100 655 656 def _ImportResolver(name): 657 # resolve based on the file being read. 658 a_dir = os.path.dirname(a_path) 659 import_path = os.path.join(a_dir, name) 660 if not os.path.exists(import_path): 661 return None 662 return open(import_path, 'r') 663 664 with open(a_path, 'r') as f: 665 src_file = SourceFile(f, _ImportResolver) 666 667 try: 668 src_file.ProcessContent(strip_expansion=opts.collapse) 669 except PDDMError as e: 670 sys.stderr.write('ERROR: %s\n...While processing "%s"\n' % 671 (e.message, a_path)) 672 return 101 673 674 if src_file.processed_content != src_file.original_content: 675 if not opts.dry_run: 676 print('Updating for "%s".' % a_path) 677 with open(a_path, 'w') as f: 678 f.write(src_file.processed_content) 679 else: 680 # Special result to indicate things need updating. 681 print('Update needed for "%s".' % a_path) 682 result = 1 683 elif opts.verbose: 684 print('No update for "%s".' % a_path) 685 686 return result 687 688 689if __name__ == '__main__': 690 sys.exit(main(sys.argv[1:])) 691