1# Copyright David Abrahams 2004. 2# Copyright Daniel Wallin 2006. 3# Distributed under the Boost 4# Software License, Version 1.0. (See accompanying 5# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 6 7import os 8import tempfile 9import litre 10import re 11import sys 12import traceback 13 14# Thanks to Jean Brouwers for this snippet 15def _caller(up=0): 16 '''Get file name, line number, function name and 17 source text of the caller's caller as 4-tuple: 18 (file, line, func, text). 19 20 The optional argument 'up' allows retrieval of 21 a caller further back up into the call stack. 22 23 Note, the source text may be None and function 24 name may be '?' in the returned result. In 25 Python 2.3+ the file name may be an absolute 26 path. 27 ''' 28 try: # just get a few frames' 29 f = traceback.extract_stack(limit=up+2) 30 if f: 31 return f[0] 32 except: 33 pass 34 # running with psyco? 35 return ('', 0, '', None) 36 37class Example: 38 closed = False 39 in_emph = None 40 41 def __init__(self, node, section, line_offset, line_hash = '#'): 42 # A list of text fragments comprising the Example. Start with a #line 43 # directive 44 self.section = section 45 self.line_hash = line_hash 46 self.node = node 47 self.body = [] 48 self.line_offset = line_offset 49 self._number_of_prefixes = 0 50 51 self.emphasized = [] # indices of text strings that have been 52 # emphasized. These are generally expected to be 53 # invalid C++ and will need special treatment 54 55 def begin_emphasis(self): 56 self.in_emph = len(self.body) 57 58 def end_emphasis(self): 59 self.emphasized.append( (self.in_emph, len(self.body)) ) 60 61 def append(self, s): 62 self.append_raw(self._make_line(s)) 63 64 def prepend(self, s): 65 self.prepend_raw(self._make_line(s)) 66 67 def append_raw(self, s): 68 self.body.append(s) 69 70 def prepend_raw(self, s): 71 self.body.insert(0,s) 72 self.emphasized = [ (x[0]+1,x[1]+1) for x in self.emphasized ] 73 self._number_of_prefixes += 1 74 75 def replace(self, s1, s2): 76 self.body = [x.replace(s1,s2) for x in self.body] 77 78 def sub(self, pattern, repl, count = 1, flags = re.MULTILINE): 79 pat = re.compile(pattern, flags) 80 for i,txt in enumerate(self.body): 81 if count > 0: 82 x, subs = pat.subn(repl, txt, count) 83 self.body[i] = x 84 count -= subs 85 86 def wrap(self, s1, s2): 87 self.append_raw(self._make_line(s2)) 88 self.prepend_raw(self._make_line(s1, offset = -s1.count('\n'))) 89 90 def replace_emphasis(self, s, index = 0): 91 """replace the index'th emphasized text with s""" 92 e = self.emphasized[index] 93 self.body[e[0]:e[1]] = [s] 94 del self.emphasized[index] 95 96 elipsis = re.compile('^([ \t]*)([.][.][.][ \t]*)$', re.MULTILINE) 97 98 def __str__(self): 99 # Comment out any remaining emphasized sections 100 b = [self.elipsis.sub(r'\1// \2', s) for s in self.body] 101 emph = self.emphasized 102 emph.reverse() 103 for e in emph: 104 b.insert(e[1], ' */') 105 b.insert(e[0], '/* ') 106 emph.reverse() 107 108 # Add initial #line 109 b.insert( 110 self._number_of_prefixes, 111 self._line_directive(self.node.line, self.node.source) 112 ) 113 114 # Add trailing newline to avoid warnings 115 b.append('\n') 116 return ''.join(b) 117 118 def __repr__(self): 119 return "Example: " + repr(str(self)) 120 121 def raw(self): 122 return ''.join(self.body) 123 124 def _make_line(self, s, offset = 0): 125 c = _caller(2)[1::-1] 126 offset -= s.count('\n') 127 return '\n%s%s\n' % (self._line_directive(offset = offset, *c), s.strip('\n')) 128 129 def _line_directive(self, line, source, offset = None): 130 if self.line_hash is None: 131 return '\n' 132 133 if offset is None: 134 offset = self.line_offset 135 136 if line is None or line <= -offset: 137 line = 1 138 else: 139 line += offset 140 141 if source is None: 142 return '%sline %d\n' % (self.line_hash, line) 143 else: 144 return '%sline %d "%s"\n' % (self.line_hash, line, source) 145 146 147def syscmd( 148 cmd 149 , expect_error = False 150 , input = None 151 , max_output_lines = None 152 ): 153 154 # On windows close() returns the exit code, on *nix it doesn't so 155 # we need to use popen2.Popen4 instead. 156 if sys.platform == 'win32': 157 stdin, stdout_stderr = os.popen4(cmd) 158 if input: stdin.write(input) 159 stdin.close() 160 161 out = stdout_stderr.read() 162 status = stdout_stderr.close() 163 else: 164 import popen2 165 process = popen2.Popen4(cmd) 166 if input: process.tochild.write(input) 167 out = process.fromchild.read() 168 status = process.wait() 169 170 if max_output_lines is not None: 171 out = '\n'.join(out.split('\n')[:max_output_lines]) 172 173 if expect_error: 174 status = not status 175 176 if status: 177 print 178 print '========== offending command ===========' 179 print cmd 180 print '------------ stdout/stderr -------------' 181 print expect_error and 'Error expected, but none seen' or out 182 elif expect_error > 1: 183 print 184 print '------ Output of Expected Error --------' 185 print out 186 print '----------------------------------------' 187 188 sys.stdout.flush() 189 190 return (status,out) 191 192 193def expand_vars(path): 194 if os.name == 'nt': 195 re_env = re.compile(r'%\w+%') 196 return re_env.sub( 197 lambda m: os.environ.get( m.group(0)[1:-1] ) 198 , path 199 ) 200 else: 201 return os.path.expandvars(path) 202 203def remove_directory_and_contents(path): 204 for root, dirs, files in os.walk(path, topdown=False): 205 for name in files: 206 os.remove(os.path.join(root, name)) 207 for name in dirs: 208 os.rmdir(os.path.join(root, name)) 209 os.rmdir(path) 210 211class BuildResult: 212 def __init__(self, path): 213 self.path = path 214 215 def __repr__(self): 216 return self.path 217 218 def __del__(self): 219 remove_directory_and_contents(self.path) 220 221class CPlusPlusTranslator(litre.LitreTranslator): 222 223 _exposed_attrs = ['compile', 'test', 'ignore', 'match_stdout', 'stack', 'config' 224 , 'example', 'prefix', 'preprocessors', 'litre_directory', 225 'litre_translator', 'includes', 'build', 'jam_prefix', 226 'run_python'] 227 228 last_run_output = '' 229 230 """Attributes that will be made available to litre code""" 231 232 def __init__(self, document, config): 233 litre.LitreTranslator.__init__(self, document, config) 234 self.in_literal = False 235 self.in_table = True 236 self.preprocessors = [] 237 self.stack = [] 238 self.example = None 239 self.prefix = [] 240 self.includes = config.includes 241 self.litre_directory = os.path.split(__file__)[0] 242 self.config = config 243 self.litre_translator = self 244 self.line_offset = 0 245 self.last_source = None 246 self.jam_prefix = [] 247 248 self.globals = { 'test_literals_in_tables' : False } 249 for m in self._exposed_attrs: 250 self.globals[m] = getattr(self, m) 251 252 self.examples = {} 253 self.current_section = None 254 255 # 256 # Stuff for use by docutils writer framework 257 # 258 def visit_emphasis(self, node): 259 if self.in_literal: 260 self.example.begin_emphasis() 261 262 def depart_emphasis(self, node): 263 if self.in_literal: 264 self.example.end_emphasis() 265 266 def visit_section(self, node): 267 self.current_section = node['ids'][0] 268 269 def visit_literal_block(self, node): 270 if node.source is None: 271 node.source = self.last_source 272 self.last_source = node.source 273 274 # create a new example 275 self.example = Example(node, self.current_section, line_offset = self.line_offset, line_hash = self.config.line_hash) 276 277 self.stack.append(self.example) 278 279 self.in_literal = True 280 281 def depart_literal_block(self, node): 282 self.in_literal = False 283 284 def visit_literal(self, node): 285 if self.in_table and self.globals['test_literals_in_tables']: 286 self.visit_literal_block(node) 287 else: 288 litre.LitreTranslator.visit_literal(self,node) 289 290 def depart_literal(self, node): 291 if self.in_table and self.globals['test_literals_in_tables']: 292 self.depart_literal_block(node) 293 else: 294 litre.LitreTranslator.depart_literal(self,node) 295 296 def visit_table(self,node): 297 self.in_table = True 298 litre.LitreTranslator.visit_table(self,node) 299 300 def depart_table(self,node): 301 self.in_table = False 302 litre.LitreTranslator.depart_table(self,node) 303 304 def visit_Text(self, node): 305 if self.in_literal: 306 self.example.append_raw(node.astext()) 307 308 def depart_document(self, node): 309 self.write_examples() 310 311 # 312 # Private stuff 313 # 314 315 def handled(self, n = 1): 316 r = self.stack[-n:] 317 del self.stack[-n:] 318 return r 319 320 def _execute(self, code): 321 """Override of litre._execute; sets up variable context before 322 evaluating code 323 """ 324 self.globals['example'] = self.example 325 eval(code, self.globals) 326 327 # 328 # Stuff for use by embedded python code 329 # 330 331 def match_stdout(self, expected = None): 332 333 if expected is None: 334 expected = self.example.raw() 335 self.handled() 336 337 if not re.search(expected, self.last_run_output, re.MULTILINE): 338 #if self.last_run_output.strip('\n') != expected.strip('\n'): 339 print 'output failed to match example' 340 print '-------- Actual Output -------------' 341 print repr(self.last_run_output) 342 print '-------- Expected Output -----------' 343 print repr(expected) 344 print '------------------------------------' 345 sys.stdout.flush() 346 347 def ignore(self, n = 1): 348 if n == 'all': 349 n = len(self.stack) 350 return self.handled(n) 351 352 def wrap(self, n, s1, s2): 353 self.stack[-1].append(s2) 354 self.stack[-n].prepend(s1) 355 356 357 def compile( 358 self 359 , howmany = 1 360 , pop = -1 361 , expect_error = False 362 , extension = '.o' 363 , options = ['-c'] 364 , built_handler = lambda built_file: None 365 , source_file = None 366 , source_suffix = '.cpp' 367 # C-style comments by default; handles C++ and YACC 368 , make_comment = lambda text: '/*\n%s\n*/' % text 369 , built_file = None 370 , command = None 371 ): 372 """ 373 Compile examples on the stack, whose topmost item is the last example 374 seen but not yet handled so far. 375 376 :howmany: How many of the topmost examples on the stack to compile. 377 You can pass a number, or 'all' to indicate that all examples should 378 be compiled. 379 380 :pop: How many of the topmost examples to discard. By default, all of 381 the examples that are compiled are discarded. 382 383 :expect_error: Whether a compilation error is to be expected. Any value 384 > 1 will cause the expected diagnostic's text to be dumped for 385 diagnostic purposes. It's common to expect an error but see a 386 completely unrelated one because of bugs in the example (you can get 387 this behavior for all examples by setting show_expected_error_output 388 in your config). 389 390 :extension: The extension of the file to build (set to .exe for 391 run) 392 393 :options: Compiler flags 394 395 :built_file: A path to use for the built file. By default, a temp 396 filename is conjured up 397 398 :built_handler: A function that's called with the name of the built file 399 upon success. 400 401 :source_file: The full name of the source file to write 402 403 :source_suffix: If source_file is None, the suffix to use for the source file 404 405 :make_comment: A function that transforms text into an appropriate comment. 406 407 :command: A function that is passed (includes, opts, target, source), where 408 opts is a string representing compiler options, target is the name of 409 the file to build, and source is the name of the file into which the 410 example code is written. By default, the function formats 411 litre.config.compiler with its argument tuple. 412 """ 413 414 # Grab one example by default 415 if howmany == 'all': 416 howmany = len(self.stack) 417 418 source = '\n'.join( 419 self.prefix 420 + [str(x) for x in self.stack[-howmany:]] 421 ) 422 423 source = reduce(lambda s, f: f(s), self.preprocessors, source) 424 425 if pop: 426 if pop < 0: 427 pop = howmany 428 del self.stack[-pop:] 429 430 if len(self.stack): 431 self.example = self.stack[-1] 432 433 cpp = self._source_file_path(source_file, source_suffix) 434 435 if built_file is None: 436 built_file = self._output_file_path(source_file, extension) 437 438 opts = ' '.join(options) 439 440 includes = ' '.join(['-I%s' % d for d in self.includes]) 441 if not command: 442 command = self.config.compiler 443 444 if type(command) == str: 445 command = lambda i, o, t, s, c = command: c % (i, o, t, s) 446 447 cmd = command(includes, opts, expand_vars(built_file), expand_vars(cpp)) 448 449 if expect_error and self.config.show_expected_error_output: 450 expect_error += 1 451 452 453 comment_cmd = command(includes, opts, built_file, os.path.basename(cpp)) 454 comment = make_comment(config.comment_text(comment_cmd, expect_error)) 455 456 self._write_source(cpp, '\n'.join([comment, source])) 457 458 #print 'wrote in', cpp 459 #print 'trying command', cmd 460 461 status, output = syscmd(cmd, expect_error) 462 463 if status or expect_error > 1: 464 print 465 if expect_error and expect_error < 2: 466 print 'Compilation failure expected, but none seen' 467 print '------------ begin offending source ------------' 468 print open(cpp).read() 469 print '------------ end offending source ------------' 470 471 if self.config.save_cpp: 472 print 'saved in', repr(cpp) 473 else: 474 self._remove_source(cpp) 475 476 sys.stdout.flush() 477 else: 478 print '.', 479 sys.stdout.flush() 480 built_handler(built_file) 481 482 self._remove_source(cpp) 483 484 try: 485 self._unlink(built_file) 486 except: 487 if not expect_error: 488 print 'failed to unlink', built_file 489 490 return status 491 492 def test( 493 self 494 , rule = 'run' 495 , howmany = 1 496 , pop = -1 497 , expect_error = False 498 , requirements = '' 499 , input = '' 500 ): 501 502 # Grab one example by default 503 if howmany == 'all': 504 howmany = len(self.stack) 505 506 source = '\n'.join( 507 self.prefix 508 + [str(x) for x in self.stack[-howmany:]] 509 ) 510 511 source = reduce(lambda s, f: f(s), self.preprocessors, source) 512 513 id = self.example.section 514 if not id: 515 id = 'top-level' 516 517 if not self.examples.has_key(self.example.section): 518 self.examples[id] = [(rule, source)] 519 else: 520 self.examples[id].append((rule, source)) 521 522 if pop: 523 if pop < 0: 524 pop = howmany 525 del self.stack[-pop:] 526 527 if len(self.stack): 528 self.example = self.stack[-1] 529 530 def write_examples(self): 531 jam = open(os.path.join(self.config.dump_dir, 'Jamfile.v2'), 'w') 532 533 jam.write(''' 534import testing ; 535 536''') 537 538 for id,examples in self.examples.items(): 539 for i in range(len(examples)): 540 cpp = '%s%d.cpp' % (id, i) 541 542 jam.write('%s %s ;\n' % (examples[i][0], cpp)) 543 544 outfile = os.path.join(self.config.dump_dir, cpp) 545 print cpp, 546 try: 547 if open(outfile, 'r').read() == examples[i][1]: 548 print ' .. skip' 549 continue 550 except: 551 pass 552 553 open(outfile, 'w').write(examples[i][1]) 554 print ' .. written' 555 556 jam.close() 557 558 def build( 559 self 560 , howmany = 1 561 , pop = -1 562 , source_file = 'example.cpp' 563 , expect_error = False 564 , target_rule = 'obj' 565 , requirements = '' 566 , input = '' 567 , output = 'example_output' 568 ): 569 570 # Grab one example by default 571 if howmany == 'all': 572 howmany = len(self.stack) 573 574 source = '\n'.join( 575 self.prefix 576 + [str(x) for x in self.stack[-howmany:]] 577 ) 578 579 source = reduce(lambda s, f: f(s), self.preprocessors, source) 580 581 if pop: 582 if pop < 0: 583 pop = howmany 584 del self.stack[-pop:] 585 586 if len(self.stack): 587 self.example = self.stack[-1] 588 589 dir = tempfile.mkdtemp() 590 cpp = os.path.join(dir, source_file) 591 self._write_source(cpp, source) 592 self._write_jamfile( 593 dir 594 , target_rule = target_rule 595 , requirements = requirements 596 , input = input 597 , output = output 598 ) 599 600 cmd = 'bjam' 601 if self.config.bjam_options: 602 cmd += ' %s' % self.config.bjam_options 603 604 os.chdir(dir) 605 status, output = syscmd(cmd, expect_error) 606 607 if status or expect_error > 1: 608 print 609 if expect_error and expect_error < 2: 610 print 'Compilation failure expected, but none seen' 611 print '------------ begin offending source ------------' 612 print open(cpp).read() 613 print '------------ begin offending Jamfile -----------' 614 print open(os.path.join(dir, 'Jamroot')).read() 615 print '------------ end offending Jamfile -------------' 616 617 sys.stdout.flush() 618 else: 619 print '.', 620 sys.stdout.flush() 621 622 if status: return None 623 else: return BuildResult(dir) 624 625 def _write_jamfile(self, path, target_rule, requirements, input, output): 626 jamfile = open(os.path.join(path, 'Jamroot'), 'w') 627 contents = r""" 628import modules ; 629 630BOOST_ROOT = [ modules.peek : BOOST_ROOT ] ; 631use-project /boost : $(BOOST_ROOT) ; 632 633%s 634 635%s %s 636 : example.cpp %s 637 : <include>. 638 %s 639 %s 640 ; 641 """ % ( 642 '\n'.join(self.jam_prefix) 643 , target_rule 644 , output 645 , input 646 , ' '.join(['<include>%s' % d for d in self.includes]) 647 , requirements 648 ) 649 650 jamfile.write(contents) 651 652 def run_python( 653 self 654 , howmany = 1 655 , pop = -1 656 , module_path = [] 657 , expect_error = False 658 ): 659 # Grab one example by default 660 if howmany == 'all': 661 howmany = len(self.stack) 662 663 if module_path == None: module_path = [] 664 665 if isinstance(module_path, BuildResult) or type(module_path) == str: 666 module_path = [module_path] 667 668 module_path = map(lambda p: str(p), module_path) 669 670 source = '\n'.join( 671 self.prefix 672 + [str(x) for x in self.stack[-howmany:]] 673 ) 674 675 if pop: 676 if pop < 0: 677 pop = howmany 678 del self.stack[-pop:] 679 680 if len(self.stack): 681 self.example = self.stack[-1] 682 683 r = re.compile(r'^(>>>|\.\.\.) (.*)$', re.MULTILINE) 684 source = r.sub(r'\2', source) 685 py = self._source_file_path(source_file = None, source_suffix = 'py') 686 open(py, 'w').write(source) 687 688 old_path = os.getenv('PYTHONPATH') 689 if old_path == None: 690 pythonpath = ':'.join(module_path) 691 old_path = '' 692 else: 693 pythonpath = old_path + ':%s' % ':'.join(module_path) 694 695 os.putenv('PYTHONPATH', pythonpath) 696 status, output = syscmd('python %s' % py) 697 698 if status or expect_error > 1: 699 print 700 if expect_error and expect_error < 2: 701 print 'Compilation failure expected, but none seen' 702 print '------------ begin offending source ------------' 703 print open(py).read() 704 print '------------ end offending Jamfile -------------' 705 706 sys.stdout.flush() 707 else: 708 print '.', 709 sys.stdout.flush() 710 711 self.last_run_output = output 712 os.putenv('PYTHONPATH', old_path) 713 self._unlink(py) 714 715 def _write_source(self, filename, contents): 716 open(filename,'w').write(contents) 717 718 def _remove_source(self, source_path): 719 os.unlink(source_path) 720 721 def _source_file_path(self, source_file, source_suffix): 722 if source_file is None: 723 cpp = tempfile.mktemp(suffix=source_suffix) 724 else: 725 cpp = os.path.join(tempfile.gettempdir(), source_file) 726 return cpp 727 728 def _output_file_path(self, source_file, extension): 729 return tempfile.mktemp(suffix=extension) 730 731 def _unlink(self, file): 732 file = expand_vars(file) 733 if os.path.exists(file): 734 os.unlink(file) 735 736 def _launch(self, exe, stdin = None): 737 status, output = syscmd(exe, input = stdin) 738 self.last_run_output = output 739 740 def run_(self, howmany = 1, stdin = None, **kw): 741 new_kw = { 'options':[], 'extension':'.exe' } 742 new_kw.update(kw) 743 744 self.compile( 745 howmany 746 , built_handler = lambda exe: self._launch(exe, stdin = stdin) 747 , **new_kw 748 ) 749 750 def astext(self): 751 return "" 752 return '\n\n ---------------- Unhandled Fragment ------------ \n\n'.join( 753 [''] # generates a leading announcement 754 + [ unicode(s) for s in self.stack] 755 ) 756 757class DumpTranslator(CPlusPlusTranslator): 758 example_index = 1 759 760 def _source_file_path(self, source_file, source_suffix): 761 if source_file is None: 762 source_file = 'example%s%s' % (self.example_index, source_suffix) 763 self.example_index += 1 764 765 cpp = os.path.join(config.dump_dir, source_file) 766 return cpp 767 768 def _output_file_path(self, source_file, extension): 769 chapter = os.path.basename(config.dump_dir) 770 return '%%TEMP%%\metaprogram-%s-example%s%s' \ 771 % ( chapter, self.example_index - 1, extension) 772 773 def _remove_source(self, source_path): 774 pass 775 776 777class WorkaroundTranslator(DumpTranslator): 778 """Translator used to test/dump workaround examples for vc6 and vc7. Just 779 like a DumpTranslator except that we leave existing files alone. 780 781 Warning: not sensitive to changes in .rst source!! If you change the actual 782 examples in source files you will have to move the example files out of the 783 way and regenerate them, then re-incorporate the workarounds. 784 """ 785 def _write_source(self, filename, contents): 786 if not os.path.exists(filename): 787 DumpTranslator._write_source(self, filename, contents) 788 789class Config: 790 save_cpp = False 791 line_hash = '#' 792 show_expected_error_output = False 793 max_output_lines = None 794 795class Writer(litre.Writer): 796 translator = CPlusPlusTranslator 797 798 def __init__( 799 self 800 , config 801 ): 802 litre.Writer.__init__(self) 803 self._config = Config() 804 defaults = Config.__dict__ 805 806 # update config elements 807 self._config.__dict__.update(config.__dict__) 808# dict([i for i in config.__dict__.items() 809# if i[0] in config.__all__])) 810 811