1"""Test suite for 2to3's parser and grammar files. 2 3This is the place to add tests for changes to 2to3's grammar, such as those 4merging the grammars for Python 2 and 3. In addition to specific tests for 5parts of the grammar we've changed, we also make sure we can parse the 6test_grammar.py files from both Python 2 and Python 3. 7""" 8 9# Testing imports 10from . import support 11from .support import driver, driver_no_print_statement 12 13# Python imports 14import difflib 15import importlib 16import operator 17import os 18import pickle 19import shutil 20import subprocess 21import sys 22import tempfile 23import unittest 24 25# Local imports 26from lib2to3.pgen2 import driver as pgen2_driver 27from lib2to3.pgen2 import tokenize 28from ..pgen2.parse import ParseError 29from lib2to3.pygram import python_symbols as syms 30 31 32class TestDriver(support.TestCase): 33 34 def test_formfeed(self): 35 s = """print 1\n\x0Cprint 2\n""" 36 t = driver.parse_string(s) 37 self.assertEqual(t.children[0].children[0].type, syms.print_stmt) 38 self.assertEqual(t.children[1].children[0].type, syms.print_stmt) 39 40 41class TestPgen2Caching(support.TestCase): 42 def test_load_grammar_from_txt_file(self): 43 pgen2_driver.load_grammar(support.grammar_path, save=False, force=True) 44 45 def test_load_grammar_from_pickle(self): 46 # Make a copy of the grammar file in a temp directory we are 47 # guaranteed to be able to write to. 48 tmpdir = tempfile.mkdtemp() 49 try: 50 grammar_copy = os.path.join( 51 tmpdir, os.path.basename(support.grammar_path)) 52 shutil.copy(support.grammar_path, grammar_copy) 53 pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) 54 55 pgen2_driver.load_grammar(grammar_copy, save=True, force=True) 56 self.assertTrue(os.path.exists(pickle_name)) 57 58 os.unlink(grammar_copy) # Only the pickle remains... 59 pgen2_driver.load_grammar(grammar_copy, save=False, force=False) 60 finally: 61 shutil.rmtree(tmpdir) 62 63 @unittest.skipIf(sys.executable is None, 'sys.executable required') 64 def test_load_grammar_from_subprocess(self): 65 tmpdir = tempfile.mkdtemp() 66 tmpsubdir = os.path.join(tmpdir, 'subdir') 67 try: 68 os.mkdir(tmpsubdir) 69 grammar_base = os.path.basename(support.grammar_path) 70 grammar_copy = os.path.join(tmpdir, grammar_base) 71 grammar_sub_copy = os.path.join(tmpsubdir, grammar_base) 72 shutil.copy(support.grammar_path, grammar_copy) 73 shutil.copy(support.grammar_path, grammar_sub_copy) 74 pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) 75 pickle_sub_name = pgen2_driver._generate_pickle_name( 76 grammar_sub_copy) 77 self.assertNotEqual(pickle_name, pickle_sub_name) 78 79 # Generate a pickle file from this process. 80 pgen2_driver.load_grammar(grammar_copy, save=True, force=True) 81 self.assertTrue(os.path.exists(pickle_name)) 82 83 # Generate a new pickle file in a subprocess with a most likely 84 # different hash randomization seed. 85 sub_env = dict(os.environ) 86 sub_env['PYTHONHASHSEED'] = 'random' 87 subprocess.check_call( 88 [sys.executable, '-c', """ 89from lib2to3.pgen2 import driver as pgen2_driver 90pgen2_driver.load_grammar(%r, save=True, force=True) 91 """ % (grammar_sub_copy,)], 92 env=sub_env) 93 self.assertTrue(os.path.exists(pickle_sub_name)) 94 95 with open(pickle_name, 'rb') as pickle_f_1, \ 96 open(pickle_sub_name, 'rb') as pickle_f_2: 97 self.assertEqual( 98 pickle_f_1.read(), pickle_f_2.read(), 99 msg='Grammar caches generated using different hash seeds' 100 ' were not identical.') 101 finally: 102 shutil.rmtree(tmpdir) 103 104 def test_load_packaged_grammar(self): 105 modname = __name__ + '.load_test' 106 class MyLoader: 107 def get_data(self, where): 108 return pickle.dumps({'elephant': 19}) 109 class MyModule: 110 __file__ = 'parsertestmodule' 111 __spec__ = importlib.util.spec_from_loader(modname, MyLoader()) 112 sys.modules[modname] = MyModule() 113 self.addCleanup(operator.delitem, sys.modules, modname) 114 g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt') 115 self.assertEqual(g.elephant, 19) 116 117 118class GrammarTest(support.TestCase): 119 def validate(self, code): 120 support.parse_string(code) 121 122 def invalid_syntax(self, code): 123 try: 124 self.validate(code) 125 except ParseError: 126 pass 127 else: 128 raise AssertionError("Syntax shouldn't have been valid") 129 130 131class TestMatrixMultiplication(GrammarTest): 132 def test_matrix_multiplication_operator(self): 133 self.validate("a @ b") 134 self.validate("a @= b") 135 136 137class TestYieldFrom(GrammarTest): 138 def test_yield_from(self): 139 self.validate("yield from x") 140 self.validate("(yield from x) + y") 141 self.invalid_syntax("yield from") 142 143 144class TestAsyncAwait(GrammarTest): 145 def test_await_expr(self): 146 self.validate("""async def foo(): 147 await x 148 """) 149 150 self.validate("""async def foo(): 151 [i async for i in b] 152 """) 153 154 self.validate("""async def foo(): 155 {i for i in b 156 async for i in a if await i 157 for b in i} 158 """) 159 160 self.validate("""async def foo(): 161 [await i for i in b if await c] 162 """) 163 164 self.validate("""async def foo(): 165 [ i for i in b if c] 166 """) 167 168 self.validate("""async def foo(): 169 170 def foo(): pass 171 172 def foo(): pass 173 174 await x 175 """) 176 177 self.validate("""async def foo(): return await a""") 178 179 self.validate("""def foo(): 180 def foo(): pass 181 async def foo(): await x 182 """) 183 184 self.invalid_syntax("await x") 185 self.invalid_syntax("""def foo(): 186 await x""") 187 188 self.invalid_syntax("""def foo(): 189 def foo(): pass 190 async def foo(): pass 191 await x 192 """) 193 194 def test_async_var(self): 195 self.validate("""async = 1""") 196 self.validate("""await = 1""") 197 self.validate("""def async(): pass""") 198 199 def test_async_with(self): 200 self.validate("""async def foo(): 201 async for a in b: pass""") 202 203 self.invalid_syntax("""def foo(): 204 async for a in b: pass""") 205 206 def test_async_for(self): 207 self.validate("""async def foo(): 208 async with a: pass""") 209 210 self.invalid_syntax("""def foo(): 211 async with a: pass""") 212 213 214class TestRaiseChanges(GrammarTest): 215 def test_2x_style_1(self): 216 self.validate("raise") 217 218 def test_2x_style_2(self): 219 self.validate("raise E, V") 220 221 def test_2x_style_3(self): 222 self.validate("raise E, V, T") 223 224 def test_2x_style_invalid_1(self): 225 self.invalid_syntax("raise E, V, T, Z") 226 227 def test_3x_style(self): 228 self.validate("raise E1 from E2") 229 230 def test_3x_style_invalid_1(self): 231 self.invalid_syntax("raise E, V from E1") 232 233 def test_3x_style_invalid_2(self): 234 self.invalid_syntax("raise E from E1, E2") 235 236 def test_3x_style_invalid_3(self): 237 self.invalid_syntax("raise from E1, E2") 238 239 def test_3x_style_invalid_4(self): 240 self.invalid_syntax("raise E from") 241 242 243# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292 244# and Lib/test/text_parser.py test_list_displays, test_set_displays, 245# test_dict_displays, test_argument_unpacking, ... changes. 246class TestUnpackingGeneralizations(GrammarTest): 247 def test_mid_positional_star(self): 248 self.validate("""func(1, *(2, 3), 4)""") 249 250 def test_double_star_dict_literal(self): 251 self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""") 252 253 def test_double_star_dict_literal_after_keywords(self): 254 self.validate("""func(spam='fried', **{'eggs':'scrambled'})""") 255 256 def test_double_star_expression(self): 257 self.validate("""func(**{'a':2} or {})""") 258 self.validate("""func(**() or {})""") 259 260 def test_star_expression(self): 261 self.validate("""func(*[] or [2])""") 262 263 def test_list_display(self): 264 self.validate("""[*{2}, 3, *[4]]""") 265 266 def test_set_display(self): 267 self.validate("""{*{2}, 3, *[4]}""") 268 269 def test_dict_display_1(self): 270 self.validate("""{**{}}""") 271 272 def test_dict_display_2(self): 273 self.validate("""{**{}, 3:4, **{5:6, 7:8}}""") 274 275 def test_argument_unpacking_1(self): 276 self.validate("""f(a, *b, *c, d)""") 277 278 def test_argument_unpacking_2(self): 279 self.validate("""f(**a, **b)""") 280 281 def test_argument_unpacking_3(self): 282 self.validate("""f(2, *a, *b, **b, **c, **d)""") 283 284 def test_trailing_commas_1(self): 285 self.validate("def f(a, b): call(a, b)") 286 self.validate("def f(a, b,): call(a, b,)") 287 288 def test_trailing_commas_2(self): 289 self.validate("def f(a, *b): call(a, *b)") 290 self.validate("def f(a, *b,): call(a, *b,)") 291 292 def test_trailing_commas_3(self): 293 self.validate("def f(a, b=1): call(a, b=1)") 294 self.validate("def f(a, b=1,): call(a, b=1,)") 295 296 def test_trailing_commas_4(self): 297 self.validate("def f(a, **b): call(a, **b)") 298 self.validate("def f(a, **b,): call(a, **b,)") 299 300 def test_trailing_commas_5(self): 301 self.validate("def f(*a, b=1): call(*a, b=1)") 302 self.validate("def f(*a, b=1,): call(*a, b=1,)") 303 304 def test_trailing_commas_6(self): 305 self.validate("def f(*a, **b): call(*a, **b)") 306 self.validate("def f(*a, **b,): call(*a, **b,)") 307 308 def test_trailing_commas_7(self): 309 self.validate("def f(*, b=1): call(*b)") 310 self.validate("def f(*, b=1,): call(*b,)") 311 312 def test_trailing_commas_8(self): 313 self.validate("def f(a=1, b=2): call(a=1, b=2)") 314 self.validate("def f(a=1, b=2,): call(a=1, b=2,)") 315 316 def test_trailing_commas_9(self): 317 self.validate("def f(a=1, **b): call(a=1, **b)") 318 self.validate("def f(a=1, **b,): call(a=1, **b,)") 319 320 def test_trailing_commas_lambda_1(self): 321 self.validate("f = lambda a, b: call(a, b)") 322 self.validate("f = lambda a, b,: call(a, b,)") 323 324 def test_trailing_commas_lambda_2(self): 325 self.validate("f = lambda a, *b: call(a, *b)") 326 self.validate("f = lambda a, *b,: call(a, *b,)") 327 328 def test_trailing_commas_lambda_3(self): 329 self.validate("f = lambda a, b=1: call(a, b=1)") 330 self.validate("f = lambda a, b=1,: call(a, b=1,)") 331 332 def test_trailing_commas_lambda_4(self): 333 self.validate("f = lambda a, **b: call(a, **b)") 334 self.validate("f = lambda a, **b,: call(a, **b,)") 335 336 def test_trailing_commas_lambda_5(self): 337 self.validate("f = lambda *a, b=1: call(*a, b=1)") 338 self.validate("f = lambda *a, b=1,: call(*a, b=1,)") 339 340 def test_trailing_commas_lambda_6(self): 341 self.validate("f = lambda *a, **b: call(*a, **b)") 342 self.validate("f = lambda *a, **b,: call(*a, **b,)") 343 344 def test_trailing_commas_lambda_7(self): 345 self.validate("f = lambda *, b=1: call(*b)") 346 self.validate("f = lambda *, b=1,: call(*b,)") 347 348 def test_trailing_commas_lambda_8(self): 349 self.validate("f = lambda a=1, b=2: call(a=1, b=2)") 350 self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)") 351 352 def test_trailing_commas_lambda_9(self): 353 self.validate("f = lambda a=1, **b: call(a=1, **b)") 354 self.validate("f = lambda a=1, **b,: call(a=1, **b,)") 355 356 357# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef 358class TestFunctionAnnotations(GrammarTest): 359 def test_1(self): 360 self.validate("""def f(x) -> list: pass""") 361 362 def test_2(self): 363 self.validate("""def f(x:int): pass""") 364 365 def test_3(self): 366 self.validate("""def f(*x:str): pass""") 367 368 def test_4(self): 369 self.validate("""def f(**x:float): pass""") 370 371 def test_5(self): 372 self.validate("""def f(x, y:1+2): pass""") 373 374 def test_6(self): 375 self.validate("""def f(a, (b:1, c:2, d)): pass""") 376 377 def test_7(self): 378 self.validate("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""") 379 380 def test_8(self): 381 s = """def f(a, (b:1, c:2, d), e:3=4, f=5, 382 *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" 383 self.validate(s) 384 385 def test_9(self): 386 s = """def f( 387 a: str, 388 b: int, 389 *, 390 c: bool = False, 391 **kwargs, 392 ) -> None: 393 call(c=c, **kwargs,)""" 394 self.validate(s) 395 396 def test_10(self): 397 s = """def f( 398 a: str, 399 ) -> None: 400 call(a,)""" 401 self.validate(s) 402 403 def test_11(self): 404 s = """def f( 405 a: str = '', 406 ) -> None: 407 call(a=a,)""" 408 self.validate(s) 409 410 def test_12(self): 411 s = """def f( 412 *args: str, 413 ) -> None: 414 call(*args,)""" 415 self.validate(s) 416 417 def test_13(self): 418 self.validate("def f(a: str, b: int) -> None: call(a, b)") 419 self.validate("def f(a: str, b: int,) -> None: call(a, b,)") 420 421 def test_14(self): 422 self.validate("def f(a: str, *b: int) -> None: call(a, *b)") 423 self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)") 424 425 def test_15(self): 426 self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)") 427 self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)") 428 429 def test_16(self): 430 self.validate("def f(a: str, **b: int) -> None: call(a, **b)") 431 self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)") 432 433 def test_17(self): 434 self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)") 435 self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)") 436 437 def test_18(self): 438 self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)") 439 self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)") 440 441 def test_19(self): 442 self.validate("def f(*, b: int=1) -> None: call(*b)") 443 self.validate("def f(*, b: int=1,) -> None: call(*b,)") 444 445 def test_20(self): 446 self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)") 447 self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)") 448 449 def test_21(self): 450 self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)") 451 self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)") 452 453 454# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot 455class TestVarAnnotations(GrammarTest): 456 def test_1(self): 457 self.validate("var1: int = 5") 458 459 def test_2(self): 460 self.validate("var2: [int, str]") 461 462 def test_3(self): 463 self.validate("def f():\n" 464 " st: str = 'Hello'\n" 465 " a.b: int = (1, 2)\n" 466 " return st\n") 467 468 def test_4(self): 469 self.validate("def fbad():\n" 470 " x: int\n" 471 " print(x)\n") 472 473 def test_5(self): 474 self.validate("class C:\n" 475 " x: int\n" 476 " s: str = 'attr'\n" 477 " z = 2\n" 478 " def __init__(self, x):\n" 479 " self.x: int = x\n") 480 481 def test_6(self): 482 self.validate("lst: List[int] = []") 483 484 485class TestExcept(GrammarTest): 486 def test_new(self): 487 s = """ 488 try: 489 x 490 except E as N: 491 y""" 492 self.validate(s) 493 494 def test_old(self): 495 s = """ 496 try: 497 x 498 except E, N: 499 y""" 500 self.validate(s) 501 502 503class TestStringLiterals(GrammarTest): 504 prefixes = ("'", '"', 505 "r'", 'r"', "R'", 'R"', 506 "u'", 'u"', "U'", 'U"', 507 "b'", 'b"', "B'", 'B"', 508 "f'", 'f"', "F'", 'F"', 509 "ur'", 'ur"', "Ur'", 'Ur"', 510 "uR'", 'uR"', "UR'", 'UR"', 511 "br'", 'br"', "Br'", 'Br"', 512 "bR'", 'bR"', "BR'", 'BR"', 513 "rb'", 'rb"', "Rb'", 'Rb"', 514 "rB'", 'rB"', "RB'", 'RB"',) 515 516 def test_lit(self): 517 for pre in self.prefixes: 518 single = "{p}spamspamspam{s}".format(p=pre, s=pre[-1]) 519 self.validate(single) 520 triple = "{p}{s}{s}eggs{s}{s}{s}".format(p=pre, s=pre[-1]) 521 self.validate(triple) 522 523 524# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms 525class TestSetLiteral(GrammarTest): 526 def test_1(self): 527 self.validate("""x = {'one'}""") 528 529 def test_2(self): 530 self.validate("""x = {'one', 1,}""") 531 532 def test_3(self): 533 self.validate("""x = {'one', 'two', 'three'}""") 534 535 def test_4(self): 536 self.validate("""x = {2, 3, 4,}""") 537 538 539# Adapted from Python 3's Lib/test/test_unicode_identifiers.py and 540# Lib/test/test_tokenize.py:TokenizeTest.test_non_ascii_identifiers 541class TestIdentfier(GrammarTest): 542 def test_non_ascii_identifiers(self): 543 self.validate("Örter = 'places'\ngrün = 'green'") 544 self.validate("蟒 = a蟒 = 锦蛇 = 1") 545 self.validate("µ = aµ = µµ = 1") 546 self.validate("�������������� = a_�������������� = 1") 547 548 549class TestNumericLiterals(GrammarTest): 550 def test_new_octal_notation(self): 551 self.validate("""0o7777777777777""") 552 self.invalid_syntax("""0o7324528887""") 553 554 def test_new_binary_notation(self): 555 self.validate("""0b101010""") 556 self.invalid_syntax("""0b0101021""") 557 558 559class TestClassDef(GrammarTest): 560 def test_new_syntax(self): 561 self.validate("class B(t=7): pass") 562 self.validate("class B(t, *args): pass") 563 self.validate("class B(t, **kwargs): pass") 564 self.validate("class B(t, *args, **kwargs): pass") 565 self.validate("class B(t, y=9, *args, **kwargs,): pass") 566 567 568class TestParserIdempotency(support.TestCase): 569 570 """A cut-down version of pytree_idempotency.py.""" 571 572 def test_all_project_files(self): 573 for filepath in support.all_project_files(): 574 with open(filepath, "rb") as fp: 575 encoding = tokenize.detect_encoding(fp.readline)[0] 576 self.assertIsNotNone(encoding, 577 "can't detect encoding for %s" % filepath) 578 with open(filepath, "r", encoding=encoding) as fp: 579 source = fp.read() 580 try: 581 tree = driver.parse_string(source) 582 except ParseError: 583 try: 584 tree = driver_no_print_statement.parse_string(source) 585 except ParseError as err: 586 self.fail('ParseError on file %s (%s)' % (filepath, err)) 587 new = str(tree) 588 if new != source: 589 print(diff_texts(source, new, filepath)) 590 self.fail("Idempotency failed: %s" % filepath) 591 592 def test_extended_unpacking(self): 593 driver.parse_string("a, *b, c = x\n") 594 driver.parse_string("[*a, b] = x\n") 595 driver.parse_string("(z, *y, w) = m\n") 596 driver.parse_string("for *z, m in d: pass\n") 597 598 599class TestLiterals(GrammarTest): 600 601 def validate(self, s): 602 driver.parse_string(support.dedent(s) + "\n\n") 603 604 def test_multiline_bytes_literals(self): 605 s = """ 606 md5test(b"\xaa" * 80, 607 (b"Test Using Larger Than Block-Size Key " 608 b"and Larger Than One Block-Size Data"), 609 "6f630fad67cda0ee1fb1f562db3aa53e") 610 """ 611 self.validate(s) 612 613 def test_multiline_bytes_tripquote_literals(self): 614 s = ''' 615 b""" 616 <?xml version="1.0" encoding="UTF-8"?> 617 <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN"> 618 """ 619 ''' 620 self.validate(s) 621 622 def test_multiline_str_literals(self): 623 s = """ 624 md5test("\xaa" * 80, 625 ("Test Using Larger Than Block-Size Key " 626 "and Larger Than One Block-Size Data"), 627 "6f630fad67cda0ee1fb1f562db3aa53e") 628 """ 629 self.validate(s) 630 631 632class TestPickleableException(unittest.TestCase): 633 def test_ParseError(self): 634 err = ParseError('msg', 2, None, (1, 'context')) 635 for proto in range(pickle.HIGHEST_PROTOCOL + 1): 636 err2 = pickle.loads(pickle.dumps(err, protocol=proto)) 637 self.assertEqual(err.args, err2.args) 638 self.assertEqual(err.msg, err2.msg) 639 self.assertEqual(err.type, err2.type) 640 self.assertEqual(err.value, err2.value) 641 self.assertEqual(err.context, err2.context) 642 643 644def diff_texts(a, b, filename): 645 a = a.splitlines() 646 b = b.splitlines() 647 return difflib.unified_diff(a, b, filename, filename, 648 "(original)", "(reserialized)", 649 lineterm="") 650 651 652if __name__ == '__main__': 653 unittest.main() 654