1import copy 2import parser 3import pickle 4import unittest 5import operator 6import struct 7from test import support 8from test.support.script_helper import assert_python_failure 9 10# 11# First, we test that we can generate trees from valid source fragments, 12# and that these valid trees are indeed allowed by the tree-loading side 13# of the parser module. 14# 15 16class RoundtripLegalSyntaxTestCase(unittest.TestCase): 17 18 def roundtrip(self, f, s): 19 st1 = f(s) 20 t = st1.totuple() 21 try: 22 st2 = parser.sequence2st(t) 23 except parser.ParserError as why: 24 self.fail("could not roundtrip %r: %s" % (s, why)) 25 26 self.assertEqual(t, st2.totuple(), 27 "could not re-generate syntax tree") 28 29 def check_expr(self, s): 30 self.roundtrip(parser.expr, s) 31 32 def test_flags_passed(self): 33 # The unicode literals flags has to be passed from the parser to AST 34 # generation. 35 suite = parser.suite("from __future__ import unicode_literals; x = ''") 36 code = suite.compile() 37 scope = {} 38 exec(code, {}, scope) 39 self.assertIsInstance(scope["x"], str) 40 41 def check_suite(self, s): 42 self.roundtrip(parser.suite, s) 43 44 def test_yield_statement(self): 45 self.check_suite("def f(): yield 1") 46 self.check_suite("def f(): yield") 47 self.check_suite("def f(): x += yield") 48 self.check_suite("def f(): x = yield 1") 49 self.check_suite("def f(): x = y = yield 1") 50 self.check_suite("def f(): x = yield") 51 self.check_suite("def f(): x = y = yield") 52 self.check_suite("def f(): 1 + (yield)*2") 53 self.check_suite("def f(): (yield 1)*2") 54 self.check_suite("def f(): return; yield 1") 55 self.check_suite("def f(): yield 1; return") 56 self.check_suite("def f(): yield from 1") 57 self.check_suite("def f(): x = yield from 1") 58 self.check_suite("def f(): f((yield from 1))") 59 self.check_suite("def f(): yield 1; return 1") 60 self.check_suite("def f():\n" 61 " for x in range(30):\n" 62 " yield x\n") 63 self.check_suite("def f():\n" 64 " if (yield):\n" 65 " yield x\n") 66 67 def test_await_statement(self): 68 self.check_suite("async def f():\n await smth()") 69 self.check_suite("async def f():\n foo = await smth()") 70 self.check_suite("async def f():\n foo, bar = await smth()") 71 self.check_suite("async def f():\n (await smth())") 72 self.check_suite("async def f():\n foo((await smth()))") 73 self.check_suite("async def f():\n await foo(); return 42") 74 75 def test_async_with_statement(self): 76 self.check_suite("async def f():\n async with 1: pass") 77 self.check_suite("async def f():\n async with a as b, c as d: pass") 78 79 def test_async_for_statement(self): 80 self.check_suite("async def f():\n async for i in (): pass") 81 self.check_suite("async def f():\n async for i, b in (): pass") 82 83 def test_nonlocal_statement(self): 84 self.check_suite("def f():\n" 85 " x = 0\n" 86 " def g():\n" 87 " nonlocal x\n") 88 self.check_suite("def f():\n" 89 " x = y = 0\n" 90 " def g():\n" 91 " nonlocal x, y\n") 92 93 def test_expressions(self): 94 self.check_expr("foo(1)") 95 self.check_expr("[1, 2, 3]") 96 self.check_expr("[x**3 for x in range(20)]") 97 self.check_expr("[x**3 for x in range(20) if x % 3]") 98 self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]") 99 self.check_expr("list(x**3 for x in range(20))") 100 self.check_expr("list(x**3 for x in range(20) if x % 3)") 101 self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)") 102 self.check_expr("foo(*args)") 103 self.check_expr("foo(*args, **kw)") 104 self.check_expr("foo(**kw)") 105 self.check_expr("foo(key=value)") 106 self.check_expr("foo(key=value, *args)") 107 self.check_expr("foo(key=value, *args, **kw)") 108 self.check_expr("foo(key=value, **kw)") 109 self.check_expr("foo(a, b, c, *args)") 110 self.check_expr("foo(a, b, c, *args, **kw)") 111 self.check_expr("foo(a, b, c, **kw)") 112 self.check_expr("foo(a, *args, keyword=23)") 113 self.check_expr("foo + bar") 114 self.check_expr("foo - bar") 115 self.check_expr("foo * bar") 116 self.check_expr("foo / bar") 117 self.check_expr("foo // bar") 118 self.check_expr("(foo := 1)") 119 self.check_expr("lambda: 0") 120 self.check_expr("lambda x: 0") 121 self.check_expr("lambda *y: 0") 122 self.check_expr("lambda *y, **z: 0") 123 self.check_expr("lambda **z: 0") 124 self.check_expr("lambda x, y: 0") 125 self.check_expr("lambda foo=bar: 0") 126 self.check_expr("lambda foo=bar, spaz=nifty+spit: 0") 127 self.check_expr("lambda foo=bar, **z: 0") 128 self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0") 129 self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0") 130 self.check_expr("lambda x, *y, **z: 0") 131 self.check_expr("(x for x in range(10))") 132 self.check_expr("foo(x for x in range(10))") 133 self.check_expr("...") 134 self.check_expr("a[...]") 135 136 def test_simple_expression(self): 137 # expr_stmt 138 self.check_suite("a") 139 140 def test_simple_assignments(self): 141 self.check_suite("a = b") 142 self.check_suite("a = b = c = d = e") 143 144 def test_var_annot(self): 145 self.check_suite("x: int = 5") 146 self.check_suite("y: List[T] = []; z: [list] = fun()") 147 self.check_suite("x: tuple = (1, 2)") 148 self.check_suite("d[f()]: int = 42") 149 self.check_suite("f(d[x]): str = 'abc'") 150 self.check_suite("x.y.z.w: complex = 42j") 151 self.check_suite("x: int") 152 self.check_suite("def f():\n" 153 " x: str\n" 154 " y: int = 5\n") 155 self.check_suite("class C:\n" 156 " x: str\n" 157 " y: int = 5\n") 158 self.check_suite("class C:\n" 159 " def __init__(self, x: int) -> None:\n" 160 " self.x: int = x\n") 161 # double check for nonsense 162 with self.assertRaises(SyntaxError): 163 exec("2+2: int", {}, {}) 164 with self.assertRaises(SyntaxError): 165 exec("[]: int = 5", {}, {}) 166 with self.assertRaises(SyntaxError): 167 exec("x, *y, z: int = range(5)", {}, {}) 168 with self.assertRaises(SyntaxError): 169 exec("x: int = 1, y = 2", {}, {}) 170 with self.assertRaises(SyntaxError): 171 exec("u = v: int", {}, {}) 172 with self.assertRaises(SyntaxError): 173 exec("False: int", {}, {}) 174 with self.assertRaises(SyntaxError): 175 exec("x.False: int", {}, {}) 176 with self.assertRaises(SyntaxError): 177 exec("x.y,: int", {}, {}) 178 with self.assertRaises(SyntaxError): 179 exec("[0]: int", {}, {}) 180 with self.assertRaises(SyntaxError): 181 exec("f(): int", {}, {}) 182 183 def test_simple_augmented_assignments(self): 184 self.check_suite("a += b") 185 self.check_suite("a -= b") 186 self.check_suite("a *= b") 187 self.check_suite("a /= b") 188 self.check_suite("a //= b") 189 self.check_suite("a %= b") 190 self.check_suite("a &= b") 191 self.check_suite("a |= b") 192 self.check_suite("a ^= b") 193 self.check_suite("a <<= b") 194 self.check_suite("a >>= b") 195 self.check_suite("a **= b") 196 197 def test_function_defs(self): 198 self.check_suite("def f(): pass") 199 self.check_suite("def f(*args): pass") 200 self.check_suite("def f(*args, **kw): pass") 201 self.check_suite("def f(**kw): pass") 202 self.check_suite("def f(foo=bar): pass") 203 self.check_suite("def f(foo=bar, *args): pass") 204 self.check_suite("def f(foo=bar, *args, **kw): pass") 205 self.check_suite("def f(foo=bar, **kw): pass") 206 207 self.check_suite("def f(a, b): pass") 208 self.check_suite("def f(a, b, *args): pass") 209 self.check_suite("def f(a, b, *args, **kw): pass") 210 self.check_suite("def f(a, b, **kw): pass") 211 self.check_suite("def f(a, b, foo=bar): pass") 212 self.check_suite("def f(a, b, foo=bar, *args): pass") 213 self.check_suite("def f(a, b, foo=bar, *args, **kw): pass") 214 self.check_suite("def f(a, b, foo=bar, **kw): pass") 215 216 self.check_suite("@staticmethod\n" 217 "def f(): pass") 218 self.check_suite("@staticmethod\n" 219 "@funcattrs(x, y)\n" 220 "def f(): pass") 221 self.check_suite("@funcattrs()\n" 222 "def f(): pass") 223 224 # keyword-only arguments 225 self.check_suite("def f(*, a): pass") 226 self.check_suite("def f(*, a = 5): pass") 227 self.check_suite("def f(*, a = 5, b): pass") 228 self.check_suite("def f(*, a, b = 5): pass") 229 self.check_suite("def f(*, a, b = 5, **kwds): pass") 230 self.check_suite("def f(*args, a): pass") 231 self.check_suite("def f(*args, a = 5): pass") 232 self.check_suite("def f(*args, a = 5, b): pass") 233 self.check_suite("def f(*args, a, b = 5): pass") 234 self.check_suite("def f(*args, a, b = 5, **kwds): pass") 235 236 # positional-only arguments 237 self.check_suite("def f(a, /): pass") 238 self.check_suite("def f(a, /,): pass") 239 self.check_suite("def f(a, b, /): pass") 240 self.check_suite("def f(a, b, /, c): pass") 241 self.check_suite("def f(a, b, /, c = 6): pass") 242 self.check_suite("def f(a, b, /, c, *, d): pass") 243 self.check_suite("def f(a, b, /, c = 1, *, d): pass") 244 self.check_suite("def f(a, b, /, c, *, d = 1): pass") 245 self.check_suite("def f(a, b=1, /, c=2, *, d = 3): pass") 246 self.check_suite("def f(a=0, b=1, /, c=2, *, d = 3): pass") 247 248 # function annotations 249 self.check_suite("def f(a: int): pass") 250 self.check_suite("def f(a: int = 5): pass") 251 self.check_suite("def f(*args: list): pass") 252 self.check_suite("def f(**kwds: dict): pass") 253 self.check_suite("def f(*, a: int): pass") 254 self.check_suite("def f(*, a: int = 5): pass") 255 self.check_suite("def f() -> int: pass") 256 257 def test_class_defs(self): 258 self.check_suite("class foo():pass") 259 self.check_suite("class foo(object):pass") 260 self.check_suite("@class_decorator\n" 261 "class foo():pass") 262 self.check_suite("@class_decorator(arg)\n" 263 "class foo():pass") 264 self.check_suite("@decorator1\n" 265 "@decorator2\n" 266 "class foo():pass") 267 268 def test_import_from_statement(self): 269 self.check_suite("from sys.path import *") 270 self.check_suite("from sys.path import dirname") 271 self.check_suite("from sys.path import (dirname)") 272 self.check_suite("from sys.path import (dirname,)") 273 self.check_suite("from sys.path import dirname as my_dirname") 274 self.check_suite("from sys.path import (dirname as my_dirname)") 275 self.check_suite("from sys.path import (dirname as my_dirname,)") 276 self.check_suite("from sys.path import dirname, basename") 277 self.check_suite("from sys.path import (dirname, basename)") 278 self.check_suite("from sys.path import (dirname, basename,)") 279 self.check_suite( 280 "from sys.path import dirname as my_dirname, basename") 281 self.check_suite( 282 "from sys.path import (dirname as my_dirname, basename)") 283 self.check_suite( 284 "from sys.path import (dirname as my_dirname, basename,)") 285 self.check_suite( 286 "from sys.path import dirname, basename as my_basename") 287 self.check_suite( 288 "from sys.path import (dirname, basename as my_basename)") 289 self.check_suite( 290 "from sys.path import (dirname, basename as my_basename,)") 291 self.check_suite("from .bogus import x") 292 293 def test_basic_import_statement(self): 294 self.check_suite("import sys") 295 self.check_suite("import sys as system") 296 self.check_suite("import sys, math") 297 self.check_suite("import sys as system, math") 298 self.check_suite("import sys, math as my_math") 299 300 def test_relative_imports(self): 301 self.check_suite("from . import name") 302 self.check_suite("from .. import name") 303 # check all the way up to '....', since '...' is tokenized 304 # differently from '.' (it's an ellipsis token). 305 self.check_suite("from ... import name") 306 self.check_suite("from .... import name") 307 self.check_suite("from .pkg import name") 308 self.check_suite("from ..pkg import name") 309 self.check_suite("from ...pkg import name") 310 self.check_suite("from ....pkg import name") 311 312 def test_pep263(self): 313 self.check_suite("# -*- coding: iso-8859-1 -*-\n" 314 "pass\n") 315 316 def test_assert(self): 317 self.check_suite("assert alo < ahi and blo < bhi\n") 318 319 def test_with(self): 320 self.check_suite("with open('x'): pass\n") 321 self.check_suite("with open('x') as f: pass\n") 322 self.check_suite("with open('x') as f, open('y') as g: pass\n") 323 324 def test_try_stmt(self): 325 self.check_suite("try: pass\nexcept: pass\n") 326 self.check_suite("try: pass\nfinally: pass\n") 327 self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n") 328 self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n" 329 "finally: pass\n") 330 self.check_suite("try: pass\nexcept: pass\nelse: pass\n") 331 self.check_suite("try: pass\nexcept: pass\nelse: pass\n" 332 "finally: pass\n") 333 334 def test_if_stmt(self): 335 self.check_suite("if True:\n pass\nelse:\n pass\n") 336 self.check_suite("if True:\n pass\nelif True:\n pass\nelse:\n pass\n") 337 338 def test_position(self): 339 # An absolutely minimal test of position information. Better 340 # tests would be a big project. 341 code = "def f(x):\n return x + 1" 342 st = parser.suite(code) 343 344 def walk(tree): 345 node_type = tree[0] 346 next = tree[1] 347 if isinstance(next, (tuple, list)): 348 for elt in tree[1:]: 349 for x in walk(elt): 350 yield x 351 else: 352 yield tree 353 354 expected = [ 355 (1, 'def', 1, 0), 356 (1, 'f', 1, 4), 357 (7, '(', 1, 5), 358 (1, 'x', 1, 6), 359 (8, ')', 1, 7), 360 (11, ':', 1, 8), 361 (4, '', 1, 9), 362 (5, '', 2, -1), 363 (1, 'return', 2, 4), 364 (1, 'x', 2, 11), 365 (14, '+', 2, 13), 366 (2, '1', 2, 15), 367 (4, '', 2, 16), 368 (6, '', 2, -1), 369 (4, '', 2, -1), 370 (0, '', 2, -1), 371 ] 372 373 self.assertEqual(list(walk(st.totuple(line_info=True, col_info=True))), 374 expected) 375 self.assertEqual(list(walk(st.totuple())), 376 [(t, n) for t, n, l, c in expected]) 377 self.assertEqual(list(walk(st.totuple(line_info=True))), 378 [(t, n, l) for t, n, l, c in expected]) 379 self.assertEqual(list(walk(st.totuple(col_info=True))), 380 [(t, n, c) for t, n, l, c in expected]) 381 self.assertEqual(list(walk(st.tolist(line_info=True, col_info=True))), 382 [list(x) for x in expected]) 383 self.assertEqual(list(walk(parser.st2tuple(st, line_info=True, 384 col_info=True))), 385 expected) 386 self.assertEqual(list(walk(parser.st2list(st, line_info=True, 387 col_info=True))), 388 [list(x) for x in expected]) 389 390 def test_extended_unpacking(self): 391 self.check_suite("*a = y") 392 self.check_suite("x, *b, = m") 393 self.check_suite("[*a, *b] = y") 394 self.check_suite("for [*x, b] in x: pass") 395 396 def test_raise_statement(self): 397 self.check_suite("raise\n") 398 self.check_suite("raise e\n") 399 self.check_suite("try:\n" 400 " suite\n" 401 "except Exception as e:\n" 402 " raise ValueError from e\n") 403 404 def test_list_displays(self): 405 self.check_expr('[]') 406 self.check_expr('[*{2}, 3, *[4]]') 407 408 def test_set_displays(self): 409 self.check_expr('{*{2}, 3, *[4]}') 410 self.check_expr('{2}') 411 self.check_expr('{2,}') 412 self.check_expr('{2, 3}') 413 self.check_expr('{2, 3,}') 414 415 def test_dict_displays(self): 416 self.check_expr('{}') 417 self.check_expr('{a:b}') 418 self.check_expr('{a:b,}') 419 self.check_expr('{a:b, c:d}') 420 self.check_expr('{a:b, c:d,}') 421 self.check_expr('{**{}}') 422 self.check_expr('{**{}, 3:4, **{5:6, 7:8}}') 423 424 def test_argument_unpacking(self): 425 self.check_expr("f(*a, **b)") 426 self.check_expr('f(a, *b, *c, *d)') 427 self.check_expr('f(**a, **b)') 428 self.check_expr('f(2, *a, *b, **b, **c, **d)') 429 self.check_expr("f(*b, *() or () and (), **{} and {}, **() or {})") 430 431 def test_set_comprehensions(self): 432 self.check_expr('{x for x in seq}') 433 self.check_expr('{f(x) for x in seq}') 434 self.check_expr('{f(x) for x in seq if condition(x)}') 435 436 def test_dict_comprehensions(self): 437 self.check_expr('{x:x for x in seq}') 438 self.check_expr('{x**2:x[3] for x in seq if condition(x)}') 439 self.check_expr('{x:x for x in seq1 for y in seq2 if condition(x, y)}') 440 441 def test_named_expressions(self): 442 self.check_suite("(a := 1)") 443 self.check_suite("(a := a)") 444 self.check_suite("if (match := pattern.search(data)) is None: pass") 445 self.check_suite("while match := pattern.search(f.read()): pass") 446 self.check_suite("[y := f(x), y**2, y**3]") 447 self.check_suite("filtered_data = [y for x in data if (y := f(x)) is None]") 448 self.check_suite("(y := f(x))") 449 self.check_suite("y0 = (y1 := f(x))") 450 self.check_suite("foo(x=(y := f(x)))") 451 self.check_suite("def foo(answer=(p := 42)): pass") 452 self.check_suite("def foo(answer: (p := 42) = 5): pass") 453 self.check_suite("lambda: (x := 1)") 454 self.check_suite("(x := lambda: 1)") 455 self.check_suite("(x := lambda: (y := 1))") # not in PEP 456 self.check_suite("lambda line: (m := re.match(pattern, line)) and m.group(1)") 457 self.check_suite("x = (y := 0)") 458 self.check_suite("(z:=(y:=(x:=0)))") 459 self.check_suite("(info := (name, phone, *rest))") 460 self.check_suite("(x:=1,2)") 461 self.check_suite("(total := total + tax)") 462 self.check_suite("len(lines := f.readlines())") 463 self.check_suite("foo(x := 3, cat='vector')") 464 self.check_suite("foo(cat=(category := 'vector'))") 465 self.check_suite("if any(len(longline := l) >= 100 for l in lines): print(longline)") 466 self.check_suite( 467 "if env_base := os.environ.get('PYTHONUSERBASE', None): return env_base" 468 ) 469 self.check_suite( 470 "if self._is_special and (ans := self._check_nans(context=context)): return ans" 471 ) 472 self.check_suite("foo(b := 2, a=1)") 473 self.check_suite("foo(b := 2, a=1)") 474 self.check_suite("foo((b := 2), a=1)") 475 self.check_suite("foo(c=(b := 2), a=1)") 476 self.check_suite("{(x := C(i)).q: x for i in y}") 477 478 479# 480# Second, we take *invalid* trees and make sure we get ParserError 481# rejections for them. 482# 483 484class IllegalSyntaxTestCase(unittest.TestCase): 485 486 def check_bad_tree(self, tree, label): 487 try: 488 parser.sequence2st(tree) 489 except parser.ParserError: 490 pass 491 else: 492 self.fail("did not detect invalid tree for %r" % label) 493 494 def test_junk(self): 495 # not even remotely valid: 496 self.check_bad_tree((1, 2, 3), "<junk>") 497 498 def test_illegal_terminal(self): 499 tree = \ 500 (257, 501 (269, 502 (270, 503 (271, 504 (277, 505 (1,))), 506 (4, ''))), 507 (4, ''), 508 (0, '')) 509 self.check_bad_tree(tree, "too small items in terminal node") 510 tree = \ 511 (257, 512 (269, 513 (270, 514 (271, 515 (277, 516 (1, b'pass'))), 517 (4, ''))), 518 (4, ''), 519 (0, '')) 520 self.check_bad_tree(tree, "non-string second item in terminal node") 521 tree = \ 522 (257, 523 (269, 524 (270, 525 (271, 526 (277, 527 (1, 'pass', '0', 0))), 528 (4, ''))), 529 (4, ''), 530 (0, '')) 531 self.check_bad_tree(tree, "non-integer third item in terminal node") 532 tree = \ 533 (257, 534 (269, 535 (270, 536 (271, 537 (277, 538 (1, 'pass', 0, 0))), 539 (4, ''))), 540 (4, ''), 541 (0, '')) 542 self.check_bad_tree(tree, "too many items in terminal node") 543 544 def test_illegal_yield_1(self): 545 # Illegal yield statement: def f(): return 1; yield 1 546 tree = \ 547 (257, 548 (264, 549 (285, 550 (259, 551 (1, 'def'), 552 (1, 'f'), 553 (260, (7, '('), (8, ')')), 554 (11, ':'), 555 (291, 556 (4, ''), 557 (5, ''), 558 (264, 559 (265, 560 (266, 561 (272, 562 (275, 563 (1, 'return'), 564 (313, 565 (292, 566 (293, 567 (294, 568 (295, 569 (297, 570 (298, 571 (299, 572 (300, 573 (301, 574 (302, (303, (304, (305, (2, '1')))))))))))))))))), 575 (264, 576 (265, 577 (266, 578 (272, 579 (276, 580 (1, 'yield'), 581 (313, 582 (292, 583 (293, 584 (294, 585 (295, 586 (297, 587 (298, 588 (299, 589 (300, 590 (301, 591 (302, 592 (303, (304, (305, (2, '1')))))))))))))))))), 593 (4, ''))), 594 (6, ''))))), 595 (4, ''), 596 (0, '')))) 597 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") 598 599 def test_illegal_yield_2(self): 600 # Illegal return in generator: def f(): return 1; yield 1 601 tree = \ 602 (257, 603 (264, 604 (265, 605 (266, 606 (278, 607 (1, 'from'), 608 (281, (1, '__future__')), 609 (1, 'import'), 610 (279, (1, 'generators')))), 611 (4, ''))), 612 (264, 613 (285, 614 (259, 615 (1, 'def'), 616 (1, 'f'), 617 (260, (7, '('), (8, ')')), 618 (11, ':'), 619 (291, 620 (4, ''), 621 (5, ''), 622 (264, 623 (265, 624 (266, 625 (272, 626 (275, 627 (1, 'return'), 628 (313, 629 (292, 630 (293, 631 (294, 632 (295, 633 (297, 634 (298, 635 (299, 636 (300, 637 (301, 638 (302, (303, (304, (305, (2, '1')))))))))))))))))), 639 (264, 640 (265, 641 (266, 642 (272, 643 (276, 644 (1, 'yield'), 645 (313, 646 (292, 647 (293, 648 (294, 649 (295, 650 (297, 651 (298, 652 (299, 653 (300, 654 (301, 655 (302, 656 (303, (304, (305, (2, '1')))))))))))))))))), 657 (4, ''))), 658 (6, ''))))), 659 (4, ''), 660 (0, '')))) 661 self.check_bad_tree(tree, "def f():\n return 1\n yield 1") 662 663 def test_a_comma_comma_c(self): 664 # Illegal input: a,,c 665 tree = \ 666 (258, 667 (311, 668 (290, 669 (291, 670 (292, 671 (293, 672 (295, 673 (296, 674 (297, 675 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))), 676 (12, ','), 677 (12, ','), 678 (290, 679 (291, 680 (292, 681 (293, 682 (295, 683 (296, 684 (297, 685 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))), 686 (4, ''), 687 (0, '')) 688 self.check_bad_tree(tree, "a,,c") 689 690 def test_illegal_operator(self): 691 # Illegal input: a $= b 692 tree = \ 693 (257, 694 (264, 695 (265, 696 (266, 697 (267, 698 (312, 699 (291, 700 (292, 701 (293, 702 (294, 703 (296, 704 (297, 705 (298, 706 (299, 707 (300, (301, (302, (303, (304, (1, 'a'))))))))))))))), 708 (268, (37, '$=')), 709 (312, 710 (291, 711 (292, 712 (293, 713 (294, 714 (296, 715 (297, 716 (298, 717 (299, 718 (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))), 719 (4, ''))), 720 (0, '')) 721 self.check_bad_tree(tree, "a $= b") 722 723 def test_malformed_global(self): 724 #doesn't have global keyword in ast 725 tree = (257, 726 (264, 727 (265, 728 (266, 729 (282, (1, 'foo'))), (4, ''))), 730 (4, ''), 731 (0, '')) 732 self.check_bad_tree(tree, "malformed global ast") 733 734 def test_missing_import_source(self): 735 # from import fred 736 tree = \ 737 (257, 738 (268, 739 (269, 740 (270, 741 (282, 742 (284, (1, 'from'), (1, 'import'), 743 (287, (285, (1, 'fred')))))), 744 (4, ''))), 745 (4, ''), (0, '')) 746 self.check_bad_tree(tree, "from import fred") 747 748 def test_illegal_encoding(self): 749 # Illegal encoding declaration 750 tree = \ 751 (341, 752 (257, (0, ''))) 753 self.check_bad_tree(tree, "missed encoding") 754 tree = \ 755 (341, 756 (257, (0, '')), 757 b'iso-8859-1') 758 self.check_bad_tree(tree, "non-string encoding") 759 tree = \ 760 (341, 761 (257, (0, '')), 762 '\udcff') 763 with self.assertRaises(UnicodeEncodeError): 764 parser.sequence2st(tree) 765 766 def test_invalid_node_id(self): 767 tree = (257, (269, (-7, ''))) 768 self.check_bad_tree(tree, "negative node id") 769 tree = (257, (269, (99, ''))) 770 self.check_bad_tree(tree, "invalid token id") 771 tree = (257, (269, (9999, (0, '')))) 772 self.check_bad_tree(tree, "invalid symbol id") 773 774 def test_ParserError_message(self): 775 try: 776 parser.sequence2st((257,(269,(257,(0,''))))) 777 except parser.ParserError as why: 778 self.assertIn("compound_stmt", str(why)) # Expected 779 self.assertIn("file_input", str(why)) # Got 780 781 782 783class CompileTestCase(unittest.TestCase): 784 785 # These tests are very minimal. :-( 786 787 def test_compile_expr(self): 788 st = parser.expr('2 + 3') 789 code = parser.compilest(st) 790 self.assertEqual(eval(code), 5) 791 792 def test_compile_suite(self): 793 st = parser.suite('x = 2; y = x + 3') 794 code = parser.compilest(st) 795 globs = {} 796 exec(code, globs) 797 self.assertEqual(globs['y'], 5) 798 799 def test_compile_error(self): 800 st = parser.suite('1 = 3 + 4') 801 self.assertRaises(SyntaxError, parser.compilest, st) 802 803 def test_compile_badunicode(self): 804 st = parser.suite('a = "\\U12345678"') 805 self.assertRaises(SyntaxError, parser.compilest, st) 806 st = parser.suite('a = "\\u1"') 807 self.assertRaises(SyntaxError, parser.compilest, st) 808 809 def test_issue_9011(self): 810 # Issue 9011: compilation of an unary minus expression changed 811 # the meaning of the ST, so that a second compilation produced 812 # incorrect results. 813 st = parser.expr('-3') 814 code1 = parser.compilest(st) 815 self.assertEqual(eval(code1), -3) 816 code2 = parser.compilest(st) 817 self.assertEqual(eval(code2), -3) 818 819 def test_compile_filename(self): 820 st = parser.expr('a + 5') 821 code = parser.compilest(st) 822 self.assertEqual(code.co_filename, '<syntax-tree>') 823 code = st.compile() 824 self.assertEqual(code.co_filename, '<syntax-tree>') 825 for filename in 'file.py', b'file.py': 826 code = parser.compilest(st, filename) 827 self.assertEqual(code.co_filename, 'file.py') 828 code = st.compile(filename) 829 self.assertEqual(code.co_filename, 'file.py') 830 for filename in bytearray(b'file.py'), memoryview(b'file.py'): 831 with self.assertWarns(DeprecationWarning): 832 code = parser.compilest(st, filename) 833 self.assertEqual(code.co_filename, 'file.py') 834 with self.assertWarns(DeprecationWarning): 835 code = st.compile(filename) 836 self.assertEqual(code.co_filename, 'file.py') 837 self.assertRaises(TypeError, parser.compilest, st, list(b'file.py')) 838 self.assertRaises(TypeError, st.compile, list(b'file.py')) 839 840 841class ParserStackLimitTestCase(unittest.TestCase): 842 """try to push the parser to/over its limits. 843 see http://bugs.python.org/issue1881 for a discussion 844 """ 845 def _nested_expression(self, level): 846 return "["*level+"]"*level 847 848 def test_deeply_nested_list(self): 849 # This has fluctuated between 99 levels in 2.x, down to 93 levels in 850 # 3.7.X and back up to 99 in 3.8.X. Related to MAXSTACK size in Parser.h 851 e = self._nested_expression(99) 852 st = parser.expr(e) 853 st.compile() 854 855 def test_trigger_memory_error(self): 856 e = self._nested_expression(100) 857 rc, out, err = assert_python_failure('-c', e) 858 # parsing the expression will result in an error message 859 # followed by a MemoryError (see #11963) 860 self.assertIn(b's_push: parser stack overflow', err) 861 self.assertIn(b'MemoryError', err) 862 863class STObjectTestCase(unittest.TestCase): 864 """Test operations on ST objects themselves""" 865 866 def test_comparisons(self): 867 # ST objects should support order and equality comparisons 868 st1 = parser.expr('2 + 3') 869 st2 = parser.suite('x = 2; y = x + 3') 870 st3 = parser.expr('list(x**3 for x in range(20))') 871 st1_copy = parser.expr('2 + 3') 872 st2_copy = parser.suite('x = 2; y = x + 3') 873 st3_copy = parser.expr('list(x**3 for x in range(20))') 874 875 # exercise fast path for object identity 876 self.assertEqual(st1 == st1, True) 877 self.assertEqual(st2 == st2, True) 878 self.assertEqual(st3 == st3, True) 879 # slow path equality 880 self.assertEqual(st1, st1_copy) 881 self.assertEqual(st2, st2_copy) 882 self.assertEqual(st3, st3_copy) 883 self.assertEqual(st1 == st2, False) 884 self.assertEqual(st1 == st3, False) 885 self.assertEqual(st2 == st3, False) 886 self.assertEqual(st1 != st1, False) 887 self.assertEqual(st2 != st2, False) 888 self.assertEqual(st3 != st3, False) 889 self.assertEqual(st1 != st1_copy, False) 890 self.assertEqual(st2 != st2_copy, False) 891 self.assertEqual(st3 != st3_copy, False) 892 self.assertEqual(st2 != st1, True) 893 self.assertEqual(st1 != st3, True) 894 self.assertEqual(st3 != st2, True) 895 # we don't particularly care what the ordering is; just that 896 # it's usable and self-consistent 897 self.assertEqual(st1 < st2, not (st2 <= st1)) 898 self.assertEqual(st1 < st3, not (st3 <= st1)) 899 self.assertEqual(st2 < st3, not (st3 <= st2)) 900 self.assertEqual(st1 < st2, st2 > st1) 901 self.assertEqual(st1 < st3, st3 > st1) 902 self.assertEqual(st2 < st3, st3 > st2) 903 self.assertEqual(st1 <= st2, st2 >= st1) 904 self.assertEqual(st3 <= st1, st1 >= st3) 905 self.assertEqual(st2 <= st3, st3 >= st2) 906 # transitivity 907 bottom = min(st1, st2, st3) 908 top = max(st1, st2, st3) 909 mid = sorted([st1, st2, st3])[1] 910 self.assertTrue(bottom < mid) 911 self.assertTrue(bottom < top) 912 self.assertTrue(mid < top) 913 self.assertTrue(bottom <= mid) 914 self.assertTrue(bottom <= top) 915 self.assertTrue(mid <= top) 916 self.assertTrue(bottom <= bottom) 917 self.assertTrue(mid <= mid) 918 self.assertTrue(top <= top) 919 # interaction with other types 920 self.assertEqual(st1 == 1588.602459, False) 921 self.assertEqual('spanish armada' != st2, True) 922 self.assertRaises(TypeError, operator.ge, st3, None) 923 self.assertRaises(TypeError, operator.le, False, st1) 924 self.assertRaises(TypeError, operator.lt, st1, 1815) 925 self.assertRaises(TypeError, operator.gt, b'waterloo', st2) 926 927 def test_copy_pickle(self): 928 sts = [ 929 parser.expr('2 + 3'), 930 parser.suite('x = 2; y = x + 3'), 931 parser.expr('list(x**3 for x in range(20))') 932 ] 933 for st in sts: 934 st_copy = copy.copy(st) 935 self.assertEqual(st_copy.totuple(), st.totuple()) 936 st_copy = copy.deepcopy(st) 937 self.assertEqual(st_copy.totuple(), st.totuple()) 938 for proto in range(pickle.HIGHEST_PROTOCOL+1): 939 st_copy = pickle.loads(pickle.dumps(st, proto)) 940 self.assertEqual(st_copy.totuple(), st.totuple()) 941 942 check_sizeof = support.check_sizeof 943 944 @support.cpython_only 945 def test_sizeof(self): 946 def XXXROUNDUP(n): 947 if n <= 1: 948 return n 949 if n <= 128: 950 return (n + 3) & ~3 951 return 1 << (n - 1).bit_length() 952 953 basesize = support.calcobjsize('Piii') 954 nodesize = struct.calcsize('hP3iP0h2i') 955 def sizeofchildren(node): 956 if node is None: 957 return 0 958 res = 0 959 hasstr = len(node) > 1 and isinstance(node[-1], str) 960 if hasstr: 961 res += len(node[-1]) + 1 962 children = node[1:-1] if hasstr else node[1:] 963 if children: 964 res += XXXROUNDUP(len(children)) * nodesize 965 for child in children: 966 res += sizeofchildren(child) 967 return res 968 969 def check_st_sizeof(st): 970 self.check_sizeof(st, basesize + nodesize + 971 sizeofchildren(st.totuple())) 972 973 check_st_sizeof(parser.expr('2 + 3')) 974 check_st_sizeof(parser.expr('2 + 3 + 4')) 975 check_st_sizeof(parser.suite('x = 2 + 3')) 976 check_st_sizeof(parser.suite('')) 977 check_st_sizeof(parser.suite('# -*- coding: utf-8 -*-')) 978 check_st_sizeof(parser.expr('[' + '2,' * 1000 + ']')) 979 980 981 # XXX tests for pickling and unpickling of ST objects should go here 982 983class OtherParserCase(unittest.TestCase): 984 985 def test_two_args_to_expr(self): 986 # See bug #12264 987 with self.assertRaises(TypeError): 988 parser.expr("a", "b") 989 990if __name__ == "__main__": 991 unittest.main() 992