1import contextlib 2import dis 3import io 4import itertools 5import math 6import opcode 7import os 8import unittest 9import sys 10import ast 11import _ast 12import tempfile 13import types 14import textwrap 15import warnings 16import _testinternalcapi 17 18from test import support 19from test.support import (script_helper, requires_debug_ranges, run_code, 20 requires_specialization, get_c_recursion_limit) 21from test.support.bytecode_helper import instructions_with_positions 22from test.support.os_helper import FakePath 23 24class TestSpecifics(unittest.TestCase): 25 26 def compile_single(self, source): 27 compile(source, "<single>", "single") 28 29 def assertInvalidSingle(self, source): 30 self.assertRaises(SyntaxError, self.compile_single, source) 31 32 def test_no_ending_newline(self): 33 compile("hi", "<test>", "exec") 34 compile("hi\r", "<test>", "exec") 35 36 def test_empty(self): 37 compile("", "<test>", "exec") 38 39 def test_other_newlines(self): 40 compile("\r\n", "<test>", "exec") 41 compile("\r", "<test>", "exec") 42 compile("hi\r\nstuff\r\ndef f():\n pass\r", "<test>", "exec") 43 compile("this_is\rreally_old_mac\rdef f():\n pass", "<test>", "exec") 44 45 def test_debug_assignment(self): 46 # catch assignments to __debug__ 47 self.assertRaises(SyntaxError, compile, '__debug__ = 1', '?', 'single') 48 import builtins 49 prev = builtins.__debug__ 50 setattr(builtins, '__debug__', 'sure') 51 self.assertEqual(__debug__, prev) 52 setattr(builtins, '__debug__', prev) 53 54 def test_argument_handling(self): 55 # detect duplicate positional and keyword arguments 56 self.assertRaises(SyntaxError, eval, 'lambda a,a:0') 57 self.assertRaises(SyntaxError, eval, 'lambda a,a=1:0') 58 self.assertRaises(SyntaxError, eval, 'lambda a=1,a=1:0') 59 self.assertRaises(SyntaxError, exec, 'def f(a, a): pass') 60 self.assertRaises(SyntaxError, exec, 'def f(a = 0, a = 1): pass') 61 self.assertRaises(SyntaxError, exec, 'def f(a): global a; a = 1') 62 63 def test_syntax_error(self): 64 self.assertRaises(SyntaxError, compile, "1+*3", "filename", "exec") 65 66 def test_none_keyword_arg(self): 67 self.assertRaises(SyntaxError, compile, "f(None=1)", "<string>", "exec") 68 69 def test_duplicate_global_local(self): 70 self.assertRaises(SyntaxError, exec, 'def f(a): global a; a = 1') 71 72 def test_exec_with_general_mapping_for_locals(self): 73 74 class M: 75 "Test mapping interface versus possible calls from eval()." 76 def __getitem__(self, key): 77 if key == 'a': 78 return 12 79 raise KeyError 80 def __setitem__(self, key, value): 81 self.results = (key, value) 82 def keys(self): 83 return list('xyz') 84 85 m = M() 86 g = globals() 87 exec('z = a', g, m) 88 self.assertEqual(m.results, ('z', 12)) 89 try: 90 exec('z = b', g, m) 91 except NameError: 92 pass 93 else: 94 self.fail('Did not detect a KeyError') 95 exec('z = dir()', g, m) 96 self.assertEqual(m.results, ('z', list('xyz'))) 97 exec('z = globals()', g, m) 98 self.assertEqual(m.results, ('z', g)) 99 exec('z = locals()', g, m) 100 self.assertEqual(m.results, ('z', m)) 101 self.assertRaises(TypeError, exec, 'z = b', m) 102 103 class A: 104 "Non-mapping" 105 pass 106 m = A() 107 self.assertRaises(TypeError, exec, 'z = a', g, m) 108 109 # Verify that dict subclasses work as well 110 class D(dict): 111 def __getitem__(self, key): 112 if key == 'a': 113 return 12 114 return dict.__getitem__(self, key) 115 d = D() 116 exec('z = a', g, d) 117 self.assertEqual(d['z'], 12) 118 119 @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") 120 def test_extended_arg(self): 121 repeat = int(get_c_recursion_limit() * 0.9) 122 longexpr = 'x = x or ' + '-x' * repeat 123 g = {} 124 code = textwrap.dedent(''' 125 def f(x): 126 %s 127 %s 128 %s 129 %s 130 %s 131 %s 132 %s 133 %s 134 %s 135 %s 136 # the expressions above have no effect, x == argument 137 while x: 138 x -= 1 139 # EXTENDED_ARG/JUMP_ABSOLUTE here 140 return x 141 ''' % ((longexpr,)*10)) 142 exec(code, g) 143 self.assertEqual(g['f'](5), 0) 144 145 def test_argument_order(self): 146 self.assertRaises(SyntaxError, exec, 'def f(a=1, b): pass') 147 148 def test_float_literals(self): 149 # testing bad float literals 150 self.assertRaises(SyntaxError, eval, "2e") 151 self.assertRaises(SyntaxError, eval, "2.0e+") 152 self.assertRaises(SyntaxError, eval, "1e-") 153 self.assertRaises(SyntaxError, eval, "3-4e/21") 154 155 def test_indentation(self): 156 # testing compile() of indented block w/o trailing newline" 157 s = textwrap.dedent(""" 158 if 1: 159 if 2: 160 pass 161 """) 162 compile(s, "<string>", "exec") 163 164 # This test is probably specific to CPython and may not generalize 165 # to other implementations. We are trying to ensure that when 166 # the first line of code starts after 256, correct line numbers 167 # in tracebacks are still produced. 168 def test_leading_newlines(self): 169 s256 = "".join(["\n"] * 256 + ["spam"]) 170 co = compile(s256, 'fn', 'exec') 171 self.assertEqual(co.co_firstlineno, 1) 172 lines = [line for _, _, line in co.co_lines()] 173 self.assertEqual(lines, [0, 257]) 174 175 def test_literals_with_leading_zeroes(self): 176 for arg in ["077787", "0xj", "0x.", "0e", "090000000000000", 177 "080000000000000", "000000000000009", "000000000000008", 178 "0b42", "0BADCAFE", "0o123456789", "0b1.1", "0o4.2", 179 "0b101j", "0o153j", "0b100e1", "0o777e1", "0777", 180 "000777", "000000000000007"]: 181 self.assertRaises(SyntaxError, eval, arg) 182 183 self.assertEqual(eval("0xff"), 255) 184 self.assertEqual(eval("0777."), 777) 185 self.assertEqual(eval("0777.0"), 777) 186 self.assertEqual(eval("000000000000000000000000000000000000000000000000000777e0"), 777) 187 self.assertEqual(eval("0777e1"), 7770) 188 self.assertEqual(eval("0e0"), 0) 189 self.assertEqual(eval("0000e-012"), 0) 190 self.assertEqual(eval("09.5"), 9.5) 191 self.assertEqual(eval("0777j"), 777j) 192 self.assertEqual(eval("000"), 0) 193 self.assertEqual(eval("00j"), 0j) 194 self.assertEqual(eval("00.0"), 0) 195 self.assertEqual(eval("0e3"), 0) 196 self.assertEqual(eval("090000000000000."), 90000000000000.) 197 self.assertEqual(eval("090000000000000.0000000000000000000000"), 90000000000000.) 198 self.assertEqual(eval("090000000000000e0"), 90000000000000.) 199 self.assertEqual(eval("090000000000000e-0"), 90000000000000.) 200 self.assertEqual(eval("090000000000000j"), 90000000000000j) 201 self.assertEqual(eval("000000000000008."), 8.) 202 self.assertEqual(eval("000000000000009."), 9.) 203 self.assertEqual(eval("0b101010"), 42) 204 self.assertEqual(eval("-0b000000000010"), -2) 205 self.assertEqual(eval("0o777"), 511) 206 self.assertEqual(eval("-0o0000010"), -8) 207 208 def test_int_literals_too_long(self): 209 n = 3000 210 source = f"a = 1\nb = 2\nc = {'3'*n}\nd = 4" 211 with support.adjust_int_max_str_digits(n): 212 compile(source, "<long_int_pass>", "exec") # no errors. 213 with support.adjust_int_max_str_digits(n-1): 214 with self.assertRaises(SyntaxError) as err_ctx: 215 compile(source, "<long_int_fail>", "exec") 216 exc = err_ctx.exception 217 self.assertEqual(exc.lineno, 3) 218 self.assertIn('Exceeds the limit ', str(exc)) 219 self.assertIn(' Consider hexadecimal ', str(exc)) 220 221 def test_unary_minus(self): 222 # Verify treatment of unary minus on negative numbers SF bug #660455 223 if sys.maxsize == 2147483647: 224 # 32-bit machine 225 all_one_bits = '0xffffffff' 226 self.assertEqual(eval(all_one_bits), 4294967295) 227 self.assertEqual(eval("-" + all_one_bits), -4294967295) 228 elif sys.maxsize == 9223372036854775807: 229 # 64-bit machine 230 all_one_bits = '0xffffffffffffffff' 231 self.assertEqual(eval(all_one_bits), 18446744073709551615) 232 self.assertEqual(eval("-" + all_one_bits), -18446744073709551615) 233 else: 234 self.fail("How many bits *does* this machine have???") 235 # Verify treatment of constant folding on -(sys.maxsize+1) 236 # i.e. -2147483648 on 32 bit platforms. Should return int. 237 self.assertIsInstance(eval("%s" % (-sys.maxsize - 1)), int) 238 self.assertIsInstance(eval("%s" % (-sys.maxsize - 2)), int) 239 240 if sys.maxsize == 9223372036854775807: 241 def test_32_63_bit_values(self): 242 a = +4294967296 # 1 << 32 243 b = -4294967296 # 1 << 32 244 c = +281474976710656 # 1 << 48 245 d = -281474976710656 # 1 << 48 246 e = +4611686018427387904 # 1 << 62 247 f = -4611686018427387904 # 1 << 62 248 g = +9223372036854775807 # 1 << 63 - 1 249 h = -9223372036854775807 # 1 << 63 - 1 250 251 for variable in self.test_32_63_bit_values.__code__.co_consts: 252 if variable is not None: 253 self.assertIsInstance(variable, int) 254 255 def test_sequence_unpacking_error(self): 256 # Verify sequence packing/unpacking with "or". SF bug #757818 257 i,j = (1, -1) or (-1, 1) 258 self.assertEqual(i, 1) 259 self.assertEqual(j, -1) 260 261 def test_none_assignment(self): 262 stmts = [ 263 'None = 0', 264 'None += 0', 265 '__builtins__.None = 0', 266 'def None(): pass', 267 'class None: pass', 268 '(a, None) = 0, 0', 269 'for None in range(10): pass', 270 'def f(None): pass', 271 'import None', 272 'import x as None', 273 'from x import None', 274 'from x import y as None' 275 ] 276 for stmt in stmts: 277 stmt += "\n" 278 self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') 279 self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') 280 281 def test_import(self): 282 succeed = [ 283 'import sys', 284 'import os, sys', 285 'import os as bar', 286 'import os.path as bar', 287 'from __future__ import nested_scopes, generators', 288 'from __future__ import (nested_scopes,\ngenerators)', 289 'from __future__ import (nested_scopes,\ngenerators,)', 290 'from sys import stdin, stderr, stdout', 291 'from sys import (stdin, stderr,\nstdout)', 292 'from sys import (stdin, stderr,\nstdout,)', 293 'from sys import (stdin\n, stderr, stdout)', 294 'from sys import (stdin\n, stderr, stdout,)', 295 'from sys import stdin as si, stdout as so, stderr as se', 296 'from sys import (stdin as si, stdout as so, stderr as se)', 297 'from sys import (stdin as si, stdout as so, stderr as se,)', 298 ] 299 fail = [ 300 'import (os, sys)', 301 'import (os), (sys)', 302 'import ((os), (sys))', 303 'import (sys', 304 'import sys)', 305 'import (os,)', 306 'import os As bar', 307 'import os.path a bar', 308 'from sys import stdin As stdout', 309 'from sys import stdin a stdout', 310 'from (sys) import stdin', 311 'from __future__ import (nested_scopes', 312 'from __future__ import nested_scopes)', 313 'from __future__ import nested_scopes,\ngenerators', 314 'from sys import (stdin', 315 'from sys import stdin)', 316 'from sys import stdin, stdout,\nstderr', 317 'from sys import stdin si', 318 'from sys import stdin,', 319 'from sys import (*)', 320 'from sys import (stdin,, stdout, stderr)', 321 'from sys import (stdin, stdout),', 322 ] 323 for stmt in succeed: 324 compile(stmt, 'tmp', 'exec') 325 for stmt in fail: 326 self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') 327 328 def test_for_distinct_code_objects(self): 329 # SF bug 1048870 330 def f(): 331 f1 = lambda x=1: x 332 f2 = lambda x=2: x 333 return f1, f2 334 f1, f2 = f() 335 self.assertNotEqual(id(f1.__code__), id(f2.__code__)) 336 337 def test_lambda_doc(self): 338 l = lambda: "foo" 339 self.assertIsNone(l.__doc__) 340 341 def test_encoding(self): 342 code = b'# -*- coding: badencoding -*-\npass\n' 343 self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec') 344 code = '# -*- coding: badencoding -*-\n"\xc2\xa4"\n' 345 compile(code, 'tmp', 'exec') 346 self.assertEqual(eval(code), '\xc2\xa4') 347 code = '"\xc2\xa4"\n' 348 self.assertEqual(eval(code), '\xc2\xa4') 349 code = b'"\xc2\xa4"\n' 350 self.assertEqual(eval(code), '\xa4') 351 code = b'# -*- coding: latin1 -*-\n"\xc2\xa4"\n' 352 self.assertEqual(eval(code), '\xc2\xa4') 353 code = b'# -*- coding: utf-8 -*-\n"\xc2\xa4"\n' 354 self.assertEqual(eval(code), '\xa4') 355 code = b'# -*- coding: iso8859-15 -*-\n"\xc2\xa4"\n' 356 self.assertEqual(eval(code), '\xc2\u20ac') 357 code = '"""\\\n# -*- coding: iso8859-15 -*-\n\xc2\xa4"""\n' 358 self.assertEqual(eval(code), '# -*- coding: iso8859-15 -*-\n\xc2\xa4') 359 code = b'"""\\\n# -*- coding: iso8859-15 -*-\n\xc2\xa4"""\n' 360 self.assertEqual(eval(code), '# -*- coding: iso8859-15 -*-\n\xa4') 361 362 def test_subscripts(self): 363 # SF bug 1448804 364 # Class to make testing subscript results easy 365 class str_map(object): 366 def __init__(self): 367 self.data = {} 368 def __getitem__(self, key): 369 return self.data[str(key)] 370 def __setitem__(self, key, value): 371 self.data[str(key)] = value 372 def __delitem__(self, key): 373 del self.data[str(key)] 374 def __contains__(self, key): 375 return str(key) in self.data 376 d = str_map() 377 # Index 378 d[1] = 1 379 self.assertEqual(d[1], 1) 380 d[1] += 1 381 self.assertEqual(d[1], 2) 382 del d[1] 383 self.assertNotIn(1, d) 384 # Tuple of indices 385 d[1, 1] = 1 386 self.assertEqual(d[1, 1], 1) 387 d[1, 1] += 1 388 self.assertEqual(d[1, 1], 2) 389 del d[1, 1] 390 self.assertNotIn((1, 1), d) 391 # Simple slice 392 d[1:2] = 1 393 self.assertEqual(d[1:2], 1) 394 d[1:2] += 1 395 self.assertEqual(d[1:2], 2) 396 del d[1:2] 397 self.assertNotIn(slice(1, 2), d) 398 # Tuple of simple slices 399 d[1:2, 1:2] = 1 400 self.assertEqual(d[1:2, 1:2], 1) 401 d[1:2, 1:2] += 1 402 self.assertEqual(d[1:2, 1:2], 2) 403 del d[1:2, 1:2] 404 self.assertNotIn((slice(1, 2), slice(1, 2)), d) 405 # Extended slice 406 d[1:2:3] = 1 407 self.assertEqual(d[1:2:3], 1) 408 d[1:2:3] += 1 409 self.assertEqual(d[1:2:3], 2) 410 del d[1:2:3] 411 self.assertNotIn(slice(1, 2, 3), d) 412 # Tuple of extended slices 413 d[1:2:3, 1:2:3] = 1 414 self.assertEqual(d[1:2:3, 1:2:3], 1) 415 d[1:2:3, 1:2:3] += 1 416 self.assertEqual(d[1:2:3, 1:2:3], 2) 417 del d[1:2:3, 1:2:3] 418 self.assertNotIn((slice(1, 2, 3), slice(1, 2, 3)), d) 419 # Ellipsis 420 d[...] = 1 421 self.assertEqual(d[...], 1) 422 d[...] += 1 423 self.assertEqual(d[...], 2) 424 del d[...] 425 self.assertNotIn(Ellipsis, d) 426 # Tuple of Ellipses 427 d[..., ...] = 1 428 self.assertEqual(d[..., ...], 1) 429 d[..., ...] += 1 430 self.assertEqual(d[..., ...], 2) 431 del d[..., ...] 432 self.assertNotIn((Ellipsis, Ellipsis), d) 433 434 def test_annotation_limit(self): 435 # more than 255 annotations, should compile ok 436 s = "def f(%s): pass" 437 s %= ', '.join('a%d:%d' % (i,i) for i in range(300)) 438 compile(s, '?', 'exec') 439 440 def test_mangling(self): 441 class A: 442 def f(): 443 __mangled = 1 444 __not_mangled__ = 2 445 import __mangled_mod 446 import __package__.module 447 448 self.assertIn("_A__mangled", A.f.__code__.co_varnames) 449 self.assertIn("__not_mangled__", A.f.__code__.co_varnames) 450 self.assertIn("_A__mangled_mod", A.f.__code__.co_varnames) 451 self.assertIn("__package__", A.f.__code__.co_varnames) 452 453 def test_condition_expression_with_dead_blocks_compiles(self): 454 # See gh-113054 455 compile('if (5 if 5 else T): 0', '<eval>', 'exec') 456 457 def test_condition_expression_with_redundant_comparisons_compiles(self): 458 # See gh-113054, gh-114083 459 exprs = [ 460 'if 9<9<9and 9or 9:9', 461 'if 9<9<9and 9or 9or 9:9', 462 'if 9<9<9and 9or 9or 9or 9:9', 463 'if 9<9<9and 9or 9or 9or 9or 9:9', 464 ] 465 for expr in exprs: 466 with self.subTest(expr=expr): 467 with self.assertWarns(SyntaxWarning): 468 compile(expr, '<eval>', 'exec') 469 470 def test_dead_code_with_except_handler_compiles(self): 471 compile(textwrap.dedent(""" 472 if None: 473 with CM: 474 x = 1 475 else: 476 x = 2 477 """), '<eval>', 'exec') 478 479 def test_try_except_in_while_with_chained_condition_compiles(self): 480 # see gh-124871 481 compile(textwrap.dedent(""" 482 name_1, name_2, name_3 = 1, 2, 3 483 while name_3 <= name_2 > name_1: 484 try: 485 raise 486 except: 487 pass 488 finally: 489 pass 490 """), '<eval>', 'exec') 491 492 def test_compile_invalid_namedexpr(self): 493 # gh-109351 494 m = ast.Module( 495 body=[ 496 ast.Expr( 497 value=ast.ListComp( 498 elt=ast.NamedExpr( 499 target=ast.Constant(value=1), 500 value=ast.Constant(value=3), 501 ), 502 generators=[ 503 ast.comprehension( 504 target=ast.Name(id="x", ctx=ast.Store()), 505 iter=ast.Name(id="y", ctx=ast.Load()), 506 ifs=[], 507 is_async=0, 508 ) 509 ], 510 ) 511 ) 512 ], 513 type_ignores=[], 514 ) 515 516 with self.assertRaisesRegex(TypeError, "NamedExpr target must be a Name"): 517 compile(ast.fix_missing_locations(m), "<file>", "exec") 518 519 def test_compile_redundant_jumps_and_nops_after_moving_cold_blocks(self): 520 # See gh-120367 521 code=textwrap.dedent(""" 522 try: 523 pass 524 except: 525 pass 526 else: 527 match name_2: 528 case b'': 529 pass 530 finally: 531 something 532 """) 533 534 tree = ast.parse(code) 535 536 # make all instruction locations the same to create redundancies 537 for node in ast.walk(tree): 538 if hasattr(node,"lineno"): 539 del node.lineno 540 del node.end_lineno 541 del node.col_offset 542 del node.end_col_offset 543 544 compile(ast.fix_missing_locations(tree), "<file>", "exec") 545 546 def test_compile_redundant_jump_after_convert_pseudo_ops(self): 547 # See gh-120367 548 code=textwrap.dedent(""" 549 if name_2: 550 pass 551 else: 552 try: 553 pass 554 except: 555 pass 556 ~name_5 557 """) 558 559 tree = ast.parse(code) 560 561 # make all instruction locations the same to create redundancies 562 for node in ast.walk(tree): 563 if hasattr(node,"lineno"): 564 del node.lineno 565 del node.end_lineno 566 del node.col_offset 567 del node.end_col_offset 568 569 compile(ast.fix_missing_locations(tree), "<file>", "exec") 570 571 def test_compile_ast(self): 572 fname = __file__ 573 if fname.lower().endswith('pyc'): 574 fname = fname[:-1] 575 with open(fname, encoding='utf-8') as f: 576 fcontents = f.read() 577 sample_code = [ 578 ['<assign>', 'x = 5'], 579 ['<ifblock>', """if True:\n pass\n"""], 580 ['<forblock>', """for n in [1, 2, 3]:\n print(n)\n"""], 581 ['<deffunc>', """def foo():\n pass\nfoo()\n"""], 582 [fname, fcontents], 583 ] 584 585 for fname, code in sample_code: 586 co1 = compile(code, '%s1' % fname, 'exec') 587 ast = compile(code, '%s2' % fname, 'exec', _ast.PyCF_ONLY_AST) 588 self.assertTrue(type(ast) == _ast.Module) 589 co2 = compile(ast, '%s3' % fname, 'exec') 590 self.assertEqual(co1, co2) 591 # the code object's filename comes from the second compilation step 592 self.assertEqual(co2.co_filename, '%s3' % fname) 593 594 # raise exception when node type doesn't match with compile mode 595 co1 = compile('print(1)', '<string>', 'exec', _ast.PyCF_ONLY_AST) 596 self.assertRaises(TypeError, compile, co1, '<ast>', 'eval') 597 598 # raise exception when node type is no start node 599 self.assertRaises(TypeError, compile, _ast.If(test=_ast.Name(id='x', ctx=_ast.Load())), '<ast>', 'exec') 600 601 # raise exception when node has invalid children 602 ast = _ast.Module() 603 ast.body = [_ast.BoolOp(op=_ast.Or())] 604 self.assertRaises(TypeError, compile, ast, '<ast>', 'exec') 605 606 def test_compile_invalid_typealias(self): 607 # gh-109341 608 m = ast.Module( 609 body=[ 610 ast.TypeAlias( 611 name=ast.Subscript( 612 value=ast.Name(id="foo", ctx=ast.Load()), 613 slice=ast.Constant(value="x"), 614 ctx=ast.Store(), 615 ), 616 type_params=[], 617 value=ast.Name(id="Callable", ctx=ast.Load()), 618 ) 619 ], 620 type_ignores=[], 621 ) 622 623 with self.assertRaisesRegex(TypeError, "TypeAlias with non-Name name"): 624 compile(ast.fix_missing_locations(m), "<file>", "exec") 625 626 def test_dict_evaluation_order(self): 627 i = 0 628 629 def f(): 630 nonlocal i 631 i += 1 632 return i 633 634 d = {f(): f(), f(): f()} 635 self.assertEqual(d, {1: 2, 3: 4}) 636 637 def test_compile_filename(self): 638 for filename in 'file.py', b'file.py': 639 code = compile('pass', filename, 'exec') 640 self.assertEqual(code.co_filename, 'file.py') 641 for filename in bytearray(b'file.py'), memoryview(b'file.py'): 642 with self.assertRaises(TypeError): 643 compile('pass', filename, 'exec') 644 self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec') 645 646 @support.cpython_only 647 def test_same_filename_used(self): 648 s = """def f(): pass\ndef g(): pass""" 649 c = compile(s, "myfile", "exec") 650 for obj in c.co_consts: 651 if isinstance(obj, types.CodeType): 652 self.assertIs(obj.co_filename, c.co_filename) 653 654 def test_single_statement(self): 655 self.compile_single("1 + 2") 656 self.compile_single("\n1 + 2") 657 self.compile_single("1 + 2\n") 658 self.compile_single("1 + 2\n\n") 659 self.compile_single("1 + 2\t\t\n") 660 self.compile_single("1 + 2\t\t\n ") 661 self.compile_single("1 + 2 # one plus two") 662 self.compile_single("1; 2") 663 self.compile_single("import sys; sys") 664 self.compile_single("def f():\n pass") 665 self.compile_single("while False:\n pass") 666 self.compile_single("if x:\n f(x)") 667 self.compile_single("if x:\n f(x)\nelse:\n g(x)") 668 self.compile_single("class T:\n pass") 669 self.compile_single("c = '''\na=1\nb=2\nc=3\n'''") 670 671 def test_bad_single_statement(self): 672 self.assertInvalidSingle('1\n2') 673 self.assertInvalidSingle('def f(): pass') 674 self.assertInvalidSingle('a = 13\nb = 187') 675 self.assertInvalidSingle('del x\ndel y') 676 self.assertInvalidSingle('f()\ng()') 677 self.assertInvalidSingle('f()\n# blah\nblah()') 678 self.assertInvalidSingle('f()\nxy # blah\nblah()') 679 self.assertInvalidSingle('x = 5 # comment\nx = 6\n') 680 self.assertInvalidSingle("c = '''\nd=1\n'''\na = 1\n\nb = 2\n") 681 682 def test_particularly_evil_undecodable(self): 683 # Issue 24022 684 src = b'0000\x00\n00000000000\n\x00\n\x9e\n' 685 with tempfile.TemporaryDirectory() as tmpd: 686 fn = os.path.join(tmpd, "bad.py") 687 with open(fn, "wb") as fp: 688 fp.write(src) 689 res = script_helper.run_python_until_end(fn)[0] 690 self.assertIn(b"source code cannot contain null bytes", res.err) 691 692 def test_yet_more_evil_still_undecodable(self): 693 # Issue #25388 694 src = b"#\x00\n#\xfd\n" 695 with tempfile.TemporaryDirectory() as tmpd: 696 fn = os.path.join(tmpd, "bad.py") 697 with open(fn, "wb") as fp: 698 fp.write(src) 699 res = script_helper.run_python_until_end(fn)[0] 700 self.assertIn(b"source code cannot contain null bytes", res.err) 701 702 @support.cpython_only 703 @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") 704 def test_compiler_recursion_limit(self): 705 # Expected limit is Py_C_RECURSION_LIMIT 706 limit = get_c_recursion_limit() 707 fail_depth = limit + 1 708 crash_depth = limit * 100 709 success_depth = int(limit * 0.8) 710 711 def check_limit(prefix, repeated, mode="single"): 712 expect_ok = prefix + repeated * success_depth 713 compile(expect_ok, '<test>', mode) 714 for depth in (fail_depth, crash_depth): 715 broken = prefix + repeated * depth 716 details = "Compiling ({!r} + {!r} * {})".format( 717 prefix, repeated, depth) 718 with self.assertRaises(RecursionError, msg=details): 719 compile(broken, '<test>', mode) 720 721 check_limit("a", "()") 722 check_limit("a", ".b") 723 check_limit("a", "[0]") 724 check_limit("a", "*a") 725 # XXX Crashes in the parser. 726 # check_limit("a", " if a else a") 727 # check_limit("if a: pass", "\nelif a: pass", mode="exec") 728 729 def test_null_terminated(self): 730 # The source code is null-terminated internally, but bytes-like 731 # objects are accepted, which could be not terminated. 732 with self.assertRaisesRegex(SyntaxError, "cannot contain null"): 733 compile("123\x00", "<dummy>", "eval") 734 with self.assertRaisesRegex(SyntaxError, "cannot contain null"): 735 compile(memoryview(b"123\x00"), "<dummy>", "eval") 736 code = compile(memoryview(b"123\x00")[1:-1], "<dummy>", "eval") 737 self.assertEqual(eval(code), 23) 738 code = compile(memoryview(b"1234")[1:-1], "<dummy>", "eval") 739 self.assertEqual(eval(code), 23) 740 code = compile(memoryview(b"$23$")[1:-1], "<dummy>", "eval") 741 self.assertEqual(eval(code), 23) 742 743 # Also test when eval() and exec() do the compilation step 744 self.assertEqual(eval(memoryview(b"1234")[1:-1]), 23) 745 namespace = dict() 746 exec(memoryview(b"ax = 123")[1:-1], namespace) 747 self.assertEqual(namespace['x'], 12) 748 749 def check_constant(self, func, expected): 750 for const in func.__code__.co_consts: 751 if repr(const) == repr(expected): 752 break 753 else: 754 self.fail("unable to find constant %r in %r" 755 % (expected, func.__code__.co_consts)) 756 757 # Merging equal constants is not a strict requirement for the Python 758 # semantics, it's a more an implementation detail. 759 @support.cpython_only 760 def test_merge_constants(self): 761 # Issue #25843: compile() must merge constants which are equal 762 # and have the same type. 763 764 def check_same_constant(const): 765 ns = {} 766 code = "f1, f2 = lambda: %r, lambda: %r" % (const, const) 767 exec(code, ns) 768 f1 = ns['f1'] 769 f2 = ns['f2'] 770 self.assertIs(f1.__code__.co_consts, f2.__code__.co_consts) 771 self.check_constant(f1, const) 772 self.assertEqual(repr(f1()), repr(const)) 773 774 check_same_constant(None) 775 check_same_constant(0) 776 check_same_constant(0.0) 777 check_same_constant(b'abc') 778 check_same_constant('abc') 779 780 # Note: "lambda: ..." emits "LOAD_CONST Ellipsis", 781 # whereas "lambda: Ellipsis" emits "LOAD_GLOBAL Ellipsis" 782 f1, f2 = lambda: ..., lambda: ... 783 self.assertIs(f1.__code__.co_consts, f2.__code__.co_consts) 784 self.check_constant(f1, Ellipsis) 785 self.assertEqual(repr(f1()), repr(Ellipsis)) 786 787 # Merge constants in tuple or frozenset 788 f1, f2 = lambda: "not a name", lambda: ("not a name",) 789 f3 = lambda x: x in {("not a name",)} 790 self.assertIs(f1.__code__.co_consts[1], 791 f2.__code__.co_consts[1][0]) 792 self.assertIs(next(iter(f3.__code__.co_consts[1])), 793 f2.__code__.co_consts[1]) 794 795 # {0} is converted to a constant frozenset({0}) by the peephole 796 # optimizer 797 f1, f2 = lambda x: x in {0}, lambda x: x in {0} 798 self.assertIs(f1.__code__.co_consts, f2.__code__.co_consts) 799 self.check_constant(f1, frozenset({0})) 800 self.assertTrue(f1(0)) 801 802 # Merging equal co_linetable is not a strict requirement 803 # for the Python semantics, it's a more an implementation detail. 804 @support.cpython_only 805 def test_merge_code_attrs(self): 806 # See https://bugs.python.org/issue42217 807 f1 = lambda x: x.y.z 808 f2 = lambda a: a.b.c 809 810 self.assertIs(f1.__code__.co_linetable, f2.__code__.co_linetable) 811 812 @support.cpython_only 813 def test_remove_unused_consts(self): 814 def f(): 815 "docstring" 816 if True: 817 return "used" 818 else: 819 return "unused" 820 821 self.assertEqual(f.__code__.co_consts, 822 (f.__doc__, "used")) 823 824 @support.cpython_only 825 def test_remove_unused_consts_no_docstring(self): 826 # the first item (None for no docstring in this case) is 827 # always retained. 828 def f(): 829 if True: 830 return "used" 831 else: 832 return "unused" 833 834 self.assertEqual(f.__code__.co_consts, 835 (None, "used")) 836 837 @support.cpython_only 838 def test_remove_unused_consts_extended_args(self): 839 N = 1000 840 code = ["def f():\n"] 841 code.append("\ts = ''\n") 842 code.append("\tfor i in range(1):\n") 843 for i in range(N): 844 code.append(f"\t\tif True: s += 't{i}'\n") 845 code.append(f"\t\tif False: s += 'f{i}'\n") 846 code.append("\treturn s\n") 847 848 code = "".join(code) 849 g = {} 850 eval(compile(code, "file.py", "exec"), g) 851 exec(code, g) 852 f = g['f'] 853 expected = tuple([None, '', 1] + [f't{i}' for i in range(N)]) 854 self.assertEqual(f.__code__.co_consts, expected) 855 expected = "".join(expected[3:]) 856 self.assertEqual(expected, f()) 857 858 # Stripping unused constants is not a strict requirement for the 859 # Python semantics, it's a more an implementation detail. 860 @support.cpython_only 861 def test_strip_unused_None(self): 862 # Python 3.10rc1 appended None to co_consts when None is not used 863 # at all. See bpo-45056. 864 def f1(): 865 "docstring" 866 return 42 867 self.assertEqual(f1.__code__.co_consts, (f1.__doc__, 42)) 868 869 # This is a regression test for a CPython specific peephole optimizer 870 # implementation bug present in a few releases. It's assertion verifies 871 # that peephole optimization was actually done though that isn't an 872 # indication of the bugs presence or not (crashing is). 873 @support.cpython_only 874 def test_peephole_opt_unreachable_code_array_access_in_bounds(self): 875 """Regression test for issue35193 when run under clang msan.""" 876 def unused_code_at_end(): 877 return 3 878 raise RuntimeError("unreachable") 879 # The above function definition will trigger the out of bounds 880 # bug in the peephole optimizer as it scans opcodes past the 881 # RETURN_VALUE opcode. This does not always crash an interpreter. 882 # When you build with the clang memory sanitizer it reliably aborts. 883 self.assertEqual( 884 'RETURN_CONST', 885 list(dis.get_instructions(unused_code_at_end))[-1].opname) 886 887 @support.cpython_only 888 def test_docstring(self): 889 src = textwrap.dedent(""" 890 def with_docstring(): 891 "docstring" 892 893 def with_fstring(): 894 f"not docstring" 895 896 def with_const_expression(): 897 "also" + " not docstring" 898 """) 899 900 for opt in [0, 1, 2]: 901 with self.subTest(opt=opt): 902 code = compile(src, "<test>", "exec", optimize=opt) 903 ns = {} 904 exec(code, ns) 905 906 if opt < 2: 907 self.assertEqual(ns['with_docstring'].__doc__, "docstring") 908 else: 909 self.assertIsNone(ns['with_docstring'].__doc__) 910 self.assertIsNone(ns['with_fstring'].__doc__) 911 self.assertIsNone(ns['with_const_expression'].__doc__) 912 913 @support.cpython_only 914 def test_docstring_omitted(self): 915 # See gh-115347 916 src = textwrap.dedent(""" 917 def f(): 918 "docstring1" 919 def h(): 920 "docstring2" 921 return 42 922 923 class C: 924 "docstring3" 925 pass 926 927 return h 928 """) 929 for opt in [-1, 0, 1, 2]: 930 with self.subTest(opt=opt): 931 code = compile(src, "<test>", "exec", optimize=opt) 932 output = io.StringIO() 933 with contextlib.redirect_stdout(output): 934 dis.dis(code) 935 self.assertNotIn('NOP' , output.getvalue()) 936 937 def test_dont_merge_constants(self): 938 # Issue #25843: compile() must not merge constants which are equal 939 # but have a different type. 940 941 def check_different_constants(const1, const2): 942 ns = {} 943 exec("f1, f2 = lambda: %r, lambda: %r" % (const1, const2), ns) 944 f1 = ns['f1'] 945 f2 = ns['f2'] 946 self.assertIsNot(f1.__code__, f2.__code__) 947 self.assertNotEqual(f1.__code__, f2.__code__) 948 self.check_constant(f1, const1) 949 self.check_constant(f2, const2) 950 self.assertEqual(repr(f1()), repr(const1)) 951 self.assertEqual(repr(f2()), repr(const2)) 952 953 check_different_constants(0, 0.0) 954 check_different_constants(+0.0, -0.0) 955 check_different_constants((0,), (0.0,)) 956 check_different_constants('a', b'a') 957 check_different_constants(('a',), (b'a',)) 958 959 # check_different_constants() cannot be used because repr(-0j) is 960 # '(-0-0j)', but when '(-0-0j)' is evaluated to 0j: we loose the sign. 961 f1, f2 = lambda: +0.0j, lambda: -0.0j 962 self.assertIsNot(f1.__code__, f2.__code__) 963 self.check_constant(f1, +0.0j) 964 self.check_constant(f2, -0.0j) 965 self.assertEqual(repr(f1()), repr(+0.0j)) 966 self.assertEqual(repr(f2()), repr(-0.0j)) 967 968 # {0} is converted to a constant frozenset({0}) by the peephole 969 # optimizer 970 f1, f2 = lambda x: x in {0}, lambda x: x in {0.0} 971 self.assertIsNot(f1.__code__, f2.__code__) 972 self.check_constant(f1, frozenset({0})) 973 self.check_constant(f2, frozenset({0.0})) 974 self.assertTrue(f1(0)) 975 self.assertTrue(f2(0.0)) 976 977 def test_path_like_objects(self): 978 # An implicit test for PyUnicode_FSDecoder(). 979 compile("42", FakePath("test_compile_pathlike"), "single") 980 981 @support.requires_resource('cpu') 982 def test_stack_overflow(self): 983 # bpo-31113: Stack overflow when compile a long sequence of 984 # complex statements. 985 compile("if a: b\n" * 200000, "<dummy>", "exec") 986 987 # Multiple users rely on the fact that CPython does not generate 988 # bytecode for dead code blocks. See bpo-37500 for more context. 989 @support.cpython_only 990 def test_dead_blocks_do_not_generate_bytecode(self): 991 def unused_block_if(): 992 if 0: 993 return 42 994 995 def unused_block_while(): 996 while 0: 997 return 42 998 999 def unused_block_if_else(): 1000 if 1: 1001 return None 1002 else: 1003 return 42 1004 1005 def unused_block_while_else(): 1006 while 1: 1007 return None 1008 else: 1009 return 42 1010 1011 funcs = [unused_block_if, unused_block_while, 1012 unused_block_if_else, unused_block_while_else] 1013 1014 for func in funcs: 1015 opcodes = list(dis.get_instructions(func)) 1016 self.assertLessEqual(len(opcodes), 3) 1017 self.assertEqual('RETURN_CONST', opcodes[-1].opname) 1018 self.assertEqual(None, opcodes[-1].argval) 1019 1020 def test_false_while_loop(self): 1021 def break_in_while(): 1022 while False: 1023 break 1024 1025 def continue_in_while(): 1026 while False: 1027 continue 1028 1029 funcs = [break_in_while, continue_in_while] 1030 1031 # Check that we did not raise but we also don't generate bytecode 1032 for func in funcs: 1033 opcodes = list(dis.get_instructions(func)) 1034 self.assertEqual(2, len(opcodes)) 1035 self.assertEqual('RETURN_CONST', opcodes[1].opname) 1036 self.assertEqual(None, opcodes[1].argval) 1037 1038 def test_consts_in_conditionals(self): 1039 def and_true(x): 1040 return True and x 1041 1042 def and_false(x): 1043 return False and x 1044 1045 def or_true(x): 1046 return True or x 1047 1048 def or_false(x): 1049 return False or x 1050 1051 funcs = [and_true, and_false, or_true, or_false] 1052 1053 # Check that condition is removed. 1054 for func in funcs: 1055 with self.subTest(func=func): 1056 opcodes = list(dis.get_instructions(func)) 1057 self.assertLessEqual(len(opcodes), 3) 1058 self.assertIn('LOAD_', opcodes[-2].opname) 1059 self.assertEqual('RETURN_VALUE', opcodes[-1].opname) 1060 1061 def test_imported_load_method(self): 1062 sources = [ 1063 """\ 1064 import os 1065 def foo(): 1066 return os.uname() 1067 """, 1068 """\ 1069 import os as operating_system 1070 def foo(): 1071 return operating_system.uname() 1072 """, 1073 """\ 1074 from os import path 1075 def foo(x): 1076 return path.join(x) 1077 """, 1078 """\ 1079 from os import path as os_path 1080 def foo(x): 1081 return os_path.join(x) 1082 """ 1083 ] 1084 for source in sources: 1085 namespace = {} 1086 exec(textwrap.dedent(source), namespace) 1087 func = namespace['foo'] 1088 with self.subTest(func=func.__name__): 1089 opcodes = list(dis.get_instructions(func)) 1090 instructions = [opcode.opname for opcode in opcodes] 1091 self.assertNotIn('LOAD_METHOD', instructions) 1092 self.assertIn('LOAD_ATTR', instructions) 1093 self.assertIn('CALL', instructions) 1094 1095 def test_lineno_procedure_call(self): 1096 def call(): 1097 ( 1098 print() 1099 ) 1100 line1 = call.__code__.co_firstlineno + 1 1101 assert line1 not in [line for (_, _, line) in call.__code__.co_lines()] 1102 1103 def test_lineno_after_implicit_return(self): 1104 TRUE = True 1105 # Don't use constant True or False, as compiler will remove test 1106 def if1(x): 1107 x() 1108 if TRUE: 1109 pass 1110 def if2(x): 1111 x() 1112 if TRUE: 1113 pass 1114 else: 1115 pass 1116 def if3(x): 1117 x() 1118 if TRUE: 1119 pass 1120 else: 1121 return None 1122 def if4(x): 1123 x() 1124 if not TRUE: 1125 pass 1126 funcs = [ if1, if2, if3, if4] 1127 lastlines = [ 3, 3, 3, 2] 1128 frame = None 1129 def save_caller_frame(): 1130 nonlocal frame 1131 frame = sys._getframe(1) 1132 for func, lastline in zip(funcs, lastlines, strict=True): 1133 with self.subTest(func=func): 1134 func(save_caller_frame) 1135 self.assertEqual(frame.f_lineno-frame.f_code.co_firstlineno, lastline) 1136 1137 def test_lineno_after_no_code(self): 1138 def no_code1(): 1139 "doc string" 1140 1141 def no_code2(): 1142 a: int 1143 1144 for func in (no_code1, no_code2): 1145 with self.subTest(func=func): 1146 if func is no_code1 and no_code1.__doc__ is None: 1147 continue 1148 code = func.__code__ 1149 [(start, end, line)] = code.co_lines() 1150 self.assertEqual(start, 0) 1151 self.assertEqual(end, len(code.co_code)) 1152 self.assertEqual(line, code.co_firstlineno) 1153 1154 def get_code_lines(self, code): 1155 last_line = -2 1156 res = [] 1157 for _, _, line in code.co_lines(): 1158 if line is not None and line != last_line: 1159 res.append(line - code.co_firstlineno) 1160 last_line = line 1161 return res 1162 1163 def test_lineno_attribute(self): 1164 def load_attr(): 1165 return ( 1166 o. 1167 a 1168 ) 1169 load_attr_lines = [ 0, 2, 3, 1 ] 1170 1171 def load_method(): 1172 return ( 1173 o. 1174 m( 1175 0 1176 ) 1177 ) 1178 load_method_lines = [ 0, 2, 3, 4, 3, 1 ] 1179 1180 def store_attr(): 1181 ( 1182 o. 1183 a 1184 ) = ( 1185 v 1186 ) 1187 store_attr_lines = [ 0, 5, 2, 3 ] 1188 1189 def aug_store_attr(): 1190 ( 1191 o. 1192 a 1193 ) += ( 1194 v 1195 ) 1196 aug_store_attr_lines = [ 0, 2, 3, 5, 1, 3 ] 1197 1198 funcs = [ load_attr, load_method, store_attr, aug_store_attr] 1199 func_lines = [ load_attr_lines, load_method_lines, 1200 store_attr_lines, aug_store_attr_lines] 1201 1202 for func, lines in zip(funcs, func_lines, strict=True): 1203 with self.subTest(func=func): 1204 code_lines = self.get_code_lines(func.__code__) 1205 self.assertEqual(lines, code_lines) 1206 1207 def test_line_number_genexp(self): 1208 1209 def return_genexp(): 1210 return (1 1211 for 1212 x 1213 in 1214 y) 1215 genexp_lines = [0, 4, 2, 0, 4] 1216 1217 genexp_code = return_genexp.__code__.co_consts[1] 1218 code_lines = self.get_code_lines(genexp_code) 1219 self.assertEqual(genexp_lines, code_lines) 1220 1221 def test_line_number_implicit_return_after_async_for(self): 1222 1223 async def test(aseq): 1224 async for i in aseq: 1225 body 1226 1227 expected_lines = [0, 1, 2, 1] 1228 code_lines = self.get_code_lines(test.__code__) 1229 self.assertEqual(expected_lines, code_lines) 1230 1231 def check_line_numbers(self, code, opnames=None): 1232 # Check that all instructions whose op matches opnames 1233 # have a line number. opnames can be a single name, or 1234 # a sequence of names. If it is None, match all ops. 1235 1236 if isinstance(opnames, str): 1237 opnames = (opnames, ) 1238 for inst in dis.Bytecode(code): 1239 if opnames and inst.opname in opnames: 1240 self.assertIsNotNone(inst.positions.lineno) 1241 1242 def test_line_number_synthetic_jump_multiple_predecessors(self): 1243 def f(): 1244 for x in it: 1245 try: 1246 if C1: 1247 yield 2 1248 except OSError: 1249 pass 1250 1251 self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') 1252 1253 def test_line_number_synthetic_jump_multiple_predecessors_nested(self): 1254 def f(): 1255 for x in it: 1256 try: 1257 X = 3 1258 except OSError: 1259 try: 1260 if C3: 1261 X = 4 1262 except OSError: 1263 pass 1264 return 42 1265 1266 self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') 1267 1268 def test_line_number_synthetic_jump_multiple_predecessors_more_nested(self): 1269 def f(): 1270 for x in it: 1271 try: 1272 X = 3 1273 except OSError: 1274 try: 1275 if C3: 1276 if C4: 1277 X = 4 1278 except OSError: 1279 try: 1280 if C3: 1281 if C4: 1282 X = 5 1283 except OSError: 1284 pass 1285 return 42 1286 1287 self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') 1288 1289 def test_lineno_of_backward_jump_conditional_in_loop(self): 1290 # Issue gh-107901 1291 def f(): 1292 for i in x: 1293 if y: 1294 pass 1295 1296 self.check_line_numbers(f.__code__, 'JUMP_BACKWARD') 1297 1298 def test_big_dict_literal(self): 1299 # The compiler has a flushing point in "compiler_dict" that calls compiles 1300 # a portion of the dictionary literal when the loop that iterates over the items 1301 # reaches 0xFFFF elements but the code was not including the boundary element, 1302 # dropping the key at position 0xFFFF. See bpo-41531 for more information 1303 1304 dict_size = 0xFFFF + 1 1305 the_dict = "{" + ",".join(f"{x}:{x}" for x in range(dict_size)) + "}" 1306 self.assertEqual(len(eval(the_dict)), dict_size) 1307 1308 def test_redundant_jump_in_if_else_break(self): 1309 # Check if bytecode containing jumps that simply point to the next line 1310 # is generated around if-else-break style structures. See bpo-42615. 1311 1312 def if_else_break(): 1313 val = 1 1314 while True: 1315 if val > 0: 1316 val -= 1 1317 else: 1318 break 1319 val = -1 1320 1321 INSTR_SIZE = 2 1322 HANDLED_JUMPS = ( 1323 'POP_JUMP_IF_FALSE', 1324 'POP_JUMP_IF_TRUE', 1325 'JUMP_ABSOLUTE', 1326 'JUMP_FORWARD', 1327 ) 1328 1329 for line, instr in enumerate( 1330 dis.Bytecode(if_else_break, show_caches=True) 1331 ): 1332 if instr.opname == 'JUMP_FORWARD': 1333 self.assertNotEqual(instr.arg, 0) 1334 elif instr.opname in HANDLED_JUMPS: 1335 self.assertNotEqual(instr.arg, (line + 1)*INSTR_SIZE) 1336 1337 def test_no_wraparound_jump(self): 1338 # See https://bugs.python.org/issue46724 1339 1340 def while_not_chained(a, b, c): 1341 while not (a < b < c): 1342 pass 1343 1344 for instr in dis.Bytecode(while_not_chained): 1345 self.assertNotEqual(instr.opname, "EXTENDED_ARG") 1346 1347 @support.cpython_only 1348 def test_uses_slice_instructions(self): 1349 1350 def check_op_count(func, op, expected): 1351 actual = 0 1352 for instr in dis.Bytecode(func): 1353 if instr.opname == op: 1354 actual += 1 1355 self.assertEqual(actual, expected) 1356 1357 def load(): 1358 return x[a:b] + x [a:] + x[:b] + x[:] 1359 1360 def store(): 1361 x[a:b] = y 1362 x [a:] = y 1363 x[:b] = y 1364 x[:] = y 1365 1366 def long_slice(): 1367 return x[a:b:c] 1368 1369 def aug(): 1370 x[a:b] += y 1371 1372 check_op_count(load, "BINARY_SLICE", 4) 1373 check_op_count(load, "BUILD_SLICE", 0) 1374 check_op_count(store, "STORE_SLICE", 4) 1375 check_op_count(store, "BUILD_SLICE", 0) 1376 check_op_count(long_slice, "BUILD_SLICE", 1) 1377 check_op_count(long_slice, "BINARY_SLICE", 0) 1378 check_op_count(aug, "BINARY_SLICE", 1) 1379 check_op_count(aug, "STORE_SLICE", 1) 1380 check_op_count(aug, "BUILD_SLICE", 0) 1381 1382 def test_compare_positions(self): 1383 for opname_prefix, op in [ 1384 ("COMPARE_", "<"), 1385 ("COMPARE_", "<="), 1386 ("COMPARE_", ">"), 1387 ("COMPARE_", ">="), 1388 ("CONTAINS_OP", "in"), 1389 ("CONTAINS_OP", "not in"), 1390 ("IS_OP", "is"), 1391 ("IS_OP", "is not"), 1392 ]: 1393 expr = f'a {op} b {op} c' 1394 expected_positions = 2 * [(2, 2, 0, len(expr))] 1395 for source in [ 1396 f"\\\n{expr}", f'if \\\n{expr}: x', f"x if \\\n{expr} else y" 1397 ]: 1398 code = compile(source, "<test>", "exec") 1399 actual_positions = [ 1400 instruction.positions 1401 for instruction in dis.get_instructions(code) 1402 if instruction.opname.startswith(opname_prefix) 1403 ] 1404 with self.subTest(source): 1405 self.assertEqual(actual_positions, expected_positions) 1406 1407 def test_if_expression_expression_empty_block(self): 1408 # See regression in gh-99708 1409 exprs = [ 1410 "assert (False if 1 else True)", 1411 "def f():\n\tif not (False if 1 else True): raise AssertionError", 1412 "def f():\n\tif not (False if 1 else True): return 12", 1413 ] 1414 for expr in exprs: 1415 with self.subTest(expr=expr): 1416 compile(expr, "<single>", "exec") 1417 1418 def test_multi_line_lambda_as_argument(self): 1419 # See gh-101928 1420 code = textwrap.dedent(""" 1421 def foo(param, lambda_exp): 1422 pass 1423 1424 foo(param=0, 1425 lambda_exp=lambda: 1426 1) 1427 """) 1428 compile(code, "<test>", "exec") 1429 1430 def test_apply_static_swaps(self): 1431 def f(x, y): 1432 a, a = x, y 1433 return a 1434 self.assertEqual(f("x", "y"), "y") 1435 1436 def test_apply_static_swaps_2(self): 1437 def f(x, y, z): 1438 a, b, a = x, y, z 1439 return a 1440 self.assertEqual(f("x", "y", "z"), "z") 1441 1442 def test_apply_static_swaps_3(self): 1443 def f(x, y, z): 1444 a, a, b = x, y, z 1445 return a 1446 self.assertEqual(f("x", "y", "z"), "y") 1447 1448 def test_variable_dependent(self): 1449 # gh-104635: Since the value of b is dependent on the value of a 1450 # the first STORE_FAST for a should not be skipped. (e.g POP_TOP). 1451 # This test case is added to prevent potential regression from aggressive optimization. 1452 def f(): 1453 a = 42; b = a + 54; a = 54 1454 return a, b 1455 self.assertEqual(f(), (54, 96)) 1456 1457 def test_duplicated_small_exit_block(self): 1458 # See gh-109627 1459 def f(): 1460 while element and something: 1461 try: 1462 return something 1463 except: 1464 pass 1465 1466 def test_cold_block_moved_to_end(self): 1467 # See gh-109719 1468 def f(): 1469 while name: 1470 try: 1471 break 1472 except: 1473 pass 1474 else: 1475 1 if 1 else 1 1476 1477 def test_remove_empty_basic_block_with_jump_target_label(self): 1478 # See gh-109823 1479 def f(x): 1480 while x: 1481 0 if 1 else 0 1482 1483 def test_remove_redundant_nop_edge_case(self): 1484 # See gh-109889 1485 def f(): 1486 a if (1 if b else c) else d 1487 1488 def test_global_declaration_in_except_used_in_else(self): 1489 # See gh-111123 1490 code = textwrap.dedent("""\ 1491 def f(): 1492 try: 1493 pass 1494 %s Exception: 1495 global a 1496 else: 1497 print(a) 1498 """) 1499 1500 g, l = {'a': 5}, {} 1501 for kw in ("except", "except*"): 1502 exec(code % kw, g, l); 1503 1504 def test_regression_gh_120225(self): 1505 async def name_4(): 1506 match b'': 1507 case True: 1508 pass 1509 case name_5 if f'e': 1510 {name_3: name_4 async for name_2 in name_5} 1511 case []: 1512 pass 1513 [[]] 1514 1515@requires_debug_ranges() 1516class TestSourcePositions(unittest.TestCase): 1517 # Ensure that compiled code snippets have correct line and column numbers 1518 # in `co_positions()`. 1519 1520 def check_positions_against_ast(self, snippet): 1521 # Basic check that makes sure each line and column is at least present 1522 # in one of the AST nodes of the source code. 1523 code = compile(snippet, 'test_compile.py', 'exec') 1524 ast_tree = compile(snippet, 'test_compile.py', 'exec', _ast.PyCF_ONLY_AST) 1525 self.assertTrue(type(ast_tree) == _ast.Module) 1526 1527 # Use an AST visitor that notes all the offsets. 1528 lines, end_lines, columns, end_columns = set(), set(), set(), set() 1529 class SourceOffsetVisitor(ast.NodeVisitor): 1530 def generic_visit(self, node): 1531 super().generic_visit(node) 1532 if not isinstance(node, (ast.expr, ast.stmt, ast.pattern)): 1533 return 1534 lines.add(node.lineno) 1535 end_lines.add(node.end_lineno) 1536 columns.add(node.col_offset) 1537 end_columns.add(node.end_col_offset) 1538 1539 SourceOffsetVisitor().visit(ast_tree) 1540 1541 # Check against the positions in the code object. 1542 for (line, end_line, col, end_col) in code.co_positions(): 1543 if line == 0: 1544 continue # This is an artificial module-start line 1545 # If the offset is not None (indicating missing data), ensure that 1546 # it was part of one of the AST nodes. 1547 if line is not None: 1548 self.assertIn(line, lines) 1549 if end_line is not None: 1550 self.assertIn(end_line, end_lines) 1551 if col is not None: 1552 self.assertIn(col, columns) 1553 if end_col is not None: 1554 self.assertIn(end_col, end_columns) 1555 1556 return code, ast_tree 1557 1558 def assertOpcodeSourcePositionIs(self, code, opcode, 1559 line, end_line, column, end_column, occurrence=1): 1560 1561 for instr, position in instructions_with_positions( 1562 dis.Bytecode(code), code.co_positions() 1563 ): 1564 if instr.opname == opcode: 1565 occurrence -= 1 1566 if not occurrence: 1567 self.assertEqual(position[0], line) 1568 self.assertEqual(position[1], end_line) 1569 self.assertEqual(position[2], column) 1570 self.assertEqual(position[3], end_column) 1571 return 1572 1573 self.fail(f"Opcode {opcode} not found in code") 1574 1575 def test_simple_assignment(self): 1576 snippet = "x = 1" 1577 self.check_positions_against_ast(snippet) 1578 1579 def test_compiles_to_extended_op_arg(self): 1580 # Make sure we still have valid positions when the code compiles to an 1581 # EXTENDED_ARG by performing a loop which needs a JUMP_ABSOLUTE after 1582 # a bunch of opcodes. 1583 snippet = "x = x\n" * 10_000 1584 snippet += ("while x != 0:\n" 1585 " x -= 1\n" 1586 "while x != 0:\n" 1587 " x += 1\n" 1588 ) 1589 1590 compiled_code, _ = self.check_positions_against_ast(snippet) 1591 1592 self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', 1593 line=10_000 + 2, end_line=10_000 + 2, 1594 column=2, end_column=8, occurrence=1) 1595 self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', 1596 line=10_000 + 4, end_line=10_000 + 4, 1597 column=2, end_column=9, occurrence=2) 1598 1599 def test_multiline_expression(self): 1600 snippet = textwrap.dedent("""\ 1601 f( 1602 1, 2, 3, 4 1603 ) 1604 """) 1605 compiled_code, _ = self.check_positions_against_ast(snippet) 1606 self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', 1607 line=1, end_line=3, column=0, end_column=1) 1608 1609 @requires_specialization 1610 def test_multiline_boolean_expression(self): 1611 snippet = textwrap.dedent("""\ 1612 if (a or 1613 (b and not c) or 1614 not ( 1615 d > 0)): 1616 x = 42 1617 """) 1618 compiled_code, _ = self.check_positions_against_ast(snippet) 1619 # jump if a is true: 1620 self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', 1621 line=1, end_line=1, column=4, end_column=5, occurrence=1) 1622 # jump if b is false: 1623 self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_FALSE', 1624 line=2, end_line=2, column=5, end_column=6, occurrence=1) 1625 # jump if c is false: 1626 self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_FALSE', 1627 line=2, end_line=2, column=15, end_column=16, occurrence=2) 1628 # compare d and 0 1629 self.assertOpcodeSourcePositionIs(compiled_code, 'COMPARE_OP', 1630 line=4, end_line=4, column=8, end_column=13, occurrence=1) 1631 # jump if comparison it True 1632 self.assertOpcodeSourcePositionIs(compiled_code, 'POP_JUMP_IF_TRUE', 1633 line=4, end_line=4, column=8, end_column=13, occurrence=2) 1634 1635 @unittest.skipIf(sys.flags.optimize, "Assertions are disabled in optimized mode") 1636 def test_multiline_assert(self): 1637 snippet = textwrap.dedent("""\ 1638 assert (a > 0 and 1639 bb > 0 and 1640 ccc == 1000000), "error msg" 1641 """) 1642 compiled_code, _ = self.check_positions_against_ast(snippet) 1643 self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_ASSERTION_ERROR', 1644 line=1, end_line=3, column=0, end_column=36, occurrence=1) 1645 # The "error msg": 1646 self.assertOpcodeSourcePositionIs(compiled_code, 'LOAD_CONST', 1647 line=3, end_line=3, column=25, end_column=36, occurrence=4) 1648 self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', 1649 line=1, end_line=3, column=0, end_column=36, occurrence=1) 1650 self.assertOpcodeSourcePositionIs(compiled_code, 'RAISE_VARARGS', 1651 line=1, end_line=3, column=8, end_column=22, occurrence=1) 1652 1653 def test_multiline_generator_expression(self): 1654 snippet = textwrap.dedent("""\ 1655 ((x, 1656 2*x) 1657 for x 1658 in [1,2,3] if (x > 0 1659 and x < 100 1660 and x != 50)) 1661 """) 1662 compiled_code, _ = self.check_positions_against_ast(snippet) 1663 compiled_code = compiled_code.co_consts[0] 1664 self.assertIsInstance(compiled_code, types.CodeType) 1665 self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE', 1666 line=1, end_line=2, column=1, end_column=8, occurrence=1) 1667 self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', 1668 line=1, end_line=2, column=1, end_column=8, occurrence=1) 1669 self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', 1670 line=4, end_line=4, column=7, end_column=14, occurrence=1) 1671 1672 def test_multiline_async_generator_expression(self): 1673 snippet = textwrap.dedent("""\ 1674 ((x, 1675 2*x) 1676 async for x 1677 in [1,2,3] if (x > 0 1678 and x < 100 1679 and x != 50)) 1680 """) 1681 compiled_code, _ = self.check_positions_against_ast(snippet) 1682 compiled_code = compiled_code.co_consts[0] 1683 self.assertIsInstance(compiled_code, types.CodeType) 1684 self.assertOpcodeSourcePositionIs(compiled_code, 'YIELD_VALUE', 1685 line=1, end_line=2, column=1, end_column=8, occurrence=2) 1686 self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', 1687 line=1, end_line=6, column=0, end_column=32, occurrence=1) 1688 1689 def test_multiline_list_comprehension(self): 1690 snippet = textwrap.dedent("""\ 1691 [(x, 1692 2*x) 1693 for x 1694 in [1,2,3] if (x > 0 1695 and x < 100 1696 and x != 50)] 1697 """) 1698 compiled_code, _ = self.check_positions_against_ast(snippet) 1699 self.assertIsInstance(compiled_code, types.CodeType) 1700 self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', 1701 line=1, end_line=2, column=1, end_column=8, occurrence=1) 1702 self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', 1703 line=1, end_line=2, column=1, end_column=8, occurrence=1) 1704 1705 def test_multiline_async_list_comprehension(self): 1706 snippet = textwrap.dedent("""\ 1707 async def f(): 1708 [(x, 1709 2*x) 1710 async for x 1711 in [1,2,3] if (x > 0 1712 and x < 100 1713 and x != 50)] 1714 """) 1715 compiled_code, _ = self.check_positions_against_ast(snippet) 1716 g = {} 1717 eval(compiled_code, g) 1718 compiled_code = g['f'].__code__ 1719 self.assertIsInstance(compiled_code, types.CodeType) 1720 self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', 1721 line=2, end_line=3, column=5, end_column=12, occurrence=1) 1722 self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', 1723 line=2, end_line=3, column=5, end_column=12, occurrence=1) 1724 self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', 1725 line=2, end_line=7, column=4, end_column=36, occurrence=1) 1726 1727 def test_multiline_set_comprehension(self): 1728 snippet = textwrap.dedent("""\ 1729 {(x, 1730 2*x) 1731 for x 1732 in [1,2,3] if (x > 0 1733 and x < 100 1734 and x != 50)} 1735 """) 1736 compiled_code, _ = self.check_positions_against_ast(snippet) 1737 self.assertIsInstance(compiled_code, types.CodeType) 1738 self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', 1739 line=1, end_line=2, column=1, end_column=8, occurrence=1) 1740 self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', 1741 line=1, end_line=2, column=1, end_column=8, occurrence=1) 1742 1743 def test_multiline_async_set_comprehension(self): 1744 snippet = textwrap.dedent("""\ 1745 async def f(): 1746 {(x, 1747 2*x) 1748 async for x 1749 in [1,2,3] if (x > 0 1750 and x < 100 1751 and x != 50)} 1752 """) 1753 compiled_code, _ = self.check_positions_against_ast(snippet) 1754 g = {} 1755 eval(compiled_code, g) 1756 compiled_code = g['f'].__code__ 1757 self.assertIsInstance(compiled_code, types.CodeType) 1758 self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', 1759 line=2, end_line=3, column=5, end_column=12, occurrence=1) 1760 self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', 1761 line=2, end_line=3, column=5, end_column=12, occurrence=1) 1762 self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', 1763 line=2, end_line=7, column=4, end_column=36, occurrence=1) 1764 1765 def test_multiline_dict_comprehension(self): 1766 snippet = textwrap.dedent("""\ 1767 {x: 1768 2*x 1769 for x 1770 in [1,2,3] if (x > 0 1771 and x < 100 1772 and x != 50)} 1773 """) 1774 compiled_code, _ = self.check_positions_against_ast(snippet) 1775 self.assertIsInstance(compiled_code, types.CodeType) 1776 self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', 1777 line=1, end_line=2, column=1, end_column=7, occurrence=1) 1778 self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', 1779 line=1, end_line=2, column=1, end_column=7, occurrence=1) 1780 1781 def test_multiline_async_dict_comprehension(self): 1782 snippet = textwrap.dedent("""\ 1783 async def f(): 1784 {x: 1785 2*x 1786 async for x 1787 in [1,2,3] if (x > 0 1788 and x < 100 1789 and x != 50)} 1790 """) 1791 compiled_code, _ = self.check_positions_against_ast(snippet) 1792 g = {} 1793 eval(compiled_code, g) 1794 compiled_code = g['f'].__code__ 1795 self.assertIsInstance(compiled_code, types.CodeType) 1796 self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', 1797 line=2, end_line=3, column=5, end_column=11, occurrence=1) 1798 self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', 1799 line=2, end_line=3, column=5, end_column=11, occurrence=1) 1800 self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_CONST', 1801 line=2, end_line=7, column=4, end_column=36, occurrence=1) 1802 1803 def test_matchcase_sequence(self): 1804 snippet = textwrap.dedent("""\ 1805 match x: 1806 case a, b: 1807 pass 1808 """) 1809 compiled_code, _ = self.check_positions_against_ast(snippet) 1810 self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', 1811 line=2, end_line=2, column=9, end_column=13, occurrence=1) 1812 self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_SEQUENCE', 1813 line=2, end_line=2, column=9, end_column=13, occurrence=1) 1814 self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', 1815 line=2, end_line=2, column=9, end_column=13, occurrence=1) 1816 self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', 1817 line=2, end_line=2, column=9, end_column=13, occurrence=2) 1818 1819 def test_matchcase_sequence_wildcard(self): 1820 snippet = textwrap.dedent("""\ 1821 match x: 1822 case a, *b, c: 1823 pass 1824 """) 1825 compiled_code, _ = self.check_positions_against_ast(snippet) 1826 self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_SEQUENCE', 1827 line=2, end_line=2, column=9, end_column=17, occurrence=1) 1828 self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_EX', 1829 line=2, end_line=2, column=9, end_column=17, occurrence=1) 1830 self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', 1831 line=2, end_line=2, column=9, end_column=17, occurrence=1) 1832 self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', 1833 line=2, end_line=2, column=9, end_column=17, occurrence=2) 1834 self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', 1835 line=2, end_line=2, column=9, end_column=17, occurrence=3) 1836 1837 def test_matchcase_mapping(self): 1838 snippet = textwrap.dedent("""\ 1839 match x: 1840 case {"a" : a, "b": b}: 1841 pass 1842 """) 1843 compiled_code, _ = self.check_positions_against_ast(snippet) 1844 self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', 1845 line=2, end_line=2, column=9, end_column=26, occurrence=1) 1846 self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_KEYS', 1847 line=2, end_line=2, column=9, end_column=26, occurrence=1) 1848 self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', 1849 line=2, end_line=2, column=9, end_column=26, occurrence=1) 1850 self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', 1851 line=2, end_line=2, column=9, end_column=26, occurrence=2) 1852 1853 def test_matchcase_mapping_wildcard(self): 1854 snippet = textwrap.dedent("""\ 1855 match x: 1856 case {"a" : a, "b": b, **c}: 1857 pass 1858 """) 1859 compiled_code, _ = self.check_positions_against_ast(snippet) 1860 self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_MAPPING', 1861 line=2, end_line=2, column=9, end_column=31, occurrence=1) 1862 self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_KEYS', 1863 line=2, end_line=2, column=9, end_column=31, occurrence=1) 1864 self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', 1865 line=2, end_line=2, column=9, end_column=31, occurrence=1) 1866 self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', 1867 line=2, end_line=2, column=9, end_column=31, occurrence=2) 1868 1869 def test_matchcase_class(self): 1870 snippet = textwrap.dedent("""\ 1871 match x: 1872 case C(a, b): 1873 pass 1874 """) 1875 compiled_code, _ = self.check_positions_against_ast(snippet) 1876 self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', 1877 line=2, end_line=2, column=9, end_column=16, occurrence=1) 1878 self.assertOpcodeSourcePositionIs(compiled_code, 'UNPACK_SEQUENCE', 1879 line=2, end_line=2, column=9, end_column=16, occurrence=1) 1880 self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', 1881 line=2, end_line=2, column=9, end_column=16, occurrence=1) 1882 self.assertOpcodeSourcePositionIs(compiled_code, 'STORE_NAME', 1883 line=2, end_line=2, column=9, end_column=16, occurrence=2) 1884 1885 def test_matchcase_or(self): 1886 snippet = textwrap.dedent("""\ 1887 match x: 1888 case C(1) | C(2): 1889 pass 1890 """) 1891 compiled_code, _ = self.check_positions_against_ast(snippet) 1892 self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', 1893 line=2, end_line=2, column=9, end_column=13, occurrence=1) 1894 self.assertOpcodeSourcePositionIs(compiled_code, 'MATCH_CLASS', 1895 line=2, end_line=2, column=16, end_column=20, occurrence=2) 1896 1897 def test_very_long_line_end_offset(self): 1898 # Make sure we get the correct column offset for offsets 1899 # too large to store in a byte. 1900 long_string = "a" * 1000 1901 snippet = f"g('{long_string}')" 1902 1903 compiled_code, _ = self.check_positions_against_ast(snippet) 1904 self.assertOpcodeSourcePositionIs(compiled_code, 'CALL', 1905 line=1, end_line=1, column=0, end_column=1005) 1906 1907 def test_complex_single_line_expression(self): 1908 snippet = "a - b @ (c * x['key'] + 23)" 1909 1910 compiled_code, _ = self.check_positions_against_ast(snippet) 1911 self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_SUBSCR', 1912 line=1, end_line=1, column=13, end_column=21) 1913 self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', 1914 line=1, end_line=1, column=9, end_column=21, occurrence=1) 1915 self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', 1916 line=1, end_line=1, column=9, end_column=26, occurrence=2) 1917 self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', 1918 line=1, end_line=1, column=4, end_column=27, occurrence=3) 1919 self.assertOpcodeSourcePositionIs(compiled_code, 'BINARY_OP', 1920 line=1, end_line=1, column=0, end_column=27, occurrence=4) 1921 1922 def test_multiline_assert_rewritten_as_method_call(self): 1923 # GH-94694: Don't crash if pytest rewrites a multiline assert as a 1924 # method call with the same location information: 1925 tree = ast.parse("assert (\n42\n)") 1926 old_node = tree.body[0] 1927 new_node = ast.Expr( 1928 ast.Call( 1929 ast.Attribute( 1930 ast.Name("spam", ast.Load()), 1931 "eggs", 1932 ast.Load(), 1933 ), 1934 [], 1935 [], 1936 ) 1937 ) 1938 ast.copy_location(new_node, old_node) 1939 ast.fix_missing_locations(new_node) 1940 tree.body[0] = new_node 1941 compile(tree, "<test>", "exec") 1942 1943 def test_push_null_load_global_positions(self): 1944 source_template = """ 1945 import abc, dis 1946 import ast as art 1947 1948 abc = None 1949 dix = dis 1950 ast = art 1951 1952 def f(): 1953 {} 1954 """ 1955 for body in [ 1956 " abc.a()", 1957 " art.a()", 1958 " ast.a()", 1959 " dis.a()", 1960 " dix.a()", 1961 " abc[...]()", 1962 " art()()", 1963 " (ast or ...)()", 1964 " [dis]()", 1965 " (dix + ...)()", 1966 ]: 1967 with self.subTest(body): 1968 namespace = {} 1969 source = textwrap.dedent(source_template.format(body)) 1970 with warnings.catch_warnings(): 1971 warnings.simplefilter('ignore', SyntaxWarning) 1972 exec(source, namespace) 1973 code = namespace["f"].__code__ 1974 self.assertOpcodeSourcePositionIs( 1975 code, 1976 "LOAD_GLOBAL", 1977 line=10, 1978 end_line=10, 1979 column=4, 1980 end_column=7, 1981 ) 1982 1983 def test_attribute_augassign(self): 1984 source = "(\n lhs \n . \n rhs \n ) += 42" 1985 code = compile(source, "<test>", "exec") 1986 self.assertOpcodeSourcePositionIs( 1987 code, "LOAD_ATTR", line=4, end_line=4, column=5, end_column=8 1988 ) 1989 self.assertOpcodeSourcePositionIs( 1990 code, "STORE_ATTR", line=4, end_line=4, column=5, end_column=8 1991 ) 1992 1993 def test_attribute_del(self): 1994 source = "del (\n lhs \n . \n rhs \n )" 1995 code = compile(source, "<test>", "exec") 1996 self.assertOpcodeSourcePositionIs( 1997 code, "DELETE_ATTR", line=4, end_line=4, column=5, end_column=8 1998 ) 1999 2000 def test_attribute_load(self): 2001 source = "(\n lhs \n . \n rhs \n )" 2002 code = compile(source, "<test>", "exec") 2003 self.assertOpcodeSourcePositionIs( 2004 code, "LOAD_ATTR", line=4, end_line=4, column=5, end_column=8 2005 ) 2006 2007 def test_attribute_store(self): 2008 source = "(\n lhs \n . \n rhs \n ) = 42" 2009 code = compile(source, "<test>", "exec") 2010 self.assertOpcodeSourcePositionIs( 2011 code, "STORE_ATTR", line=4, end_line=4, column=5, end_column=8 2012 ) 2013 2014 def test_method_call(self): 2015 source = "(\n lhs \n . \n rhs \n )()" 2016 code = compile(source, "<test>", "exec") 2017 self.assertOpcodeSourcePositionIs( 2018 code, "LOAD_ATTR", line=4, end_line=4, column=5, end_column=8 2019 ) 2020 self.assertOpcodeSourcePositionIs( 2021 code, "CALL", line=4, end_line=5, column=5, end_column=10 2022 ) 2023 2024 def test_weird_attribute_position_regressions(self): 2025 def f(): 2026 (bar. 2027 baz) 2028 (bar. 2029 baz( 2030 )) 2031 files().setdefault( 2032 0 2033 ).setdefault( 2034 0 2035 ) 2036 for line, end_line, column, end_column in f.__code__.co_positions(): 2037 self.assertIsNotNone(line) 2038 self.assertIsNotNone(end_line) 2039 self.assertIsNotNone(column) 2040 self.assertIsNotNone(end_column) 2041 self.assertLessEqual((line, column), (end_line, end_column)) 2042 2043 @support.cpython_only 2044 def test_column_offset_deduplication(self): 2045 # GH-95150: Code with different column offsets shouldn't be merged! 2046 for source in [ 2047 "lambda: a", 2048 "(a for b in c)", 2049 ]: 2050 with self.subTest(source): 2051 code = compile(f"{source}, {source}", "<test>", "eval") 2052 self.assertEqual(len(code.co_consts), 2) 2053 self.assertIsInstance(code.co_consts[0], types.CodeType) 2054 self.assertIsInstance(code.co_consts[1], types.CodeType) 2055 self.assertNotEqual(code.co_consts[0], code.co_consts[1]) 2056 self.assertNotEqual( 2057 list(code.co_consts[0].co_positions()), 2058 list(code.co_consts[1].co_positions()), 2059 ) 2060 2061 def test_load_super_attr(self): 2062 source = "class C:\n def __init__(self):\n super().__init__()" 2063 for const in compile(source, "<test>", "exec").co_consts[0].co_consts: 2064 if isinstance(const, types.CodeType): 2065 code = const 2066 break 2067 self.assertOpcodeSourcePositionIs( 2068 code, "LOAD_GLOBAL", line=3, end_line=3, column=4, end_column=9 2069 ) 2070 2071 def test_lambda_return_position(self): 2072 snippets = [ 2073 "f = lambda: x", 2074 "f = lambda: 42", 2075 "f = lambda: 1 + 2", 2076 "f = lambda: a + b", 2077 ] 2078 for snippet in snippets: 2079 with self.subTest(snippet=snippet): 2080 lamb = run_code(snippet)["f"] 2081 positions = lamb.__code__.co_positions() 2082 # assert that all positions are within the lambda 2083 for i, pos in enumerate(positions): 2084 with self.subTest(i=i, pos=pos): 2085 start_line, end_line, start_col, end_col = pos 2086 if i == 0 and start_col == end_col == 0: 2087 # ignore the RESUME in the beginning 2088 continue 2089 self.assertEqual(start_line, 1) 2090 self.assertEqual(end_line, 1) 2091 code_start = snippet.find(":") + 2 2092 code_end = len(snippet) 2093 self.assertGreaterEqual(start_col, code_start) 2094 self.assertLessEqual(end_col, code_end) 2095 self.assertGreaterEqual(end_col, start_col) 2096 self.assertLessEqual(end_col, code_end) 2097 2098 def test_return_in_with_positions(self): 2099 # See gh-98442 2100 def f(): 2101 with xyz: 2102 1 2103 2 2104 3 2105 4 2106 return R 2107 2108 # All instructions should have locations on a single line 2109 for instr in dis.get_instructions(f): 2110 start_line, end_line, _, _ = instr.positions 2111 self.assertEqual(start_line, end_line) 2112 2113 # Expect three load None instructions for the no-exception __exit__ call, 2114 # and one RETURN_VALUE. 2115 # They should all have the locations of the context manager ('xyz'). 2116 2117 load_none = [instr for instr in dis.get_instructions(f) if 2118 instr.opname == 'LOAD_CONST' and instr.argval is None] 2119 return_value = [instr for instr in dis.get_instructions(f) if 2120 instr.opname == 'RETURN_VALUE'] 2121 2122 self.assertEqual(len(load_none), 3) 2123 self.assertEqual(len(return_value), 1) 2124 for instr in load_none + return_value: 2125 start_line, end_line, start_col, end_col = instr.positions 2126 self.assertEqual(start_line, f.__code__.co_firstlineno + 1) 2127 self.assertEqual(end_line, f.__code__.co_firstlineno + 1) 2128 self.assertEqual(start_col, 17) 2129 self.assertEqual(end_col, 20) 2130 2131 2132class TestStaticAttributes(unittest.TestCase): 2133 2134 def test_basic(self): 2135 class C: 2136 def f(self): 2137 self.a = self.b = 42 2138 # read fields are not included 2139 self.f() 2140 self.arr[3] 2141 2142 self.assertIsInstance(C.__static_attributes__, tuple) 2143 self.assertEqual(sorted(C.__static_attributes__), ['a', 'b']) 2144 2145 def test_nested_function(self): 2146 class C: 2147 def f(self): 2148 self.x = 1 2149 self.y = 2 2150 self.x = 3 # check deduplication 2151 2152 def g(self, obj): 2153 self.y = 4 2154 self.z = 5 2155 2156 def h(self, a): 2157 self.u = 6 2158 self.v = 7 2159 2160 obj.self = 8 2161 2162 self.assertEqual(sorted(C.__static_attributes__), ['u', 'v', 'x', 'y', 'z']) 2163 2164 def test_nested_class(self): 2165 class C: 2166 def f(self): 2167 self.x = 42 2168 self.y = 42 2169 2170 class D: 2171 def g(self): 2172 self.y = 42 2173 self.z = 42 2174 2175 self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) 2176 self.assertEqual(sorted(C.D.__static_attributes__), ['y', 'z']) 2177 2178 def test_subclass(self): 2179 class C: 2180 def f(self): 2181 self.x = 42 2182 self.y = 42 2183 2184 class D(C): 2185 def g(self): 2186 self.y = 42 2187 self.z = 42 2188 2189 self.assertEqual(sorted(C.__static_attributes__), ['x', 'y']) 2190 self.assertEqual(sorted(D.__static_attributes__), ['y', 'z']) 2191 2192 2193class TestExpressionStackSize(unittest.TestCase): 2194 # These tests check that the computed stack size for a code object 2195 # stays within reasonable bounds (see issue #21523 for an example 2196 # dysfunction). 2197 N = 100 2198 2199 def check_stack_size(self, code): 2200 # To assert that the alleged stack size is not O(N), we 2201 # check that it is smaller than log(N). 2202 if isinstance(code, str): 2203 code = compile(code, "<foo>", "single") 2204 max_size = math.ceil(math.log(len(code.co_code))) 2205 self.assertLessEqual(code.co_stacksize, max_size) 2206 2207 def test_and(self): 2208 self.check_stack_size("x and " * self.N + "x") 2209 2210 def test_or(self): 2211 self.check_stack_size("x or " * self.N + "x") 2212 2213 def test_and_or(self): 2214 self.check_stack_size("x and x or " * self.N + "x") 2215 2216 def test_chained_comparison(self): 2217 self.check_stack_size("x < " * self.N + "x") 2218 2219 def test_if_else(self): 2220 self.check_stack_size("x if x else " * self.N + "x") 2221 2222 def test_binop(self): 2223 self.check_stack_size("x + " * self.N + "x") 2224 2225 def test_list(self): 2226 self.check_stack_size("[" + "x, " * self.N + "x]") 2227 2228 def test_tuple(self): 2229 self.check_stack_size("(" + "x, " * self.N + "x)") 2230 2231 def test_set(self): 2232 self.check_stack_size("{" + "x, " * self.N + "x}") 2233 2234 def test_dict(self): 2235 self.check_stack_size("{" + "x:x, " * self.N + "x:x}") 2236 2237 def test_func_args(self): 2238 self.check_stack_size("f(" + "x, " * self.N + ")") 2239 2240 def test_func_kwargs(self): 2241 kwargs = (f'a{i}=x' for i in range(self.N)) 2242 self.check_stack_size("f(" + ", ".join(kwargs) + ")") 2243 2244 def test_meth_args(self): 2245 self.check_stack_size("o.m(" + "x, " * self.N + ")") 2246 2247 def test_meth_kwargs(self): 2248 kwargs = (f'a{i}=x' for i in range(self.N)) 2249 self.check_stack_size("o.m(" + ", ".join(kwargs) + ")") 2250 2251 def test_func_and(self): 2252 code = "def f(x):\n" 2253 code += " x and x\n" * self.N 2254 self.check_stack_size(code) 2255 2256 def test_stack_3050(self): 2257 M = 3050 2258 code = "x," * M + "=t" 2259 # This raised on 3.10.0 to 3.10.5 2260 compile(code, "<foo>", "single") 2261 2262 def test_stack_3050_2(self): 2263 M = 3050 2264 args = ", ".join(f"arg{i}:type{i}" for i in range(M)) 2265 code = f"def f({args}):\n pass" 2266 # This raised on 3.10.0 to 3.10.5 2267 compile(code, "<foo>", "single") 2268 2269 2270class TestStackSizeStability(unittest.TestCase): 2271 # Check that repeating certain snippets doesn't increase the stack size 2272 # beyond what a single snippet requires. 2273 2274 def check_stack_size(self, snippet, async_=False): 2275 def compile_snippet(i): 2276 ns = {} 2277 script = """def func():\n""" + i * snippet 2278 if async_: 2279 script = "async " + script 2280 code = compile(script, "<script>", "exec") 2281 exec(code, ns, ns) 2282 return ns['func'].__code__ 2283 2284 sizes = [compile_snippet(i).co_stacksize for i in range(2, 5)] 2285 if len(set(sizes)) != 1: 2286 import dis, io 2287 out = io.StringIO() 2288 dis.dis(compile_snippet(1), file=out) 2289 self.fail("stack sizes diverge with # of consecutive snippets: " 2290 "%s\n%s\n%s" % (sizes, snippet, out.getvalue())) 2291 2292 def test_if(self): 2293 snippet = """ 2294 if x: 2295 a 2296 """ 2297 self.check_stack_size(snippet) 2298 2299 def test_if_else(self): 2300 snippet = """ 2301 if x: 2302 a 2303 elif y: 2304 b 2305 else: 2306 c 2307 """ 2308 self.check_stack_size(snippet) 2309 2310 def test_try_except_bare(self): 2311 snippet = """ 2312 try: 2313 a 2314 except: 2315 b 2316 """ 2317 self.check_stack_size(snippet) 2318 2319 def test_try_except_qualified(self): 2320 snippet = """ 2321 try: 2322 a 2323 except ImportError: 2324 b 2325 except: 2326 c 2327 else: 2328 d 2329 """ 2330 self.check_stack_size(snippet) 2331 2332 def test_try_except_as(self): 2333 snippet = """ 2334 try: 2335 a 2336 except ImportError as e: 2337 b 2338 except: 2339 c 2340 else: 2341 d 2342 """ 2343 self.check_stack_size(snippet) 2344 2345 def test_try_except_star_qualified(self): 2346 snippet = """ 2347 try: 2348 a 2349 except* ImportError: 2350 b 2351 else: 2352 c 2353 """ 2354 self.check_stack_size(snippet) 2355 2356 def test_try_except_star_as(self): 2357 snippet = """ 2358 try: 2359 a 2360 except* ImportError as e: 2361 b 2362 else: 2363 c 2364 """ 2365 self.check_stack_size(snippet) 2366 2367 def test_try_except_star_finally(self): 2368 snippet = """ 2369 try: 2370 a 2371 except* A: 2372 b 2373 finally: 2374 c 2375 """ 2376 self.check_stack_size(snippet) 2377 2378 def test_try_finally(self): 2379 snippet = """ 2380 try: 2381 a 2382 finally: 2383 b 2384 """ 2385 self.check_stack_size(snippet) 2386 2387 def test_with(self): 2388 snippet = """ 2389 with x as y: 2390 a 2391 """ 2392 self.check_stack_size(snippet) 2393 2394 def test_while_else(self): 2395 snippet = """ 2396 while x: 2397 a 2398 else: 2399 b 2400 """ 2401 self.check_stack_size(snippet) 2402 2403 def test_for(self): 2404 snippet = """ 2405 for x in y: 2406 a 2407 """ 2408 self.check_stack_size(snippet) 2409 2410 def test_for_else(self): 2411 snippet = """ 2412 for x in y: 2413 a 2414 else: 2415 b 2416 """ 2417 self.check_stack_size(snippet) 2418 2419 def test_for_break_continue(self): 2420 snippet = """ 2421 for x in y: 2422 if z: 2423 break 2424 elif u: 2425 continue 2426 else: 2427 a 2428 else: 2429 b 2430 """ 2431 self.check_stack_size(snippet) 2432 2433 def test_for_break_continue_inside_try_finally_block(self): 2434 snippet = """ 2435 for x in y: 2436 try: 2437 if z: 2438 break 2439 elif u: 2440 continue 2441 else: 2442 a 2443 finally: 2444 f 2445 else: 2446 b 2447 """ 2448 self.check_stack_size(snippet) 2449 2450 def test_for_break_continue_inside_finally_block(self): 2451 snippet = """ 2452 for x in y: 2453 try: 2454 t 2455 finally: 2456 if z: 2457 break 2458 elif u: 2459 continue 2460 else: 2461 a 2462 else: 2463 b 2464 """ 2465 self.check_stack_size(snippet) 2466 2467 def test_for_break_continue_inside_except_block(self): 2468 snippet = """ 2469 for x in y: 2470 try: 2471 t 2472 except: 2473 if z: 2474 break 2475 elif u: 2476 continue 2477 else: 2478 a 2479 else: 2480 b 2481 """ 2482 self.check_stack_size(snippet) 2483 2484 def test_for_break_continue_inside_with_block(self): 2485 snippet = """ 2486 for x in y: 2487 with c: 2488 if z: 2489 break 2490 elif u: 2491 continue 2492 else: 2493 a 2494 else: 2495 b 2496 """ 2497 self.check_stack_size(snippet) 2498 2499 def test_return_inside_try_finally_block(self): 2500 snippet = """ 2501 try: 2502 if z: 2503 return 2504 else: 2505 a 2506 finally: 2507 f 2508 """ 2509 self.check_stack_size(snippet) 2510 2511 def test_return_inside_finally_block(self): 2512 snippet = """ 2513 try: 2514 t 2515 finally: 2516 if z: 2517 return 2518 else: 2519 a 2520 """ 2521 self.check_stack_size(snippet) 2522 2523 def test_return_inside_except_block(self): 2524 snippet = """ 2525 try: 2526 t 2527 except: 2528 if z: 2529 return 2530 else: 2531 a 2532 """ 2533 self.check_stack_size(snippet) 2534 2535 def test_return_inside_with_block(self): 2536 snippet = """ 2537 with c: 2538 if z: 2539 return 2540 else: 2541 a 2542 """ 2543 self.check_stack_size(snippet) 2544 2545 def test_async_with(self): 2546 snippet = """ 2547 async with x as y: 2548 a 2549 """ 2550 self.check_stack_size(snippet, async_=True) 2551 2552 def test_async_for(self): 2553 snippet = """ 2554 async for x in y: 2555 a 2556 """ 2557 self.check_stack_size(snippet, async_=True) 2558 2559 def test_async_for_else(self): 2560 snippet = """ 2561 async for x in y: 2562 a 2563 else: 2564 b 2565 """ 2566 self.check_stack_size(snippet, async_=True) 2567 2568 def test_for_break_continue_inside_async_with_block(self): 2569 snippet = """ 2570 for x in y: 2571 async with c: 2572 if z: 2573 break 2574 elif u: 2575 continue 2576 else: 2577 a 2578 else: 2579 b 2580 """ 2581 self.check_stack_size(snippet, async_=True) 2582 2583 def test_return_inside_async_with_block(self): 2584 snippet = """ 2585 async with c: 2586 if z: 2587 return 2588 else: 2589 a 2590 """ 2591 self.check_stack_size(snippet, async_=True) 2592 2593class TestInstructionSequence(unittest.TestCase): 2594 def compare_instructions(self, seq, expected): 2595 self.assertEqual([(opcode.opname[i[0]],) + i[1:] for i in seq.get_instructions()], 2596 expected) 2597 2598 def test_basics(self): 2599 seq = _testinternalcapi.new_instruction_sequence() 2600 2601 def add_op(seq, opname, oparg, bl, bc=0, el=0, ec=0): 2602 seq.addop(opcode.opmap[opname], oparg, bl, bc, el, el) 2603 2604 add_op(seq, 'LOAD_CONST', 1, 1) 2605 add_op(seq, 'JUMP', lbl1 := seq.new_label(), 2) 2606 add_op(seq, 'LOAD_CONST', 1, 3) 2607 add_op(seq, 'JUMP', lbl2 := seq.new_label(), 4) 2608 seq.use_label(lbl1) 2609 add_op(seq, 'LOAD_CONST', 2, 4) 2610 seq.use_label(lbl2) 2611 add_op(seq, 'RETURN_VALUE', 0, 3) 2612 2613 expected = [('LOAD_CONST', 1, 1), 2614 ('JUMP', 4, 2), 2615 ('LOAD_CONST', 1, 3), 2616 ('JUMP', 5, 4), 2617 ('LOAD_CONST', 2, 4), 2618 ('RETURN_VALUE', None, 3), 2619 ] 2620 2621 self.compare_instructions(seq, [ex + (0,0,0) for ex in expected]) 2622 2623 def test_nested(self): 2624 seq = _testinternalcapi.new_instruction_sequence() 2625 seq.addop(opcode.opmap['LOAD_CONST'], 1, 1, 0, 0, 0) 2626 nested = _testinternalcapi.new_instruction_sequence() 2627 nested.addop(opcode.opmap['LOAD_CONST'], 2, 2, 0, 0, 0) 2628 2629 self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) 2630 self.compare_instructions(nested, [('LOAD_CONST', 2, 2, 0, 0, 0)]) 2631 2632 seq.add_nested(nested) 2633 self.compare_instructions(seq, [('LOAD_CONST', 1, 1, 0, 0, 0)]) 2634 self.compare_instructions(seq.get_nested()[0], [('LOAD_CONST', 2, 2, 0, 0, 0)]) 2635 2636 def test_static_attributes_are_sorted(self): 2637 code = ( 2638 'class T:\n' 2639 ' def __init__(self):\n' 2640 ' self.{V1} = 10\n' 2641 ' self.{V2} = 10\n' 2642 ' def foo(self):\n' 2643 ' self.{V3} = 10\n' 2644 ) 2645 attributes = ("a", "b", "c") 2646 for perm in itertools.permutations(attributes): 2647 var_names = {f'V{i + 1}': name for i, name in enumerate(perm)} 2648 ns = run_code(code.format(**var_names)) 2649 t = ns['T'] 2650 self.assertEqual(t.__static_attributes__, attributes) 2651 2652 2653if __name__ == "__main__": 2654 unittest.main() 2655