1import dis 2from itertools import combinations, product 3import opcode 4import sys 5import textwrap 6import unittest 7 8from test import support 9from test.support.bytecode_helper import BytecodeTestCase, CfgOptimizationTestCase 10 11 12def compile_pattern_with_fast_locals(pattern): 13 source = textwrap.dedent( 14 f""" 15 def f(x): 16 match x: 17 case {pattern}: 18 pass 19 """ 20 ) 21 namespace = {} 22 exec(source, namespace) 23 return namespace["f"].__code__ 24 25 26def count_instr_recursively(f, opname): 27 count = 0 28 for instr in dis.get_instructions(f): 29 if instr.opname == opname: 30 count += 1 31 if hasattr(f, '__code__'): 32 f = f.__code__ 33 for c in f.co_consts: 34 if hasattr(c, 'co_code'): 35 count += count_instr_recursively(c, opname) 36 return count 37 38 39class TestTranforms(BytecodeTestCase): 40 41 def check_jump_targets(self, code): 42 instructions = list(dis.get_instructions(code)) 43 targets = {instr.offset: instr for instr in instructions} 44 for instr in instructions: 45 if 'JUMP_' not in instr.opname: 46 continue 47 tgt = targets[instr.argval] 48 # jump to unconditional jump 49 if tgt.opname in ('JUMP_BACKWARD', 'JUMP_FORWARD'): 50 self.fail(f'{instr.opname} at {instr.offset} ' 51 f'jumps to {tgt.opname} at {tgt.offset}') 52 # unconditional jump to RETURN_VALUE 53 if (instr.opname in ('JUMP_BACKWARD', 'JUMP_FORWARD') and 54 tgt.opname == 'RETURN_VALUE'): 55 self.fail(f'{instr.opname} at {instr.offset} ' 56 f'jumps to {tgt.opname} at {tgt.offset}') 57 58 def check_lnotab(self, code): 59 "Check that the lnotab byte offsets are sensible." 60 code = dis._get_code_object(code) 61 lnotab = list(dis.findlinestarts(code)) 62 # Don't bother checking if the line info is sensible, because 63 # most of the line info we can get at comes from lnotab. 64 min_bytecode = min(t[0] for t in lnotab) 65 max_bytecode = max(t[0] for t in lnotab) 66 self.assertGreaterEqual(min_bytecode, 0) 67 self.assertLess(max_bytecode, len(code.co_code)) 68 # This could conceivably test more (and probably should, as there 69 # aren't very many tests of lnotab), if peepholer wasn't scheduled 70 # to be replaced anyway. 71 72 def test_unot(self): 73 # UNARY_NOT POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE' 74 def unot(x): 75 if not x == 2: 76 del x 77 self.assertNotInBytecode(unot, 'UNARY_NOT') 78 self.assertNotInBytecode(unot, 'POP_JUMP_IF_FALSE') 79 self.assertInBytecode(unot, 'POP_JUMP_IF_TRUE') 80 self.check_lnotab(unot) 81 82 def test_elim_inversion_of_is_or_in(self): 83 for line, cmp_op, invert in ( 84 ('not a is b', 'IS_OP', 1,), 85 ('not a is not b', 'IS_OP', 0,), 86 ('not a in b', 'CONTAINS_OP', 1,), 87 ('not a not in b', 'CONTAINS_OP', 0,), 88 ): 89 with self.subTest(line=line): 90 code = compile(line, '', 'single') 91 self.assertInBytecode(code, cmp_op, invert) 92 self.check_lnotab(code) 93 94 def test_global_as_constant(self): 95 # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False 96 def f(): 97 x = None 98 x = None 99 return x 100 def g(): 101 x = True 102 return x 103 def h(): 104 x = False 105 return x 106 107 for func, elem in ((f, None), (g, True), (h, False)): 108 with self.subTest(func=func): 109 self.assertNotInBytecode(func, 'LOAD_GLOBAL') 110 self.assertInBytecode(func, 'LOAD_CONST', elem) 111 self.check_lnotab(func) 112 113 def f(): 114 'Adding a docstring made this test fail in Py2.5.0' 115 return None 116 117 self.assertNotInBytecode(f, 'LOAD_GLOBAL') 118 self.assertInBytecode(f, 'RETURN_CONST', None) 119 self.check_lnotab(f) 120 121 def test_while_one(self): 122 # Skip over: LOAD_CONST trueconst POP_JUMP_IF_FALSE xx 123 def f(): 124 while 1: 125 pass 126 return list 127 for elem in ('LOAD_CONST', 'POP_JUMP_IF_FALSE'): 128 self.assertNotInBytecode(f, elem) 129 for elem in ('JUMP_BACKWARD',): 130 self.assertInBytecode(f, elem) 131 self.check_lnotab(f) 132 133 def test_pack_unpack(self): 134 for line, elem in ( 135 ('a, = a,', 'RETURN_CONST',), 136 ('a, b = a, b', 'SWAP',), 137 ('a, b, c = a, b, c', 'SWAP',), 138 ): 139 with self.subTest(line=line): 140 code = compile(line,'','single') 141 self.assertInBytecode(code, elem) 142 self.assertNotInBytecode(code, 'BUILD_TUPLE') 143 self.assertNotInBytecode(code, 'UNPACK_SEQUENCE') 144 self.check_lnotab(code) 145 146 def test_folding_of_tuples_of_constants(self): 147 for line, elem in ( 148 ('a = 1,2,3', (1, 2, 3)), 149 ('("a","b","c")', ('a', 'b', 'c')), 150 ('a,b,c = 1,2,3', (1, 2, 3)), 151 ('(None, 1, None)', (None, 1, None)), 152 ('((1, 2), 3, 4)', ((1, 2), 3, 4)), 153 ): 154 with self.subTest(line=line): 155 code = compile(line,'','single') 156 self.assertInBytecode(code, 'LOAD_CONST', elem) 157 self.assertNotInBytecode(code, 'BUILD_TUPLE') 158 self.check_lnotab(code) 159 160 # Long tuples should be folded too. 161 code = compile(repr(tuple(range(10000))),'','single') 162 self.assertNotInBytecode(code, 'BUILD_TUPLE') 163 # One LOAD_CONST for the tuple, one for the None return value 164 load_consts = [instr for instr in dis.get_instructions(code) 165 if instr.opname == 'LOAD_CONST'] 166 self.assertEqual(len(load_consts), 1) 167 self.check_lnotab(code) 168 169 # Bug 1053819: Tuple of constants misidentified when presented with: 170 # . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . . 171 # The following would segfault upon compilation 172 def crater(): 173 (~[ 174 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 175 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 176 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 177 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 178 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 179 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 180 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 181 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 182 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 183 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 184 ],) 185 self.check_lnotab(crater) 186 187 def test_folding_of_lists_of_constants(self): 188 for line, elem in ( 189 # in/not in constants with BUILD_LIST should be folded to a tuple: 190 ('a in [1,2,3]', (1, 2, 3)), 191 ('a not in ["a","b","c"]', ('a', 'b', 'c')), 192 ('a in [None, 1, None]', (None, 1, None)), 193 ('a not in [(1, 2), 3, 4]', ((1, 2), 3, 4)), 194 ): 195 with self.subTest(line=line): 196 code = compile(line, '', 'single') 197 self.assertInBytecode(code, 'LOAD_CONST', elem) 198 self.assertNotInBytecode(code, 'BUILD_LIST') 199 self.check_lnotab(code) 200 201 def test_folding_of_sets_of_constants(self): 202 for line, elem in ( 203 # in/not in constants with BUILD_SET should be folded to a frozenset: 204 ('a in {1,2,3}', frozenset({1, 2, 3})), 205 ('a not in {"a","b","c"}', frozenset({'a', 'c', 'b'})), 206 ('a in {None, 1, None}', frozenset({1, None})), 207 ('a not in {(1, 2), 3, 4}', frozenset({(1, 2), 3, 4})), 208 ('a in {1, 2, 3, 3, 2, 1}', frozenset({1, 2, 3})), 209 ): 210 with self.subTest(line=line): 211 code = compile(line, '', 'single') 212 self.assertNotInBytecode(code, 'BUILD_SET') 213 self.assertInBytecode(code, 'LOAD_CONST', elem) 214 self.check_lnotab(code) 215 216 # Ensure that the resulting code actually works: 217 def f(a): 218 return a in {1, 2, 3} 219 220 def g(a): 221 return a not in {1, 2, 3} 222 223 self.assertTrue(f(3)) 224 self.assertTrue(not f(4)) 225 self.check_lnotab(f) 226 227 self.assertTrue(not g(3)) 228 self.assertTrue(g(4)) 229 self.check_lnotab(g) 230 231 232 def test_folding_of_binops_on_constants(self): 233 for line, elem in ( 234 ('a = 2+3+4', 9), # chained fold 235 ('"@"*4', '@@@@'), # check string ops 236 ('a="abc" + "def"', 'abcdef'), # check string ops 237 ('a = 3**4', 81), # binary power 238 ('a = 3*4', 12), # binary multiply 239 ('a = 13//4', 3), # binary floor divide 240 ('a = 14%4', 2), # binary modulo 241 ('a = 2+3', 5), # binary add 242 ('a = 13-4', 9), # binary subtract 243 ('a = (12,13)[1]', 13), # binary subscr 244 ('a = 13 << 2', 52), # binary lshift 245 ('a = 13 >> 2', 3), # binary rshift 246 ('a = 13 & 7', 5), # binary and 247 ('a = 13 ^ 7', 10), # binary xor 248 ('a = 13 | 7', 15), # binary or 249 ): 250 with self.subTest(line=line): 251 code = compile(line, '', 'single') 252 self.assertInBytecode(code, 'LOAD_CONST', elem) 253 for instr in dis.get_instructions(code): 254 self.assertFalse(instr.opname.startswith('BINARY_')) 255 self.check_lnotab(code) 256 257 # Verify that unfoldables are skipped 258 code = compile('a=2+"b"', '', 'single') 259 self.assertInBytecode(code, 'LOAD_CONST', 2) 260 self.assertInBytecode(code, 'LOAD_CONST', 'b') 261 self.check_lnotab(code) 262 263 # Verify that large sequences do not result from folding 264 code = compile('a="x"*10000', '', 'single') 265 self.assertInBytecode(code, 'LOAD_CONST', 10000) 266 self.assertNotIn("x"*10000, code.co_consts) 267 self.check_lnotab(code) 268 code = compile('a=1<<1000', '', 'single') 269 self.assertInBytecode(code, 'LOAD_CONST', 1000) 270 self.assertNotIn(1<<1000, code.co_consts) 271 self.check_lnotab(code) 272 code = compile('a=2**1000', '', 'single') 273 self.assertInBytecode(code, 'LOAD_CONST', 1000) 274 self.assertNotIn(2**1000, code.co_consts) 275 self.check_lnotab(code) 276 277 def test_binary_subscr_on_unicode(self): 278 # valid code get optimized 279 code = compile('"foo"[0]', '', 'single') 280 self.assertInBytecode(code, 'LOAD_CONST', 'f') 281 self.assertNotInBytecode(code, 'BINARY_SUBSCR') 282 self.check_lnotab(code) 283 code = compile('"\u0061\uffff"[1]', '', 'single') 284 self.assertInBytecode(code, 'LOAD_CONST', '\uffff') 285 self.assertNotInBytecode(code,'BINARY_SUBSCR') 286 self.check_lnotab(code) 287 288 # With PEP 393, non-BMP char get optimized 289 code = compile('"\U00012345"[0]', '', 'single') 290 self.assertInBytecode(code, 'LOAD_CONST', '\U00012345') 291 self.assertNotInBytecode(code, 'BINARY_SUBSCR') 292 self.check_lnotab(code) 293 294 # invalid code doesn't get optimized 295 # out of range 296 code = compile('"fuu"[10]', '', 'single') 297 self.assertInBytecode(code, 'BINARY_SUBSCR') 298 self.check_lnotab(code) 299 300 def test_folding_of_unaryops_on_constants(self): 301 for line, elem in ( 302 ('-0.5', -0.5), # unary negative 303 ('-0.0', -0.0), # -0.0 304 ('-(1.0-1.0)', -0.0), # -0.0 after folding 305 ('-0', 0), # -0 306 ('~-2', 1), # unary invert 307 ('+1', 1), # unary positive 308 ): 309 with self.subTest(line=line): 310 code = compile(line, '', 'single') 311 self.assertInBytecode(code, 'LOAD_CONST', elem) 312 for instr in dis.get_instructions(code): 313 self.assertFalse(instr.opname.startswith('UNARY_')) 314 self.check_lnotab(code) 315 316 # Check that -0.0 works after marshaling 317 def negzero(): 318 return -(1.0-1.0) 319 320 for instr in dis.get_instructions(negzero): 321 self.assertFalse(instr.opname.startswith('UNARY_')) 322 self.check_lnotab(negzero) 323 324 # Verify that unfoldables are skipped 325 for line, elem, opname in ( 326 ('-"abc"', 'abc', 'UNARY_NEGATIVE'), 327 ('~"abc"', 'abc', 'UNARY_INVERT'), 328 ): 329 with self.subTest(line=line): 330 code = compile(line, '', 'single') 331 self.assertInBytecode(code, 'LOAD_CONST', elem) 332 self.assertInBytecode(code, opname) 333 self.check_lnotab(code) 334 335 def test_elim_extra_return(self): 336 # RETURN LOAD_CONST None RETURN --> RETURN 337 def f(x): 338 return x 339 self.assertNotInBytecode(f, 'LOAD_CONST', None) 340 returns = [instr for instr in dis.get_instructions(f) 341 if instr.opname == 'RETURN_VALUE'] 342 self.assertEqual(len(returns), 1) 343 self.check_lnotab(f) 344 345 def test_elim_jump_to_return(self): 346 # JUMP_FORWARD to RETURN --> RETURN 347 def f(cond, true_value, false_value): 348 # Intentionally use two-line expression to test issue37213. 349 return (true_value if cond 350 else false_value) 351 self.check_jump_targets(f) 352 self.assertNotInBytecode(f, 'JUMP_FORWARD') 353 self.assertNotInBytecode(f, 'JUMP_BACKWARD') 354 returns = [instr for instr in dis.get_instructions(f) 355 if instr.opname == 'RETURN_VALUE'] 356 self.assertEqual(len(returns), 2) 357 self.check_lnotab(f) 358 359 def test_elim_jump_to_uncond_jump(self): 360 # POP_JUMP_IF_FALSE to JUMP_FORWARD --> POP_JUMP_IF_FALSE to non-jump 361 def f(): 362 if a: 363 # Intentionally use two-line expression to test issue37213. 364 if (c 365 or d): 366 foo() 367 else: 368 baz() 369 self.check_jump_targets(f) 370 self.check_lnotab(f) 371 372 def test_elim_jump_to_uncond_jump2(self): 373 # POP_JUMP_IF_FALSE to JUMP_BACKWARD --> POP_JUMP_IF_FALSE to non-jump 374 def f(): 375 while a: 376 # Intentionally use two-line expression to test issue37213. 377 if (c 378 or d): 379 a = foo() 380 self.check_jump_targets(f) 381 self.check_lnotab(f) 382 383 def test_elim_jump_to_uncond_jump3(self): 384 # Intentionally use two-line expressions to test issue37213. 385 # POP_JUMP_IF_FALSE to POP_JUMP_IF_FALSE --> POP_JUMP_IF_FALSE to non-jump 386 def f(a, b, c): 387 return ((a and b) 388 and c) 389 self.check_jump_targets(f) 390 self.check_lnotab(f) 391 self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_FALSE'), 2) 392 # POP_JUMP_IF_TRUE to POP_JUMP_IF_TRUE --> POP_JUMP_IF_TRUE to non-jump 393 def f(a, b, c): 394 return ((a or b) 395 or c) 396 self.check_jump_targets(f) 397 self.check_lnotab(f) 398 self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_TRUE'), 2) 399 # JUMP_IF_FALSE_OR_POP to JUMP_IF_TRUE_OR_POP --> POP_JUMP_IF_FALSE to non-jump 400 def f(a, b, c): 401 return ((a and b) 402 or c) 403 self.check_jump_targets(f) 404 self.check_lnotab(f) 405 self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_FALSE'), 1) 406 self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_TRUE'), 1) 407 # POP_JUMP_IF_TRUE to POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE to non-jump 408 def f(a, b, c): 409 return ((a or b) 410 and c) 411 self.check_jump_targets(f) 412 self.check_lnotab(f) 413 self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_FALSE'), 1) 414 self.assertEqual(count_instr_recursively(f, 'POP_JUMP_IF_TRUE'), 1) 415 416 def test_elim_jump_to_uncond_jump4(self): 417 def f(): 418 for i in range(5): 419 if i > 3: 420 print(i) 421 self.check_jump_targets(f) 422 423 def test_elim_jump_after_return1(self): 424 # Eliminate dead code: jumps immediately after returns can't be reached 425 def f(cond1, cond2): 426 if cond1: return 1 427 if cond2: return 2 428 while 1: 429 return 3 430 while 1: 431 if cond1: return 4 432 return 5 433 return 6 434 self.assertNotInBytecode(f, 'JUMP_FORWARD') 435 self.assertNotInBytecode(f, 'JUMP_BACKWARD') 436 returns = [instr for instr in dis.get_instructions(f) 437 if instr.opname == 'RETURN_VALUE'] 438 self.assertLessEqual(len(returns), 6) 439 self.check_lnotab(f) 440 441 def test_make_function_doesnt_bail(self): 442 def f(): 443 def g()->1+1: 444 pass 445 return g 446 self.assertNotInBytecode(f, 'BINARY_OP') 447 self.check_lnotab(f) 448 449 def test_constant_folding(self): 450 # Issue #11244: aggressive constant folding. 451 exprs = [ 452 '3 * -5', 453 '-3 * 5', 454 '2 * (3 * 4)', 455 '(2 * 3) * 4', 456 '(-1, 2, 3)', 457 '(1, -2, 3)', 458 '(1, 2, -3)', 459 '(1, 2, -3) * 6', 460 'lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}', 461 ] 462 for e in exprs: 463 with self.subTest(e=e): 464 code = compile(e, '', 'single') 465 for instr in dis.get_instructions(code): 466 self.assertFalse(instr.opname.startswith('UNARY_')) 467 self.assertFalse(instr.opname.startswith('BINARY_')) 468 self.assertFalse(instr.opname.startswith('BUILD_')) 469 self.check_lnotab(code) 470 471 def test_in_literal_list(self): 472 def containtest(): 473 return x in [a, b] 474 self.assertEqual(count_instr_recursively(containtest, 'BUILD_LIST'), 0) 475 self.check_lnotab(containtest) 476 477 def test_iterate_literal_list(self): 478 def forloop(): 479 for x in [a, b]: 480 pass 481 self.assertEqual(count_instr_recursively(forloop, 'BUILD_LIST'), 0) 482 self.check_lnotab(forloop) 483 484 def test_condition_with_binop_with_bools(self): 485 def f(): 486 if True or False: 487 return 1 488 return 0 489 self.assertEqual(f(), 1) 490 self.check_lnotab(f) 491 492 def test_if_with_if_expression(self): 493 # Check bpo-37289 494 def f(x): 495 if (True if x else False): 496 return True 497 return False 498 self.assertTrue(f(True)) 499 self.check_lnotab(f) 500 501 def test_trailing_nops(self): 502 # Check the lnotab of a function that even after trivial 503 # optimization has trailing nops, which the lnotab adjustment has to 504 # handle properly (bpo-38115). 505 def f(x): 506 while 1: 507 return 3 508 while 1: 509 return 5 510 return 6 511 self.check_lnotab(f) 512 513 def test_assignment_idiom_in_comprehensions(self): 514 def listcomp(): 515 return [y for x in a for y in [f(x)]] 516 self.assertEqual(count_instr_recursively(listcomp, 'FOR_ITER'), 1) 517 def setcomp(): 518 return {y for x in a for y in [f(x)]} 519 self.assertEqual(count_instr_recursively(setcomp, 'FOR_ITER'), 1) 520 def dictcomp(): 521 return {y: y for x in a for y in [f(x)]} 522 self.assertEqual(count_instr_recursively(dictcomp, 'FOR_ITER'), 1) 523 def genexpr(): 524 return (y for x in a for y in [f(x)]) 525 self.assertEqual(count_instr_recursively(genexpr, 'FOR_ITER'), 1) 526 527 @support.requires_resource('cpu') 528 def test_format_combinations(self): 529 flags = '-+ #0' 530 testcases = [ 531 *product(('', '1234', 'абвг'), 'sra'), 532 *product((1234, -1234), 'duioxX'), 533 *product((1234.5678901, -1234.5678901), 'duifegFEG'), 534 *product((float('inf'), -float('inf')), 'fegFEG'), 535 ] 536 width_precs = [ 537 *product(('', '1', '30'), ('', '.', '.0', '.2')), 538 ('', '.40'), 539 ('30', '.40'), 540 ] 541 for value, suffix in testcases: 542 for width, prec in width_precs: 543 for r in range(len(flags) + 1): 544 for spec in combinations(flags, r): 545 fmt = '%' + ''.join(spec) + width + prec + suffix 546 with self.subTest(fmt=fmt, value=value): 547 s1 = fmt % value 548 s2 = eval(f'{fmt!r} % (x,)', {'x': value}) 549 self.assertEqual(s2, s1, f'{fmt = }') 550 551 def test_format_misc(self): 552 def format(fmt, *values): 553 vars = [f'x{i+1}' for i in range(len(values))] 554 if len(vars) == 1: 555 args = '(' + vars[0] + ',)' 556 else: 557 args = '(' + ', '.join(vars) + ')' 558 return eval(f'{fmt!r} % {args}', dict(zip(vars, values))) 559 560 self.assertEqual(format('string'), 'string') 561 self.assertEqual(format('x = %s!', 1234), 'x = 1234!') 562 self.assertEqual(format('x = %d!', 1234), 'x = 1234!') 563 self.assertEqual(format('x = %x!', 1234), 'x = 4d2!') 564 self.assertEqual(format('x = %f!', 1234), 'x = 1234.000000!') 565 self.assertEqual(format('x = %s!', 1234.5678901), 'x = 1234.5678901!') 566 self.assertEqual(format('x = %f!', 1234.5678901), 'x = 1234.567890!') 567 self.assertEqual(format('x = %d!', 1234.5678901), 'x = 1234!') 568 self.assertEqual(format('x = %s%% %%%%', 1234), 'x = 1234% %%') 569 self.assertEqual(format('x = %s!', '%% %s'), 'x = %% %s!') 570 self.assertEqual(format('x = %s, y = %d', 12, 34), 'x = 12, y = 34') 571 572 def test_format_errors(self): 573 with self.assertRaisesRegex(TypeError, 574 'not enough arguments for format string'): 575 eval("'%s' % ()") 576 with self.assertRaisesRegex(TypeError, 577 'not all arguments converted during string formatting'): 578 eval("'%s' % (x, y)", {'x': 1, 'y': 2}) 579 with self.assertRaisesRegex(ValueError, 'incomplete format'): 580 eval("'%s%' % (x,)", {'x': 1234}) 581 with self.assertRaisesRegex(ValueError, 'incomplete format'): 582 eval("'%s%%%' % (x,)", {'x': 1234}) 583 with self.assertRaisesRegex(TypeError, 584 'not enough arguments for format string'): 585 eval("'%s%z' % (x,)", {'x': 1234}) 586 with self.assertRaisesRegex(ValueError, 'unsupported format character'): 587 eval("'%s%z' % (x, 5)", {'x': 1234}) 588 with self.assertRaisesRegex(TypeError, 'a real number is required, not str'): 589 eval("'%d' % (x,)", {'x': '1234'}) 590 with self.assertRaisesRegex(TypeError, 'an integer is required, not float'): 591 eval("'%x' % (x,)", {'x': 1234.56}) 592 with self.assertRaisesRegex(TypeError, 'an integer is required, not str'): 593 eval("'%x' % (x,)", {'x': '1234'}) 594 with self.assertRaisesRegex(TypeError, 'must be real number, not str'): 595 eval("'%f' % (x,)", {'x': '1234'}) 596 with self.assertRaisesRegex(TypeError, 597 'not enough arguments for format string'): 598 eval("'%s, %s' % (x, *y)", {'x': 1, 'y': []}) 599 with self.assertRaisesRegex(TypeError, 600 'not all arguments converted during string formatting'): 601 eval("'%s, %s' % (x, *y)", {'x': 1, 'y': [2, 3]}) 602 603 def test_static_swaps_unpack_two(self): 604 def f(a, b): 605 a, b = a, b 606 b, a = a, b 607 self.assertNotInBytecode(f, "SWAP") 608 609 def test_static_swaps_unpack_three(self): 610 def f(a, b, c): 611 a, b, c = a, b, c 612 a, c, b = a, b, c 613 b, a, c = a, b, c 614 b, c, a = a, b, c 615 c, a, b = a, b, c 616 c, b, a = a, b, c 617 self.assertNotInBytecode(f, "SWAP") 618 619 def test_static_swaps_match_mapping(self): 620 for a, b, c in product("_a", "_b", "_c"): 621 pattern = f"{{'a': {a}, 'b': {b}, 'c': {c}}}" 622 with self.subTest(pattern): 623 code = compile_pattern_with_fast_locals(pattern) 624 self.assertNotInBytecode(code, "SWAP") 625 626 def test_static_swaps_match_class(self): 627 forms = [ 628 "C({}, {}, {})", 629 "C({}, {}, c={})", 630 "C({}, b={}, c={})", 631 "C(a={}, b={}, c={})" 632 ] 633 for a, b, c in product("_a", "_b", "_c"): 634 for form in forms: 635 pattern = form.format(a, b, c) 636 with self.subTest(pattern): 637 code = compile_pattern_with_fast_locals(pattern) 638 self.assertNotInBytecode(code, "SWAP") 639 640 def test_static_swaps_match_sequence(self): 641 swaps = {"*_, b, c", "a, *_, c", "a, b, *_"} 642 forms = ["{}, {}, {}", "{}, {}, *{}", "{}, *{}, {}", "*{}, {}, {}"] 643 for a, b, c in product("_a", "_b", "_c"): 644 for form in forms: 645 pattern = form.format(a, b, c) 646 with self.subTest(pattern): 647 code = compile_pattern_with_fast_locals(pattern) 648 if pattern in swaps: 649 # If this fails... great! Remove this pattern from swaps 650 # to prevent regressing on any improvement: 651 self.assertInBytecode(code, "SWAP") 652 else: 653 self.assertNotInBytecode(code, "SWAP") 654 655 656class TestBuglets(unittest.TestCase): 657 658 def test_bug_11510(self): 659 # folded constant set optimization was commingled with the tuple 660 # unpacking optimization which would fail if the set had duplicate 661 # elements so that the set length was unexpected 662 def f(): 663 x, y = {1, 1} 664 return x, y 665 with self.assertRaises(ValueError): 666 f() 667 668 def test_bpo_42057(self): 669 for i in range(10): 670 try: 671 raise Exception 672 except Exception or Exception: 673 pass 674 675 def test_bpo_45773_pop_jump_if_true(self): 676 compile("while True or spam: pass", "<test>", "exec") 677 678 def test_bpo_45773_pop_jump_if_false(self): 679 compile("while True or not spam: pass", "<test>", "exec") 680 681 682class TestMarkingVariablesAsUnKnown(BytecodeTestCase): 683 684 def setUp(self): 685 self.addCleanup(sys.settrace, sys.gettrace()) 686 sys.settrace(None) 687 688 def test_load_fast_known_simple(self): 689 def f(): 690 x = 1 691 y = x + x 692 self.assertInBytecode(f, 'LOAD_FAST_LOAD_FAST') 693 694 def test_load_fast_unknown_simple(self): 695 def f(): 696 if condition(): 697 x = 1 698 print(x) 699 self.assertInBytecode(f, 'LOAD_FAST_CHECK') 700 self.assertNotInBytecode(f, 'LOAD_FAST') 701 702 def test_load_fast_unknown_because_del(self): 703 def f(): 704 x = 1 705 del x 706 print(x) 707 self.assertInBytecode(f, 'LOAD_FAST_CHECK') 708 self.assertNotInBytecode(f, 'LOAD_FAST') 709 710 def test_load_fast_known_because_parameter(self): 711 def f1(x): 712 print(x) 713 self.assertInBytecode(f1, 'LOAD_FAST') 714 self.assertNotInBytecode(f1, 'LOAD_FAST_CHECK') 715 716 def f2(*, x): 717 print(x) 718 self.assertInBytecode(f2, 'LOAD_FAST') 719 self.assertNotInBytecode(f2, 'LOAD_FAST_CHECK') 720 721 def f3(*args): 722 print(args) 723 self.assertInBytecode(f3, 'LOAD_FAST') 724 self.assertNotInBytecode(f3, 'LOAD_FAST_CHECK') 725 726 def f4(**kwargs): 727 print(kwargs) 728 self.assertInBytecode(f4, 'LOAD_FAST') 729 self.assertNotInBytecode(f4, 'LOAD_FAST_CHECK') 730 731 def f5(x=0): 732 print(x) 733 self.assertInBytecode(f5, 'LOAD_FAST') 734 self.assertNotInBytecode(f5, 'LOAD_FAST_CHECK') 735 736 def test_load_fast_known_because_already_loaded(self): 737 def f(): 738 if condition(): 739 x = 1 740 print(x) 741 print(x) 742 self.assertInBytecode(f, 'LOAD_FAST_CHECK') 743 self.assertInBytecode(f, 'LOAD_FAST') 744 745 def test_load_fast_known_multiple_branches(self): 746 def f(): 747 if condition(): 748 x = 1 749 else: 750 x = 2 751 print(x) 752 self.assertInBytecode(f, 'LOAD_FAST') 753 self.assertNotInBytecode(f, 'LOAD_FAST_CHECK') 754 755 def test_load_fast_unknown_after_error(self): 756 def f(): 757 try: 758 res = 1 / 0 759 except ZeroDivisionError: 760 pass 761 return res 762 # LOAD_FAST (known) still occurs in the no-exception branch. 763 # Assert that it doesn't occur in the LOAD_FAST_CHECK branch. 764 self.assertInBytecode(f, 'LOAD_FAST_CHECK') 765 766 def test_load_fast_unknown_after_error_2(self): 767 def f(): 768 try: 769 1 / 0 770 except: 771 print(a, b, c, d, e, f, g) 772 a = b = c = d = e = f = g = 1 773 self.assertInBytecode(f, 'LOAD_FAST_CHECK') 774 self.assertNotInBytecode(f, 'LOAD_FAST') 775 776 def test_load_fast_too_many_locals(self): 777 # When there get to be too many locals to analyze completely, 778 # later locals are all converted to LOAD_FAST_CHECK, except 779 # when a store or prior load occurred in the same basicblock. 780 def f(): 781 a00 = a01 = a02 = a03 = a04 = a05 = a06 = a07 = a08 = a09 = 1 782 a10 = a11 = a12 = a13 = a14 = a15 = a16 = a17 = a18 = a19 = 1 783 a20 = a21 = a22 = a23 = a24 = a25 = a26 = a27 = a28 = a29 = 1 784 a30 = a31 = a32 = a33 = a34 = a35 = a36 = a37 = a38 = a39 = 1 785 a40 = a41 = a42 = a43 = a44 = a45 = a46 = a47 = a48 = a49 = 1 786 a50 = a51 = a52 = a53 = a54 = a55 = a56 = a57 = a58 = a59 = 1 787 a60 = a61 = a62 = a63 = a64 = a65 = a66 = a67 = a68 = a69 = 1 788 a70 = a71 = a72 = a73 = a74 = a75 = a76 = a77 = a78 = a79 = 1 789 del a72, a73 790 print(a73) 791 print(a70, a71, a72, a73) 792 while True: 793 print(a00, a01, a62, a63) 794 print(a64, a65, a78, a79) 795 796 self.assertInBytecode(f, 'LOAD_FAST_LOAD_FAST', ("a00", "a01")) 797 self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', "a00") 798 self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', "a01") 799 for i in 62, 63: 800 # First 64 locals: analyze completely 801 self.assertInBytecode(f, 'LOAD_FAST', f"a{i:02}") 802 self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', f"a{i:02}") 803 for i in 64, 65, 78, 79: 804 # Locals >=64 not in the same basicblock 805 self.assertInBytecode(f, 'LOAD_FAST_CHECK', f"a{i:02}") 806 self.assertNotInBytecode(f, 'LOAD_FAST', f"a{i:02}") 807 for i in 70, 71: 808 # Locals >=64 in the same basicblock 809 self.assertInBytecode(f, 'LOAD_FAST', f"a{i:02}") 810 self.assertNotInBytecode(f, 'LOAD_FAST_CHECK', f"a{i:02}") 811 # del statements should invalidate within basicblocks. 812 self.assertInBytecode(f, 'LOAD_FAST_CHECK', "a72") 813 self.assertNotInBytecode(f, 'LOAD_FAST', "a72") 814 # previous checked loads within a basicblock enable unchecked loads 815 self.assertInBytecode(f, 'LOAD_FAST_CHECK', "a73") 816 self.assertInBytecode(f, 'LOAD_FAST', "a73") 817 818 def test_setting_lineno_no_undefined(self): 819 code = textwrap.dedent("""\ 820 def f(): 821 x = y = 2 822 if not x: 823 return 4 824 for i in range(55): 825 x + 6 826 L = 7 827 L = 8 828 L = 9 829 L = 10 830 """) 831 ns = {} 832 exec(code, ns) 833 f = ns['f'] 834 self.assertInBytecode(f, "LOAD_FAST") 835 self.assertNotInBytecode(f, "LOAD_FAST_CHECK") 836 co_code = f.__code__.co_code 837 def trace(frame, event, arg): 838 if event == 'line' and frame.f_lineno == 9: 839 frame.f_lineno = 3 840 sys.settrace(None) 841 return None 842 return trace 843 sys.settrace(trace) 844 result = f() 845 self.assertIsNone(result) 846 self.assertInBytecode(f, "LOAD_FAST") 847 self.assertNotInBytecode(f, "LOAD_FAST_CHECK") 848 self.assertEqual(f.__code__.co_code, co_code) 849 850 def test_setting_lineno_one_undefined(self): 851 code = textwrap.dedent("""\ 852 def f(): 853 x = y = 2 854 if not x: 855 return 4 856 for i in range(55): 857 x + 6 858 del x 859 L = 8 860 L = 9 861 L = 10 862 """) 863 ns = {} 864 exec(code, ns) 865 f = ns['f'] 866 self.assertInBytecode(f, "LOAD_FAST") 867 self.assertNotInBytecode(f, "LOAD_FAST_CHECK") 868 co_code = f.__code__.co_code 869 def trace(frame, event, arg): 870 if event == 'line' and frame.f_lineno == 9: 871 frame.f_lineno = 3 872 sys.settrace(None) 873 return None 874 return trace 875 e = r"assigning None to 1 unbound local" 876 with self.assertWarnsRegex(RuntimeWarning, e): 877 sys.settrace(trace) 878 result = f() 879 self.assertEqual(result, 4) 880 self.assertInBytecode(f, "LOAD_FAST") 881 self.assertNotInBytecode(f, "LOAD_FAST_CHECK") 882 self.assertEqual(f.__code__.co_code, co_code) 883 884 def test_setting_lineno_two_undefined(self): 885 code = textwrap.dedent("""\ 886 def f(): 887 x = y = 2 888 if not x: 889 return 4 890 for i in range(55): 891 x + 6 892 del x, y 893 L = 8 894 L = 9 895 L = 10 896 """) 897 ns = {} 898 exec(code, ns) 899 f = ns['f'] 900 self.assertInBytecode(f, "LOAD_FAST") 901 self.assertNotInBytecode(f, "LOAD_FAST_CHECK") 902 co_code = f.__code__.co_code 903 def trace(frame, event, arg): 904 if event == 'line' and frame.f_lineno == 9: 905 frame.f_lineno = 3 906 sys.settrace(None) 907 return None 908 return trace 909 e = r"assigning None to 2 unbound locals" 910 with self.assertWarnsRegex(RuntimeWarning, e): 911 sys.settrace(trace) 912 result = f() 913 self.assertEqual(result, 4) 914 self.assertInBytecode(f, "LOAD_FAST") 915 self.assertNotInBytecode(f, "LOAD_FAST_CHECK") 916 self.assertEqual(f.__code__.co_code, co_code) 917 918 def make_function_with_no_checks(self): 919 code = textwrap.dedent("""\ 920 def f(): 921 x = 2 922 L = 3 923 L = 4 924 L = 5 925 if not L: 926 x + 7 927 y = 2 928 """) 929 ns = {} 930 exec(code, ns) 931 f = ns['f'] 932 self.assertInBytecode(f, "LOAD_FAST") 933 self.assertNotInBytecode(f, "LOAD_FAST_CHECK") 934 return f 935 936 def test_modifying_local_does_not_add_check(self): 937 f = self.make_function_with_no_checks() 938 def trace(frame, event, arg): 939 if event == 'line' and frame.f_lineno == 4: 940 frame.f_locals["x"] = 42 941 sys.settrace(None) 942 return None 943 return trace 944 sys.settrace(trace) 945 f() 946 self.assertInBytecode(f, "LOAD_FAST") 947 self.assertNotInBytecode(f, "LOAD_FAST_CHECK") 948 949 def test_initializing_local_does_not_add_check(self): 950 f = self.make_function_with_no_checks() 951 def trace(frame, event, arg): 952 if event == 'line' and frame.f_lineno == 4: 953 frame.f_locals["y"] = 42 954 sys.settrace(None) 955 return None 956 return trace 957 sys.settrace(trace) 958 f() 959 self.assertInBytecode(f, "LOAD_FAST") 960 self.assertNotInBytecode(f, "LOAD_FAST_CHECK") 961 962 963class DirectCfgOptimizerTests(CfgOptimizationTestCase): 964 965 def cfg_optimization_test(self, insts, expected_insts, 966 consts=None, expected_consts=None, 967 nlocals=0): 968 969 self.check_instructions(insts) 970 self.check_instructions(expected_insts) 971 972 if expected_consts is None: 973 expected_consts = consts 974 seq = self.seq_from_insts(insts) 975 opt_insts, opt_consts = self.get_optimized(seq, consts, nlocals) 976 expected_insts = self.seq_from_insts(expected_insts).get_instructions() 977 self.assertInstructionsMatch(opt_insts, expected_insts) 978 self.assertEqual(opt_consts, expected_consts) 979 980 def test_conditional_jump_forward_non_const_condition(self): 981 insts = [ 982 ('LOAD_NAME', 1, 11), 983 ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), 984 ('LOAD_CONST', 2, 13), 985 ('RETURN_VALUE', None, 13), 986 lbl, 987 ('LOAD_CONST', 3, 14), 988 ('RETURN_VALUE', None, 14), 989 ] 990 expected_insts = [ 991 ('LOAD_NAME', 1, 11), 992 ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), 993 ('RETURN_CONST', 1, 13), 994 lbl, 995 ('RETURN_CONST', 2, 14), 996 ] 997 self.cfg_optimization_test(insts, 998 expected_insts, 999 consts=[0, 1, 2, 3, 4], 1000 expected_consts=[0, 2, 3]) 1001 1002 def test_conditional_jump_forward_const_condition(self): 1003 # The unreachable branch of the jump is removed, the jump 1004 # becomes redundant and is replaced by a NOP (for the lineno) 1005 1006 insts = [ 1007 ('LOAD_CONST', 3, 11), 1008 ('POP_JUMP_IF_TRUE', lbl := self.Label(), 12), 1009 ('LOAD_CONST', 2, 13), 1010 lbl, 1011 ('LOAD_CONST', 3, 14), 1012 ('RETURN_VALUE', None, 14), 1013 ] 1014 expected_insts = [ 1015 ('NOP', None, 11), 1016 ('NOP', None, 12), 1017 ('RETURN_CONST', 1, 14), 1018 ] 1019 self.cfg_optimization_test(insts, 1020 expected_insts, 1021 consts=[0, 1, 2, 3, 4], 1022 expected_consts=[0, 3]) 1023 1024 def test_conditional_jump_backward_non_const_condition(self): 1025 insts = [ 1026 lbl1 := self.Label(), 1027 ('LOAD_NAME', 1, 11), 1028 ('POP_JUMP_IF_TRUE', lbl1, 12), 1029 ('LOAD_NAME', 2, 13), 1030 ('RETURN_VALUE', None, 13), 1031 ] 1032 expected = [ 1033 lbl := self.Label(), 1034 ('LOAD_NAME', 1, 11), 1035 ('POP_JUMP_IF_TRUE', lbl, 12), 1036 ('LOAD_NAME', 2, 13), 1037 ('RETURN_VALUE', None, 13), 1038 ] 1039 self.cfg_optimization_test(insts, expected, consts=list(range(5))) 1040 1041 def test_conditional_jump_backward_const_condition(self): 1042 # The unreachable branch of the jump is removed 1043 insts = [ 1044 lbl1 := self.Label(), 1045 ('LOAD_CONST', 3, 11), 1046 ('POP_JUMP_IF_TRUE', lbl1, 12), 1047 ('LOAD_CONST', 2, 13), 1048 ('RETURN_VALUE', None, 13), 1049 ] 1050 expected_insts = [ 1051 lbl := self.Label(), 1052 ('NOP', None, 11), 1053 ('JUMP', lbl, 12), 1054 ] 1055 self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) 1056 1057 def test_except_handler_label(self): 1058 insts = [ 1059 ('SETUP_FINALLY', handler := self.Label(), 10), 1060 ('POP_BLOCK', None, -1), 1061 ('RETURN_CONST', 1, 11), 1062 handler, 1063 ('RETURN_CONST', 2, 12), 1064 ] 1065 expected_insts = [ 1066 ('SETUP_FINALLY', handler := self.Label(), 10), 1067 ('RETURN_CONST', 1, 11), 1068 handler, 1069 ('RETURN_CONST', 2, 12), 1070 ] 1071 self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) 1072 1073 def test_no_unsafe_static_swap(self): 1074 # We can't change order of two stores to the same location 1075 insts = [ 1076 ('LOAD_CONST', 0, 1), 1077 ('LOAD_CONST', 1, 2), 1078 ('LOAD_CONST', 2, 3), 1079 ('SWAP', 3, 4), 1080 ('STORE_FAST', 1, 4), 1081 ('STORE_FAST', 1, 4), 1082 ('POP_TOP', None, 4), 1083 ('LOAD_CONST', 0, 5), 1084 ('RETURN_VALUE', None, 5) 1085 ] 1086 expected_insts = [ 1087 ('LOAD_CONST', 0, 1), 1088 ('LOAD_CONST', 1, 2), 1089 ('NOP', None, 3), 1090 ('STORE_FAST', 1, 4), 1091 ('POP_TOP', None, 4), 1092 ('RETURN_CONST', 0) 1093 ] 1094 self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) 1095 1096 def test_dead_store_elimination_in_same_lineno(self): 1097 insts = [ 1098 ('LOAD_CONST', 0, 1), 1099 ('LOAD_CONST', 1, 2), 1100 ('LOAD_CONST', 2, 3), 1101 ('STORE_FAST', 1, 4), 1102 ('STORE_FAST', 1, 4), 1103 ('STORE_FAST', 1, 4), 1104 ('LOAD_CONST', 0, 5), 1105 ('RETURN_VALUE', None, 5) 1106 ] 1107 expected_insts = [ 1108 ('LOAD_CONST', 0, 1), 1109 ('LOAD_CONST', 1, 2), 1110 ('NOP', None, 3), 1111 ('POP_TOP', None, 4), 1112 ('STORE_FAST', 1, 4), 1113 ('RETURN_CONST', 0, 5) 1114 ] 1115 self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) 1116 1117 def test_no_dead_store_elimination_in_different_lineno(self): 1118 insts = [ 1119 ('LOAD_CONST', 0, 1), 1120 ('LOAD_CONST', 1, 2), 1121 ('LOAD_CONST', 2, 3), 1122 ('STORE_FAST', 1, 4), 1123 ('STORE_FAST', 1, 5), 1124 ('STORE_FAST', 1, 6), 1125 ('LOAD_CONST', 0, 5), 1126 ('RETURN_VALUE', None, 5) 1127 ] 1128 expected_insts = [ 1129 ('LOAD_CONST', 0, 1), 1130 ('LOAD_CONST', 1, 2), 1131 ('LOAD_CONST', 2, 3), 1132 ('STORE_FAST', 1, 4), 1133 ('STORE_FAST', 1, 5), 1134 ('STORE_FAST', 1, 6), 1135 ('RETURN_CONST', 0, 5) 1136 ] 1137 self.cfg_optimization_test(insts, expected_insts, consts=list(range(3)), nlocals=1) 1138 1139 def test_unconditional_jump_threading(self): 1140 1141 def get_insts(lno1, lno2, op1, op2): 1142 return [ 1143 lbl2 := self.Label(), 1144 ('LOAD_NAME', 0, 10), 1145 (op1, lbl1 := self.Label(), lno1), 1146 ('LOAD_NAME', 1, 20), 1147 lbl1, 1148 (op2, lbl2, lno2), 1149 ] 1150 1151 1152 for op1 in ('JUMP', 'JUMP_NO_INTERRUPT'): 1153 for op2 in ('JUMP', 'JUMP_NO_INTERRUPT'): 1154 # different lines 1155 lno1, lno2 = (4, 5) 1156 with self.subTest(lno = (lno1, lno2), ops = (op1, op2)): 1157 insts = get_insts(lno1, lno2, op1, op2) 1158 op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' 1159 expected_insts = [ 1160 ('LOAD_NAME', 0, 10), 1161 ('NOP', None, 4), 1162 (op, 0, 5), 1163 ] 1164 self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) 1165 1166 # Threading 1167 for lno1, lno2 in [(-1, -1), (-1, 5), (6, -1), (7, 7)]: 1168 with self.subTest(lno = (lno1, lno2), ops = (op1, op2)): 1169 insts = get_insts(lno1, lno2, op1, op2) 1170 lno = lno1 if lno1 != -1 else lno2 1171 if lno == -1: 1172 lno = 10 # Propagated from the line before 1173 1174 op = 'JUMP' if 'JUMP' in (op1, op2) else 'JUMP_NO_INTERRUPT' 1175 expected_insts = [ 1176 ('LOAD_NAME', 0, 10), 1177 (op, 0, lno), 1178 ] 1179 self.cfg_optimization_test(insts, expected_insts, consts=list(range(5))) 1180 1181if __name__ == "__main__": 1182 unittest.main() 1183