1import dis 2import math 3import os 4import unittest 5import sys 6import _ast 7import tempfile 8import types 9from test import support 10from test.support import script_helper 11from test.support.os_helper import FakePath 12 13 14class TestSpecifics(unittest.TestCase): 15 16 def compile_single(self, source): 17 compile(source, "<single>", "single") 18 19 def assertInvalidSingle(self, source): 20 self.assertRaises(SyntaxError, self.compile_single, source) 21 22 def test_no_ending_newline(self): 23 compile("hi", "<test>", "exec") 24 compile("hi\r", "<test>", "exec") 25 26 def test_empty(self): 27 compile("", "<test>", "exec") 28 29 def test_other_newlines(self): 30 compile("\r\n", "<test>", "exec") 31 compile("\r", "<test>", "exec") 32 compile("hi\r\nstuff\r\ndef f():\n pass\r", "<test>", "exec") 33 compile("this_is\rreally_old_mac\rdef f():\n pass", "<test>", "exec") 34 35 def test_debug_assignment(self): 36 # catch assignments to __debug__ 37 self.assertRaises(SyntaxError, compile, '__debug__ = 1', '?', 'single') 38 import builtins 39 prev = builtins.__debug__ 40 setattr(builtins, '__debug__', 'sure') 41 self.assertEqual(__debug__, prev) 42 setattr(builtins, '__debug__', prev) 43 44 def test_argument_handling(self): 45 # detect duplicate positional and keyword arguments 46 self.assertRaises(SyntaxError, eval, 'lambda a,a:0') 47 self.assertRaises(SyntaxError, eval, 'lambda a,a=1:0') 48 self.assertRaises(SyntaxError, eval, 'lambda a=1,a=1:0') 49 self.assertRaises(SyntaxError, exec, 'def f(a, a): pass') 50 self.assertRaises(SyntaxError, exec, 'def f(a = 0, a = 1): pass') 51 self.assertRaises(SyntaxError, exec, 'def f(a): global a; a = 1') 52 53 def test_syntax_error(self): 54 self.assertRaises(SyntaxError, compile, "1+*3", "filename", "exec") 55 56 def test_none_keyword_arg(self): 57 self.assertRaises(SyntaxError, compile, "f(None=1)", "<string>", "exec") 58 59 def test_duplicate_global_local(self): 60 self.assertRaises(SyntaxError, exec, 'def f(a): global a; a = 1') 61 62 def test_exec_with_general_mapping_for_locals(self): 63 64 class M: 65 "Test mapping interface versus possible calls from eval()." 66 def __getitem__(self, key): 67 if key == 'a': 68 return 12 69 raise KeyError 70 def __setitem__(self, key, value): 71 self.results = (key, value) 72 def keys(self): 73 return list('xyz') 74 75 m = M() 76 g = globals() 77 exec('z = a', g, m) 78 self.assertEqual(m.results, ('z', 12)) 79 try: 80 exec('z = b', g, m) 81 except NameError: 82 pass 83 else: 84 self.fail('Did not detect a KeyError') 85 exec('z = dir()', g, m) 86 self.assertEqual(m.results, ('z', list('xyz'))) 87 exec('z = globals()', g, m) 88 self.assertEqual(m.results, ('z', g)) 89 exec('z = locals()', g, m) 90 self.assertEqual(m.results, ('z', m)) 91 self.assertRaises(TypeError, exec, 'z = b', m) 92 93 class A: 94 "Non-mapping" 95 pass 96 m = A() 97 self.assertRaises(TypeError, exec, 'z = a', g, m) 98 99 # Verify that dict subclasses work as well 100 class D(dict): 101 def __getitem__(self, key): 102 if key == 'a': 103 return 12 104 return dict.__getitem__(self, key) 105 d = D() 106 exec('z = a', g, d) 107 self.assertEqual(d['z'], 12) 108 109 def test_extended_arg(self): 110 longexpr = 'x = x or ' + '-x' * 2500 111 g = {} 112 code = ''' 113def f(x): 114 %s 115 %s 116 %s 117 %s 118 %s 119 %s 120 %s 121 %s 122 %s 123 %s 124 # the expressions above have no effect, x == argument 125 while x: 126 x -= 1 127 # EXTENDED_ARG/JUMP_ABSOLUTE here 128 return x 129''' % ((longexpr,)*10) 130 exec(code, g) 131 self.assertEqual(g['f'](5), 0) 132 133 def test_argument_order(self): 134 self.assertRaises(SyntaxError, exec, 'def f(a=1, b): pass') 135 136 def test_float_literals(self): 137 # testing bad float literals 138 self.assertRaises(SyntaxError, eval, "2e") 139 self.assertRaises(SyntaxError, eval, "2.0e+") 140 self.assertRaises(SyntaxError, eval, "1e-") 141 self.assertRaises(SyntaxError, eval, "3-4e/21") 142 143 def test_indentation(self): 144 # testing compile() of indented block w/o trailing newline" 145 s = """ 146if 1: 147 if 2: 148 pass""" 149 compile(s, "<string>", "exec") 150 151 # This test is probably specific to CPython and may not generalize 152 # to other implementations. We are trying to ensure that when 153 # the first line of code starts after 256, correct line numbers 154 # in tracebacks are still produced. 155 def test_leading_newlines(self): 156 s256 = "".join(["\n"] * 256 + ["spam"]) 157 co = compile(s256, 'fn', 'exec') 158 self.assertEqual(co.co_firstlineno, 1) 159 self.assertEqual(list(co.co_lines()), [(0, 8, 257)]) 160 161 def test_literals_with_leading_zeroes(self): 162 for arg in ["077787", "0xj", "0x.", "0e", "090000000000000", 163 "080000000000000", "000000000000009", "000000000000008", 164 "0b42", "0BADCAFE", "0o123456789", "0b1.1", "0o4.2", 165 "0b101j", "0o153j", "0b100e1", "0o777e1", "0777", 166 "000777", "000000000000007"]: 167 self.assertRaises(SyntaxError, eval, arg) 168 169 self.assertEqual(eval("0xff"), 255) 170 self.assertEqual(eval("0777."), 777) 171 self.assertEqual(eval("0777.0"), 777) 172 self.assertEqual(eval("000000000000000000000000000000000000000000000000000777e0"), 777) 173 self.assertEqual(eval("0777e1"), 7770) 174 self.assertEqual(eval("0e0"), 0) 175 self.assertEqual(eval("0000e-012"), 0) 176 self.assertEqual(eval("09.5"), 9.5) 177 self.assertEqual(eval("0777j"), 777j) 178 self.assertEqual(eval("000"), 0) 179 self.assertEqual(eval("00j"), 0j) 180 self.assertEqual(eval("00.0"), 0) 181 self.assertEqual(eval("0e3"), 0) 182 self.assertEqual(eval("090000000000000."), 90000000000000.) 183 self.assertEqual(eval("090000000000000.0000000000000000000000"), 90000000000000.) 184 self.assertEqual(eval("090000000000000e0"), 90000000000000.) 185 self.assertEqual(eval("090000000000000e-0"), 90000000000000.) 186 self.assertEqual(eval("090000000000000j"), 90000000000000j) 187 self.assertEqual(eval("000000000000008."), 8.) 188 self.assertEqual(eval("000000000000009."), 9.) 189 self.assertEqual(eval("0b101010"), 42) 190 self.assertEqual(eval("-0b000000000010"), -2) 191 self.assertEqual(eval("0o777"), 511) 192 self.assertEqual(eval("-0o0000010"), -8) 193 194 def test_unary_minus(self): 195 # Verify treatment of unary minus on negative numbers SF bug #660455 196 if sys.maxsize == 2147483647: 197 # 32-bit machine 198 all_one_bits = '0xffffffff' 199 self.assertEqual(eval(all_one_bits), 4294967295) 200 self.assertEqual(eval("-" + all_one_bits), -4294967295) 201 elif sys.maxsize == 9223372036854775807: 202 # 64-bit machine 203 all_one_bits = '0xffffffffffffffff' 204 self.assertEqual(eval(all_one_bits), 18446744073709551615) 205 self.assertEqual(eval("-" + all_one_bits), -18446744073709551615) 206 else: 207 self.fail("How many bits *does* this machine have???") 208 # Verify treatment of constant folding on -(sys.maxsize+1) 209 # i.e. -2147483648 on 32 bit platforms. Should return int. 210 self.assertIsInstance(eval("%s" % (-sys.maxsize - 1)), int) 211 self.assertIsInstance(eval("%s" % (-sys.maxsize - 2)), int) 212 213 if sys.maxsize == 9223372036854775807: 214 def test_32_63_bit_values(self): 215 a = +4294967296 # 1 << 32 216 b = -4294967296 # 1 << 32 217 c = +281474976710656 # 1 << 48 218 d = -281474976710656 # 1 << 48 219 e = +4611686018427387904 # 1 << 62 220 f = -4611686018427387904 # 1 << 62 221 g = +9223372036854775807 # 1 << 63 - 1 222 h = -9223372036854775807 # 1 << 63 - 1 223 224 for variable in self.test_32_63_bit_values.__code__.co_consts: 225 if variable is not None: 226 self.assertIsInstance(variable, int) 227 228 def test_sequence_unpacking_error(self): 229 # Verify sequence packing/unpacking with "or". SF bug #757818 230 i,j = (1, -1) or (-1, 1) 231 self.assertEqual(i, 1) 232 self.assertEqual(j, -1) 233 234 def test_none_assignment(self): 235 stmts = [ 236 'None = 0', 237 'None += 0', 238 '__builtins__.None = 0', 239 'def None(): pass', 240 'class None: pass', 241 '(a, None) = 0, 0', 242 'for None in range(10): pass', 243 'def f(None): pass', 244 'import None', 245 'import x as None', 246 'from x import None', 247 'from x import y as None' 248 ] 249 for stmt in stmts: 250 stmt += "\n" 251 self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') 252 self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') 253 254 def test_import(self): 255 succeed = [ 256 'import sys', 257 'import os, sys', 258 'import os as bar', 259 'import os.path as bar', 260 'from __future__ import nested_scopes, generators', 261 'from __future__ import (nested_scopes,\ngenerators)', 262 'from __future__ import (nested_scopes,\ngenerators,)', 263 'from sys import stdin, stderr, stdout', 264 'from sys import (stdin, stderr,\nstdout)', 265 'from sys import (stdin, stderr,\nstdout,)', 266 'from sys import (stdin\n, stderr, stdout)', 267 'from sys import (stdin\n, stderr, stdout,)', 268 'from sys import stdin as si, stdout as so, stderr as se', 269 'from sys import (stdin as si, stdout as so, stderr as se)', 270 'from sys import (stdin as si, stdout as so, stderr as se,)', 271 ] 272 fail = [ 273 'import (os, sys)', 274 'import (os), (sys)', 275 'import ((os), (sys))', 276 'import (sys', 277 'import sys)', 278 'import (os,)', 279 'import os As bar', 280 'import os.path a bar', 281 'from sys import stdin As stdout', 282 'from sys import stdin a stdout', 283 'from (sys) import stdin', 284 'from __future__ import (nested_scopes', 285 'from __future__ import nested_scopes)', 286 'from __future__ import nested_scopes,\ngenerators', 287 'from sys import (stdin', 288 'from sys import stdin)', 289 'from sys import stdin, stdout,\nstderr', 290 'from sys import stdin si', 291 'from sys import stdin,', 292 'from sys import (*)', 293 'from sys import (stdin,, stdout, stderr)', 294 'from sys import (stdin, stdout),', 295 ] 296 for stmt in succeed: 297 compile(stmt, 'tmp', 'exec') 298 for stmt in fail: 299 self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') 300 301 def test_for_distinct_code_objects(self): 302 # SF bug 1048870 303 def f(): 304 f1 = lambda x=1: x 305 f2 = lambda x=2: x 306 return f1, f2 307 f1, f2 = f() 308 self.assertNotEqual(id(f1.__code__), id(f2.__code__)) 309 310 def test_lambda_doc(self): 311 l = lambda: "foo" 312 self.assertIsNone(l.__doc__) 313 314 def test_encoding(self): 315 code = b'# -*- coding: badencoding -*-\npass\n' 316 self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec') 317 code = '# -*- coding: badencoding -*-\n"\xc2\xa4"\n' 318 compile(code, 'tmp', 'exec') 319 self.assertEqual(eval(code), '\xc2\xa4') 320 code = '"\xc2\xa4"\n' 321 self.assertEqual(eval(code), '\xc2\xa4') 322 code = b'"\xc2\xa4"\n' 323 self.assertEqual(eval(code), '\xa4') 324 code = b'# -*- coding: latin1 -*-\n"\xc2\xa4"\n' 325 self.assertEqual(eval(code), '\xc2\xa4') 326 code = b'# -*- coding: utf-8 -*-\n"\xc2\xa4"\n' 327 self.assertEqual(eval(code), '\xa4') 328 code = b'# -*- coding: iso8859-15 -*-\n"\xc2\xa4"\n' 329 self.assertEqual(eval(code), '\xc2\u20ac') 330 code = '"""\\\n# -*- coding: iso8859-15 -*-\n\xc2\xa4"""\n' 331 self.assertEqual(eval(code), '# -*- coding: iso8859-15 -*-\n\xc2\xa4') 332 code = b'"""\\\n# -*- coding: iso8859-15 -*-\n\xc2\xa4"""\n' 333 self.assertEqual(eval(code), '# -*- coding: iso8859-15 -*-\n\xa4') 334 335 def test_subscripts(self): 336 # SF bug 1448804 337 # Class to make testing subscript results easy 338 class str_map(object): 339 def __init__(self): 340 self.data = {} 341 def __getitem__(self, key): 342 return self.data[str(key)] 343 def __setitem__(self, key, value): 344 self.data[str(key)] = value 345 def __delitem__(self, key): 346 del self.data[str(key)] 347 def __contains__(self, key): 348 return str(key) in self.data 349 d = str_map() 350 # Index 351 d[1] = 1 352 self.assertEqual(d[1], 1) 353 d[1] += 1 354 self.assertEqual(d[1], 2) 355 del d[1] 356 self.assertNotIn(1, d) 357 # Tuple of indices 358 d[1, 1] = 1 359 self.assertEqual(d[1, 1], 1) 360 d[1, 1] += 1 361 self.assertEqual(d[1, 1], 2) 362 del d[1, 1] 363 self.assertNotIn((1, 1), d) 364 # Simple slice 365 d[1:2] = 1 366 self.assertEqual(d[1:2], 1) 367 d[1:2] += 1 368 self.assertEqual(d[1:2], 2) 369 del d[1:2] 370 self.assertNotIn(slice(1, 2), d) 371 # Tuple of simple slices 372 d[1:2, 1:2] = 1 373 self.assertEqual(d[1:2, 1:2], 1) 374 d[1:2, 1:2] += 1 375 self.assertEqual(d[1:2, 1:2], 2) 376 del d[1:2, 1:2] 377 self.assertNotIn((slice(1, 2), slice(1, 2)), d) 378 # Extended slice 379 d[1:2:3] = 1 380 self.assertEqual(d[1:2:3], 1) 381 d[1:2:3] += 1 382 self.assertEqual(d[1:2:3], 2) 383 del d[1:2:3] 384 self.assertNotIn(slice(1, 2, 3), d) 385 # Tuple of extended slices 386 d[1:2:3, 1:2:3] = 1 387 self.assertEqual(d[1:2:3, 1:2:3], 1) 388 d[1:2:3, 1:2:3] += 1 389 self.assertEqual(d[1:2:3, 1:2:3], 2) 390 del d[1:2:3, 1:2:3] 391 self.assertNotIn((slice(1, 2, 3), slice(1, 2, 3)), d) 392 # Ellipsis 393 d[...] = 1 394 self.assertEqual(d[...], 1) 395 d[...] += 1 396 self.assertEqual(d[...], 2) 397 del d[...] 398 self.assertNotIn(Ellipsis, d) 399 # Tuple of Ellipses 400 d[..., ...] = 1 401 self.assertEqual(d[..., ...], 1) 402 d[..., ...] += 1 403 self.assertEqual(d[..., ...], 2) 404 del d[..., ...] 405 self.assertNotIn((Ellipsis, Ellipsis), d) 406 407 def test_annotation_limit(self): 408 # more than 255 annotations, should compile ok 409 s = "def f(%s): pass" 410 s %= ', '.join('a%d:%d' % (i,i) for i in range(300)) 411 compile(s, '?', 'exec') 412 413 def test_mangling(self): 414 class A: 415 def f(): 416 __mangled = 1 417 __not_mangled__ = 2 418 import __mangled_mod 419 import __package__.module 420 421 self.assertIn("_A__mangled", A.f.__code__.co_varnames) 422 self.assertIn("__not_mangled__", A.f.__code__.co_varnames) 423 self.assertIn("_A__mangled_mod", A.f.__code__.co_varnames) 424 self.assertIn("__package__", A.f.__code__.co_varnames) 425 426 def test_compile_ast(self): 427 fname = __file__ 428 if fname.lower().endswith('pyc'): 429 fname = fname[:-1] 430 with open(fname, encoding='utf-8') as f: 431 fcontents = f.read() 432 sample_code = [ 433 ['<assign>', 'x = 5'], 434 ['<ifblock>', """if True:\n pass\n"""], 435 ['<forblock>', """for n in [1, 2, 3]:\n print(n)\n"""], 436 ['<deffunc>', """def foo():\n pass\nfoo()\n"""], 437 [fname, fcontents], 438 ] 439 440 for fname, code in sample_code: 441 co1 = compile(code, '%s1' % fname, 'exec') 442 ast = compile(code, '%s2' % fname, 'exec', _ast.PyCF_ONLY_AST) 443 self.assertTrue(type(ast) == _ast.Module) 444 co2 = compile(ast, '%s3' % fname, 'exec') 445 self.assertEqual(co1, co2) 446 # the code object's filename comes from the second compilation step 447 self.assertEqual(co2.co_filename, '%s3' % fname) 448 449 # raise exception when node type doesn't match with compile mode 450 co1 = compile('print(1)', '<string>', 'exec', _ast.PyCF_ONLY_AST) 451 self.assertRaises(TypeError, compile, co1, '<ast>', 'eval') 452 453 # raise exception when node type is no start node 454 self.assertRaises(TypeError, compile, _ast.If(), '<ast>', 'exec') 455 456 # raise exception when node has invalid children 457 ast = _ast.Module() 458 ast.body = [_ast.BoolOp()] 459 self.assertRaises(TypeError, compile, ast, '<ast>', 'exec') 460 461 def test_dict_evaluation_order(self): 462 i = 0 463 464 def f(): 465 nonlocal i 466 i += 1 467 return i 468 469 d = {f(): f(), f(): f()} 470 self.assertEqual(d, {1: 2, 3: 4}) 471 472 def test_compile_filename(self): 473 for filename in 'file.py', b'file.py': 474 code = compile('pass', filename, 'exec') 475 self.assertEqual(code.co_filename, 'file.py') 476 for filename in bytearray(b'file.py'), memoryview(b'file.py'): 477 with self.assertWarns(DeprecationWarning): 478 code = compile('pass', filename, 'exec') 479 self.assertEqual(code.co_filename, 'file.py') 480 self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec') 481 482 @support.cpython_only 483 def test_same_filename_used(self): 484 s = """def f(): pass\ndef g(): pass""" 485 c = compile(s, "myfile", "exec") 486 for obj in c.co_consts: 487 if isinstance(obj, types.CodeType): 488 self.assertIs(obj.co_filename, c.co_filename) 489 490 def test_single_statement(self): 491 self.compile_single("1 + 2") 492 self.compile_single("\n1 + 2") 493 self.compile_single("1 + 2\n") 494 self.compile_single("1 + 2\n\n") 495 self.compile_single("1 + 2\t\t\n") 496 self.compile_single("1 + 2\t\t\n ") 497 self.compile_single("1 + 2 # one plus two") 498 self.compile_single("1; 2") 499 self.compile_single("import sys; sys") 500 self.compile_single("def f():\n pass") 501 self.compile_single("while False:\n pass") 502 self.compile_single("if x:\n f(x)") 503 self.compile_single("if x:\n f(x)\nelse:\n g(x)") 504 self.compile_single("class T:\n pass") 505 self.compile_single("c = '''\na=1\nb=2\nc=3\n'''") 506 507 def test_bad_single_statement(self): 508 self.assertInvalidSingle('1\n2') 509 self.assertInvalidSingle('def f(): pass') 510 self.assertInvalidSingle('a = 13\nb = 187') 511 self.assertInvalidSingle('del x\ndel y') 512 self.assertInvalidSingle('f()\ng()') 513 self.assertInvalidSingle('f()\n# blah\nblah()') 514 self.assertInvalidSingle('f()\nxy # blah\nblah()') 515 self.assertInvalidSingle('x = 5 # comment\nx = 6\n') 516 self.assertInvalidSingle("c = '''\nd=1\n'''\na = 1\n\nb = 2\n") 517 518 def test_particularly_evil_undecodable(self): 519 # Issue 24022 520 src = b'0000\x00\n00000000000\n\x00\n\x9e\n' 521 with tempfile.TemporaryDirectory() as tmpd: 522 fn = os.path.join(tmpd, "bad.py") 523 with open(fn, "wb") as fp: 524 fp.write(src) 525 res = script_helper.run_python_until_end(fn)[0] 526 self.assertIn(b"Non-UTF-8", res.err) 527 528 def test_yet_more_evil_still_undecodable(self): 529 # Issue #25388 530 src = b"#\x00\n#\xfd\n" 531 with tempfile.TemporaryDirectory() as tmpd: 532 fn = os.path.join(tmpd, "bad.py") 533 with open(fn, "wb") as fp: 534 fp.write(src) 535 res = script_helper.run_python_until_end(fn)[0] 536 self.assertIn(b"Non-UTF-8", res.err) 537 538 @support.cpython_only 539 def test_compiler_recursion_limit(self): 540 # Expected limit is sys.getrecursionlimit() * the scaling factor 541 # in symtable.c (currently 3) 542 # We expect to fail *at* that limit, because we use up some of 543 # the stack depth limit in the test suite code 544 # So we check the expected limit and 75% of that 545 # XXX (ncoghlan): duplicating the scaling factor here is a little 546 # ugly. Perhaps it should be exposed somewhere... 547 fail_depth = sys.getrecursionlimit() * 3 548 crash_depth = sys.getrecursionlimit() * 300 549 success_depth = int(fail_depth * 0.75) 550 551 def check_limit(prefix, repeated, mode="single"): 552 expect_ok = prefix + repeated * success_depth 553 compile(expect_ok, '<test>', mode) 554 for depth in (fail_depth, crash_depth): 555 broken = prefix + repeated * depth 556 details = "Compiling ({!r} + {!r} * {})".format( 557 prefix, repeated, depth) 558 with self.assertRaises(RecursionError, msg=details): 559 compile(broken, '<test>', mode) 560 561 check_limit("a", "()") 562 check_limit("a", ".b") 563 check_limit("a", "[0]") 564 check_limit("a", "*a") 565 # XXX Crashes in the parser. 566 # check_limit("a", " if a else a") 567 # check_limit("if a: pass", "\nelif a: pass", mode="exec") 568 569 def test_null_terminated(self): 570 # The source code is null-terminated internally, but bytes-like 571 # objects are accepted, which could be not terminated. 572 with self.assertRaisesRegex(ValueError, "cannot contain null"): 573 compile("123\x00", "<dummy>", "eval") 574 with self.assertRaisesRegex(ValueError, "cannot contain null"): 575 compile(memoryview(b"123\x00"), "<dummy>", "eval") 576 code = compile(memoryview(b"123\x00")[1:-1], "<dummy>", "eval") 577 self.assertEqual(eval(code), 23) 578 code = compile(memoryview(b"1234")[1:-1], "<dummy>", "eval") 579 self.assertEqual(eval(code), 23) 580 code = compile(memoryview(b"$23$")[1:-1], "<dummy>", "eval") 581 self.assertEqual(eval(code), 23) 582 583 # Also test when eval() and exec() do the compilation step 584 self.assertEqual(eval(memoryview(b"1234")[1:-1]), 23) 585 namespace = dict() 586 exec(memoryview(b"ax = 123")[1:-1], namespace) 587 self.assertEqual(namespace['x'], 12) 588 589 def check_constant(self, func, expected): 590 for const in func.__code__.co_consts: 591 if repr(const) == repr(expected): 592 break 593 else: 594 self.fail("unable to find constant %r in %r" 595 % (expected, func.__code__.co_consts)) 596 597 # Merging equal constants is not a strict requirement for the Python 598 # semantics, it's a more an implementation detail. 599 @support.cpython_only 600 def test_merge_constants(self): 601 # Issue #25843: compile() must merge constants which are equal 602 # and have the same type. 603 604 def check_same_constant(const): 605 ns = {} 606 code = "f1, f2 = lambda: %r, lambda: %r" % (const, const) 607 exec(code, ns) 608 f1 = ns['f1'] 609 f2 = ns['f2'] 610 self.assertIs(f1.__code__, f2.__code__) 611 self.check_constant(f1, const) 612 self.assertEqual(repr(f1()), repr(const)) 613 614 check_same_constant(None) 615 check_same_constant(0) 616 check_same_constant(0.0) 617 check_same_constant(b'abc') 618 check_same_constant('abc') 619 620 # Note: "lambda: ..." emits "LOAD_CONST Ellipsis", 621 # whereas "lambda: Ellipsis" emits "LOAD_GLOBAL Ellipsis" 622 f1, f2 = lambda: ..., lambda: ... 623 self.assertIs(f1.__code__, f2.__code__) 624 self.check_constant(f1, Ellipsis) 625 self.assertEqual(repr(f1()), repr(Ellipsis)) 626 627 # Merge constants in tuple or frozenset 628 f1, f2 = lambda: "not a name", lambda: ("not a name",) 629 f3 = lambda x: x in {("not a name",)} 630 self.assertIs(f1.__code__.co_consts[1], 631 f2.__code__.co_consts[1][0]) 632 self.assertIs(next(iter(f3.__code__.co_consts[1])), 633 f2.__code__.co_consts[1]) 634 635 # {0} is converted to a constant frozenset({0}) by the peephole 636 # optimizer 637 f1, f2 = lambda x: x in {0}, lambda x: x in {0} 638 self.assertIs(f1.__code__, f2.__code__) 639 self.check_constant(f1, frozenset({0})) 640 self.assertTrue(f1(0)) 641 642 # Merging equal co_linetable and co_code is not a strict requirement 643 # for the Python semantics, it's a more an implementation detail. 644 @support.cpython_only 645 def test_merge_code_attrs(self): 646 # See https://bugs.python.org/issue42217 647 f1 = lambda x: x.y.z 648 f2 = lambda a: a.b.c 649 650 self.assertIs(f1.__code__.co_linetable, f2.__code__.co_linetable) 651 self.assertIs(f1.__code__.co_code, f2.__code__.co_code) 652 653 # Stripping unused constants is not a strict requirement for the 654 # Python semantics, it's a more an implementation detail. 655 @support.cpython_only 656 def test_strip_unused_consts(self): 657 # Python 3.10rc1 appended None to co_consts when None is not used 658 # at all. See bpo-45056. 659 def f1(): 660 "docstring" 661 return 42 662 self.assertEqual(f1.__code__.co_consts, ("docstring", 42)) 663 664 # This is a regression test for a CPython specific peephole optimizer 665 # implementation bug present in a few releases. It's assertion verifies 666 # that peephole optimization was actually done though that isn't an 667 # indication of the bugs presence or not (crashing is). 668 @support.cpython_only 669 def test_peephole_opt_unreachable_code_array_access_in_bounds(self): 670 """Regression test for issue35193 when run under clang msan.""" 671 def unused_code_at_end(): 672 return 3 673 raise RuntimeError("unreachable") 674 # The above function definition will trigger the out of bounds 675 # bug in the peephole optimizer as it scans opcodes past the 676 # RETURN_VALUE opcode. This does not always crash an interpreter. 677 # When you build with the clang memory sanitizer it reliably aborts. 678 self.assertEqual( 679 'RETURN_VALUE', 680 list(dis.get_instructions(unused_code_at_end))[-1].opname) 681 682 def test_dont_merge_constants(self): 683 # Issue #25843: compile() must not merge constants which are equal 684 # but have a different type. 685 686 def check_different_constants(const1, const2): 687 ns = {} 688 exec("f1, f2 = lambda: %r, lambda: %r" % (const1, const2), ns) 689 f1 = ns['f1'] 690 f2 = ns['f2'] 691 self.assertIsNot(f1.__code__, f2.__code__) 692 self.assertNotEqual(f1.__code__, f2.__code__) 693 self.check_constant(f1, const1) 694 self.check_constant(f2, const2) 695 self.assertEqual(repr(f1()), repr(const1)) 696 self.assertEqual(repr(f2()), repr(const2)) 697 698 check_different_constants(0, 0.0) 699 check_different_constants(+0.0, -0.0) 700 check_different_constants((0,), (0.0,)) 701 check_different_constants('a', b'a') 702 check_different_constants(('a',), (b'a',)) 703 704 # check_different_constants() cannot be used because repr(-0j) is 705 # '(-0-0j)', but when '(-0-0j)' is evaluated to 0j: we loose the sign. 706 f1, f2 = lambda: +0.0j, lambda: -0.0j 707 self.assertIsNot(f1.__code__, f2.__code__) 708 self.check_constant(f1, +0.0j) 709 self.check_constant(f2, -0.0j) 710 self.assertEqual(repr(f1()), repr(+0.0j)) 711 self.assertEqual(repr(f2()), repr(-0.0j)) 712 713 # {0} is converted to a constant frozenset({0}) by the peephole 714 # optimizer 715 f1, f2 = lambda x: x in {0}, lambda x: x in {0.0} 716 self.assertIsNot(f1.__code__, f2.__code__) 717 self.check_constant(f1, frozenset({0})) 718 self.check_constant(f2, frozenset({0.0})) 719 self.assertTrue(f1(0)) 720 self.assertTrue(f2(0.0)) 721 722 def test_path_like_objects(self): 723 # An implicit test for PyUnicode_FSDecoder(). 724 compile("42", FakePath("test_compile_pathlike"), "single") 725 726 def test_stack_overflow(self): 727 # bpo-31113: Stack overflow when compile a long sequence of 728 # complex statements. 729 compile("if a: b\n" * 200000, "<dummy>", "exec") 730 731 # Multiple users rely on the fact that CPython does not generate 732 # bytecode for dead code blocks. See bpo-37500 for more context. 733 @support.cpython_only 734 def test_dead_blocks_do_not_generate_bytecode(self): 735 def unused_block_if(): 736 if 0: 737 return 42 738 739 def unused_block_while(): 740 while 0: 741 return 42 742 743 def unused_block_if_else(): 744 if 1: 745 return None 746 else: 747 return 42 748 749 def unused_block_while_else(): 750 while 1: 751 return None 752 else: 753 return 42 754 755 funcs = [unused_block_if, unused_block_while, 756 unused_block_if_else, unused_block_while_else] 757 758 for func in funcs: 759 opcodes = list(dis.get_instructions(func)) 760 self.assertLessEqual(len(opcodes), 3) 761 self.assertEqual('LOAD_CONST', opcodes[-2].opname) 762 self.assertEqual(None, opcodes[-2].argval) 763 self.assertEqual('RETURN_VALUE', opcodes[-1].opname) 764 765 def test_false_while_loop(self): 766 def break_in_while(): 767 while False: 768 break 769 770 def continue_in_while(): 771 while False: 772 continue 773 774 funcs = [break_in_while, continue_in_while] 775 776 # Check that we did not raise but we also don't generate bytecode 777 for func in funcs: 778 opcodes = list(dis.get_instructions(func)) 779 self.assertEqual(2, len(opcodes)) 780 self.assertEqual('LOAD_CONST', opcodes[0].opname) 781 self.assertEqual(None, opcodes[0].argval) 782 self.assertEqual('RETURN_VALUE', opcodes[1].opname) 783 784 def test_consts_in_conditionals(self): 785 def and_true(x): 786 return True and x 787 788 def and_false(x): 789 return False and x 790 791 def or_true(x): 792 return True or x 793 794 def or_false(x): 795 return False or x 796 797 funcs = [and_true, and_false, or_true, or_false] 798 799 # Check that condition is removed. 800 for func in funcs: 801 with self.subTest(func=func): 802 opcodes = list(dis.get_instructions(func)) 803 self.assertEqual(2, len(opcodes)) 804 self.assertIn('LOAD_', opcodes[0].opname) 805 self.assertEqual('RETURN_VALUE', opcodes[1].opname) 806 807 def test_lineno_procedure_call(self): 808 def call(): 809 ( 810 print() 811 ) 812 line1 = call.__code__.co_firstlineno + 1 813 assert line1 not in [line for (_, _, line) in call.__code__.co_lines()] 814 815 def test_lineno_after_implicit_return(self): 816 TRUE = True 817 # Don't use constant True or False, as compiler will remove test 818 def if1(x): 819 x() 820 if TRUE: 821 pass 822 def if2(x): 823 x() 824 if TRUE: 825 pass 826 else: 827 pass 828 def if3(x): 829 x() 830 if TRUE: 831 pass 832 else: 833 return None 834 def if4(x): 835 x() 836 if not TRUE: 837 pass 838 funcs = [ if1, if2, if3, if4] 839 lastlines = [ 3, 3, 3, 2] 840 frame = None 841 def save_caller_frame(): 842 nonlocal frame 843 frame = sys._getframe(1) 844 for func, lastline in zip(funcs, lastlines, strict=True): 845 with self.subTest(func=func): 846 func(save_caller_frame) 847 self.assertEqual(frame.f_lineno-frame.f_code.co_firstlineno, lastline) 848 849 def test_lineno_after_no_code(self): 850 def no_code1(): 851 "doc string" 852 853 def no_code2(): 854 a: int 855 856 for func in (no_code1, no_code2): 857 with self.subTest(func=func): 858 code = func.__code__ 859 lines = list(code.co_lines()) 860 self.assertEqual(len(lines), 1) 861 start, end, line = lines[0] 862 self.assertEqual(start, 0) 863 self.assertEqual(end, len(code.co_code)) 864 self.assertEqual(line, code.co_firstlineno) 865 866 def test_lineno_attribute(self): 867 def load_attr(): 868 return ( 869 o. 870 a 871 ) 872 load_attr_lines = [ 2, 3, 1 ] 873 874 def load_method(): 875 return ( 876 o. 877 m( 878 0 879 ) 880 ) 881 load_method_lines = [ 2, 3, 4, 3, 1 ] 882 883 def store_attr(): 884 ( 885 o. 886 a 887 ) = ( 888 v 889 ) 890 store_attr_lines = [ 5, 2, 3 ] 891 892 def aug_store_attr(): 893 ( 894 o. 895 a 896 ) += ( 897 v 898 ) 899 aug_store_attr_lines = [ 2, 3, 5, 1, 3 ] 900 901 funcs = [ load_attr, load_method, store_attr, aug_store_attr] 902 func_lines = [ load_attr_lines, load_method_lines, 903 store_attr_lines, aug_store_attr_lines] 904 905 for func, lines in zip(funcs, func_lines, strict=True): 906 with self.subTest(func=func): 907 code_lines = [ line-func.__code__.co_firstlineno 908 for (_, _, line) in func.__code__.co_lines() ] 909 self.assertEqual(lines, code_lines) 910 911 def test_line_number_genexp(self): 912 913 def return_genexp(): 914 return (1 915 for 916 x 917 in 918 y) 919 genexp_lines = [None, 1, 3, 1] 920 921 genexp_code = return_genexp.__code__.co_consts[1] 922 code_lines = [ None if line is None else line-return_genexp.__code__.co_firstlineno 923 for (_, _, line) in genexp_code.co_lines() ] 924 self.assertEqual(genexp_lines, code_lines) 925 926 def test_line_number_implicit_return_after_async_for(self): 927 928 async def test(aseq): 929 async for i in aseq: 930 body 931 932 expected_lines = [None, 1, 2, 1] 933 code_lines = [ None if line is None else line-test.__code__.co_firstlineno 934 for (_, _, line) in test.__code__.co_lines() ] 935 self.assertEqual(expected_lines, code_lines) 936 937 def test_big_dict_literal(self): 938 # The compiler has a flushing point in "compiler_dict" that calls compiles 939 # a portion of the dictionary literal when the loop that iterates over the items 940 # reaches 0xFFFF elements but the code was not including the boundary element, 941 # dropping the key at position 0xFFFF. See bpo-41531 for more information 942 943 dict_size = 0xFFFF + 1 944 the_dict = "{" + ",".join(f"{x}:{x}" for x in range(dict_size)) + "}" 945 self.assertEqual(len(eval(the_dict)), dict_size) 946 947 def test_redundant_jump_in_if_else_break(self): 948 # Check if bytecode containing jumps that simply point to the next line 949 # is generated around if-else-break style structures. See bpo-42615. 950 951 def if_else_break(): 952 val = 1 953 while True: 954 if val > 0: 955 val -= 1 956 else: 957 break 958 val = -1 959 960 INSTR_SIZE = 2 961 HANDLED_JUMPS = ( 962 'POP_JUMP_IF_FALSE', 963 'POP_JUMP_IF_TRUE', 964 'JUMP_ABSOLUTE', 965 'JUMP_FORWARD', 966 ) 967 968 for line, instr in enumerate(dis.Bytecode(if_else_break)): 969 if instr.opname == 'JUMP_FORWARD': 970 self.assertNotEqual(instr.arg, 0) 971 elif instr.opname in HANDLED_JUMPS: 972 self.assertNotEqual(instr.arg, (line + 1)*INSTR_SIZE) 973 974 975class TestExpressionStackSize(unittest.TestCase): 976 # These tests check that the computed stack size for a code object 977 # stays within reasonable bounds (see issue #21523 for an example 978 # dysfunction). 979 N = 100 980 981 def check_stack_size(self, code): 982 # To assert that the alleged stack size is not O(N), we 983 # check that it is smaller than log(N). 984 if isinstance(code, str): 985 code = compile(code, "<foo>", "single") 986 max_size = math.ceil(math.log(len(code.co_code))) 987 self.assertLessEqual(code.co_stacksize, max_size) 988 989 def test_and(self): 990 self.check_stack_size("x and " * self.N + "x") 991 992 def test_or(self): 993 self.check_stack_size("x or " * self.N + "x") 994 995 def test_and_or(self): 996 self.check_stack_size("x and x or " * self.N + "x") 997 998 def test_chained_comparison(self): 999 self.check_stack_size("x < " * self.N + "x") 1000 1001 def test_if_else(self): 1002 self.check_stack_size("x if x else " * self.N + "x") 1003 1004 def test_binop(self): 1005 self.check_stack_size("x + " * self.N + "x") 1006 1007 def test_list(self): 1008 self.check_stack_size("[" + "x, " * self.N + "x]") 1009 1010 def test_tuple(self): 1011 self.check_stack_size("(" + "x, " * self.N + "x)") 1012 1013 def test_set(self): 1014 self.check_stack_size("{" + "x, " * self.N + "x}") 1015 1016 def test_dict(self): 1017 self.check_stack_size("{" + "x:x, " * self.N + "x:x}") 1018 1019 def test_func_args(self): 1020 self.check_stack_size("f(" + "x, " * self.N + ")") 1021 1022 def test_func_kwargs(self): 1023 kwargs = (f'a{i}=x' for i in range(self.N)) 1024 self.check_stack_size("f(" + ", ".join(kwargs) + ")") 1025 1026 def test_func_args(self): 1027 self.check_stack_size("o.m(" + "x, " * self.N + ")") 1028 1029 def test_meth_kwargs(self): 1030 kwargs = (f'a{i}=x' for i in range(self.N)) 1031 self.check_stack_size("o.m(" + ", ".join(kwargs) + ")") 1032 1033 def test_func_and(self): 1034 code = "def f(x):\n" 1035 code += " x and x\n" * self.N 1036 self.check_stack_size(code) 1037 1038 1039class TestStackSizeStability(unittest.TestCase): 1040 # Check that repeating certain snippets doesn't increase the stack size 1041 # beyond what a single snippet requires. 1042 1043 def check_stack_size(self, snippet, async_=False): 1044 def compile_snippet(i): 1045 ns = {} 1046 script = """def func():\n""" + i * snippet 1047 if async_: 1048 script = "async " + script 1049 code = compile(script, "<script>", "exec") 1050 exec(code, ns, ns) 1051 return ns['func'].__code__ 1052 1053 sizes = [compile_snippet(i).co_stacksize for i in range(2, 5)] 1054 if len(set(sizes)) != 1: 1055 import dis, io 1056 out = io.StringIO() 1057 dis.dis(compile_snippet(1), file=out) 1058 self.fail("stack sizes diverge with # of consecutive snippets: " 1059 "%s\n%s\n%s" % (sizes, snippet, out.getvalue())) 1060 1061 def test_if(self): 1062 snippet = """ 1063 if x: 1064 a 1065 """ 1066 self.check_stack_size(snippet) 1067 1068 def test_if_else(self): 1069 snippet = """ 1070 if x: 1071 a 1072 elif y: 1073 b 1074 else: 1075 c 1076 """ 1077 self.check_stack_size(snippet) 1078 1079 def test_try_except_bare(self): 1080 snippet = """ 1081 try: 1082 a 1083 except: 1084 b 1085 """ 1086 self.check_stack_size(snippet) 1087 1088 def test_try_except_qualified(self): 1089 snippet = """ 1090 try: 1091 a 1092 except ImportError: 1093 b 1094 except: 1095 c 1096 else: 1097 d 1098 """ 1099 self.check_stack_size(snippet) 1100 1101 def test_try_except_as(self): 1102 snippet = """ 1103 try: 1104 a 1105 except ImportError as e: 1106 b 1107 except: 1108 c 1109 else: 1110 d 1111 """ 1112 self.check_stack_size(snippet) 1113 1114 def test_try_finally(self): 1115 snippet = """ 1116 try: 1117 a 1118 finally: 1119 b 1120 """ 1121 self.check_stack_size(snippet) 1122 1123 def test_with(self): 1124 snippet = """ 1125 with x as y: 1126 a 1127 """ 1128 self.check_stack_size(snippet) 1129 1130 def test_while_else(self): 1131 snippet = """ 1132 while x: 1133 a 1134 else: 1135 b 1136 """ 1137 self.check_stack_size(snippet) 1138 1139 def test_for(self): 1140 snippet = """ 1141 for x in y: 1142 a 1143 """ 1144 self.check_stack_size(snippet) 1145 1146 def test_for_else(self): 1147 snippet = """ 1148 for x in y: 1149 a 1150 else: 1151 b 1152 """ 1153 self.check_stack_size(snippet) 1154 1155 def test_for_break_continue(self): 1156 snippet = """ 1157 for x in y: 1158 if z: 1159 break 1160 elif u: 1161 continue 1162 else: 1163 a 1164 else: 1165 b 1166 """ 1167 self.check_stack_size(snippet) 1168 1169 def test_for_break_continue_inside_try_finally_block(self): 1170 snippet = """ 1171 for x in y: 1172 try: 1173 if z: 1174 break 1175 elif u: 1176 continue 1177 else: 1178 a 1179 finally: 1180 f 1181 else: 1182 b 1183 """ 1184 self.check_stack_size(snippet) 1185 1186 def test_for_break_continue_inside_finally_block(self): 1187 snippet = """ 1188 for x in y: 1189 try: 1190 t 1191 finally: 1192 if z: 1193 break 1194 elif u: 1195 continue 1196 else: 1197 a 1198 else: 1199 b 1200 """ 1201 self.check_stack_size(snippet) 1202 1203 def test_for_break_continue_inside_except_block(self): 1204 snippet = """ 1205 for x in y: 1206 try: 1207 t 1208 except: 1209 if z: 1210 break 1211 elif u: 1212 continue 1213 else: 1214 a 1215 else: 1216 b 1217 """ 1218 self.check_stack_size(snippet) 1219 1220 def test_for_break_continue_inside_with_block(self): 1221 snippet = """ 1222 for x in y: 1223 with c: 1224 if z: 1225 break 1226 elif u: 1227 continue 1228 else: 1229 a 1230 else: 1231 b 1232 """ 1233 self.check_stack_size(snippet) 1234 1235 def test_return_inside_try_finally_block(self): 1236 snippet = """ 1237 try: 1238 if z: 1239 return 1240 else: 1241 a 1242 finally: 1243 f 1244 """ 1245 self.check_stack_size(snippet) 1246 1247 def test_return_inside_finally_block(self): 1248 snippet = """ 1249 try: 1250 t 1251 finally: 1252 if z: 1253 return 1254 else: 1255 a 1256 """ 1257 self.check_stack_size(snippet) 1258 1259 def test_return_inside_except_block(self): 1260 snippet = """ 1261 try: 1262 t 1263 except: 1264 if z: 1265 return 1266 else: 1267 a 1268 """ 1269 self.check_stack_size(snippet) 1270 1271 def test_return_inside_with_block(self): 1272 snippet = """ 1273 with c: 1274 if z: 1275 return 1276 else: 1277 a 1278 """ 1279 self.check_stack_size(snippet) 1280 1281 def test_async_with(self): 1282 snippet = """ 1283 async with x as y: 1284 a 1285 """ 1286 self.check_stack_size(snippet, async_=True) 1287 1288 def test_async_for(self): 1289 snippet = """ 1290 async for x in y: 1291 a 1292 """ 1293 self.check_stack_size(snippet, async_=True) 1294 1295 def test_async_for_else(self): 1296 snippet = """ 1297 async for x in y: 1298 a 1299 else: 1300 b 1301 """ 1302 self.check_stack_size(snippet, async_=True) 1303 1304 def test_for_break_continue_inside_async_with_block(self): 1305 snippet = """ 1306 for x in y: 1307 async with c: 1308 if z: 1309 break 1310 elif u: 1311 continue 1312 else: 1313 a 1314 else: 1315 b 1316 """ 1317 self.check_stack_size(snippet, async_=True) 1318 1319 def test_return_inside_async_with_block(self): 1320 snippet = """ 1321 async with c: 1322 if z: 1323 return 1324 else: 1325 a 1326 """ 1327 self.check_stack_size(snippet, async_=True) 1328 1329 1330if __name__ == "__main__": 1331 unittest.main() 1332