1import unittest 2from test.test_support import verbose, run_unittest, start_threads 3import sys 4import time 5import gc 6import weakref 7 8try: 9 import threading 10except ImportError: 11 threading = None 12 13### Support code 14############################################################################### 15 16# Bug 1055820 has several tests of longstanding bugs involving weakrefs and 17# cyclic gc. 18 19# An instance of C1055820 has a self-loop, so becomes cyclic trash when 20# unreachable. 21class C1055820(object): 22 def __init__(self, i): 23 self.i = i 24 self.loop = self 25 26class GC_Detector(object): 27 # Create an instance I. Then gc hasn't happened again so long as 28 # I.gc_happened is false. 29 30 def __init__(self): 31 self.gc_happened = False 32 33 def it_happened(ignored): 34 self.gc_happened = True 35 36 # Create a piece of cyclic trash that triggers it_happened when 37 # gc collects it. 38 self.wr = weakref.ref(C1055820(666), it_happened) 39 40 41### Tests 42############################################################################### 43 44class GCTests(unittest.TestCase): 45 def test_list(self): 46 l = [] 47 l.append(l) 48 gc.collect() 49 del l 50 self.assertEqual(gc.collect(), 1) 51 52 def test_dict(self): 53 d = {} 54 d[1] = d 55 gc.collect() 56 del d 57 self.assertEqual(gc.collect(), 1) 58 59 def test_tuple(self): 60 # since tuples are immutable we close the loop with a list 61 l = [] 62 t = (l,) 63 l.append(t) 64 gc.collect() 65 del t 66 del l 67 self.assertEqual(gc.collect(), 2) 68 69 def test_class(self): 70 class A: 71 pass 72 A.a = A 73 gc.collect() 74 del A 75 self.assertNotEqual(gc.collect(), 0) 76 77 def test_newstyleclass(self): 78 class A(object): 79 pass 80 gc.collect() 81 del A 82 self.assertNotEqual(gc.collect(), 0) 83 84 def test_instance(self): 85 class A: 86 pass 87 a = A() 88 a.a = a 89 gc.collect() 90 del a 91 self.assertNotEqual(gc.collect(), 0) 92 93 def test_newinstance(self): 94 class A(object): 95 pass 96 a = A() 97 a.a = a 98 gc.collect() 99 del a 100 self.assertNotEqual(gc.collect(), 0) 101 class B(list): 102 pass 103 class C(B, A): 104 pass 105 a = C() 106 a.a = a 107 gc.collect() 108 del a 109 self.assertNotEqual(gc.collect(), 0) 110 del B, C 111 self.assertNotEqual(gc.collect(), 0) 112 A.a = A() 113 del A 114 self.assertNotEqual(gc.collect(), 0) 115 self.assertEqual(gc.collect(), 0) 116 117 def test_method(self): 118 # Tricky: self.__init__ is a bound method, it references the instance. 119 class A: 120 def __init__(self): 121 self.init = self.__init__ 122 a = A() 123 gc.collect() 124 del a 125 self.assertNotEqual(gc.collect(), 0) 126 127 def test_finalizer(self): 128 # A() is uncollectable if it is part of a cycle, make sure it shows up 129 # in gc.garbage. 130 class A: 131 def __del__(self): pass 132 class B: 133 pass 134 a = A() 135 a.a = a 136 id_a = id(a) 137 b = B() 138 b.b = b 139 gc.collect() 140 del a 141 del b 142 self.assertNotEqual(gc.collect(), 0) 143 for obj in gc.garbage: 144 if id(obj) == id_a: 145 del obj.a 146 break 147 else: 148 self.fail("didn't find obj in garbage (finalizer)") 149 gc.garbage.remove(obj) 150 151 def test_finalizer_newclass(self): 152 # A() is uncollectable if it is part of a cycle, make sure it shows up 153 # in gc.garbage. 154 class A(object): 155 def __del__(self): pass 156 class B(object): 157 pass 158 a = A() 159 a.a = a 160 id_a = id(a) 161 b = B() 162 b.b = b 163 gc.collect() 164 del a 165 del b 166 self.assertNotEqual(gc.collect(), 0) 167 for obj in gc.garbage: 168 if id(obj) == id_a: 169 del obj.a 170 break 171 else: 172 self.fail("didn't find obj in garbage (finalizer)") 173 gc.garbage.remove(obj) 174 175 def test_function(self): 176 # Tricky: f -> d -> f, code should call d.clear() after the exec to 177 # break the cycle. 178 d = {} 179 exec("def f(): pass\n") in d 180 gc.collect() 181 del d 182 self.assertEqual(gc.collect(), 2) 183 184 def test_frame(self): 185 def f(): 186 frame = sys._getframe() 187 gc.collect() 188 f() 189 self.assertEqual(gc.collect(), 1) 190 191 def test_saveall(self): 192 # Verify that cyclic garbage like lists show up in gc.garbage if the 193 # SAVEALL option is enabled. 194 195 # First make sure we don't save away other stuff that just happens to 196 # be waiting for collection. 197 gc.collect() 198 # if this fails, someone else created immortal trash 199 self.assertEqual(gc.garbage, []) 200 201 L = [] 202 L.append(L) 203 id_L = id(L) 204 205 debug = gc.get_debug() 206 gc.set_debug(debug | gc.DEBUG_SAVEALL) 207 del L 208 gc.collect() 209 gc.set_debug(debug) 210 211 self.assertEqual(len(gc.garbage), 1) 212 obj = gc.garbage.pop() 213 self.assertEqual(id(obj), id_L) 214 215 def test_del(self): 216 # __del__ methods can trigger collection, make this to happen 217 thresholds = gc.get_threshold() 218 gc.enable() 219 gc.set_threshold(1) 220 221 class A: 222 def __del__(self): 223 dir(self) 224 a = A() 225 del a 226 227 gc.disable() 228 gc.set_threshold(*thresholds) 229 230 def test_del_newclass(self): 231 # __del__ methods can trigger collection, make this to happen 232 thresholds = gc.get_threshold() 233 gc.enable() 234 gc.set_threshold(1) 235 236 class A(object): 237 def __del__(self): 238 dir(self) 239 a = A() 240 del a 241 242 gc.disable() 243 gc.set_threshold(*thresholds) 244 245 # The following two tests are fragile: 246 # They precisely count the number of allocations, 247 # which is highly implementation-dependent. 248 # For example: 249 # - disposed tuples are not freed, but reused 250 # - the call to assertEqual somehow avoids building its args tuple 251 def test_get_count(self): 252 # Avoid future allocation of method object 253 assertEqual = self._baseAssertEqual 254 gc.collect() 255 assertEqual(gc.get_count(), (0, 0, 0)) 256 a = dict() 257 # since gc.collect(), we created two objects: 258 # the dict, and the tuple returned by get_count() 259 assertEqual(gc.get_count(), (2, 0, 0)) 260 261 def test_collect_generations(self): 262 # Avoid future allocation of method object 263 assertEqual = self.assertEqual 264 gc.collect() 265 a = dict() 266 gc.collect(0) 267 assertEqual(gc.get_count(), (0, 1, 0)) 268 gc.collect(1) 269 assertEqual(gc.get_count(), (0, 0, 1)) 270 gc.collect(2) 271 assertEqual(gc.get_count(), (0, 0, 0)) 272 273 def test_trashcan(self): 274 class Ouch: 275 n = 0 276 def __del__(self): 277 Ouch.n = Ouch.n + 1 278 if Ouch.n % 17 == 0: 279 gc.collect() 280 281 # "trashcan" is a hack to prevent stack overflow when deallocating 282 # very deeply nested tuples etc. It works in part by abusing the 283 # type pointer and refcount fields, and that can yield horrible 284 # problems when gc tries to traverse the structures. 285 # If this test fails (as it does in 2.0, 2.1 and 2.2), it will 286 # most likely die via segfault. 287 288 # Note: In 2.3 the possibility for compiling without cyclic gc was 289 # removed, and that in turn allows the trashcan mechanism to work 290 # via much simpler means (e.g., it never abuses the type pointer or 291 # refcount fields anymore). Since it's much less likely to cause a 292 # problem now, the various constants in this expensive (we force a lot 293 # of full collections) test are cut back from the 2.2 version. 294 gc.enable() 295 N = 150 296 for count in range(2): 297 t = [] 298 for i in range(N): 299 t = [t, Ouch()] 300 u = [] 301 for i in range(N): 302 u = [u, Ouch()] 303 v = {} 304 for i in range(N): 305 v = {1: v, 2: Ouch()} 306 gc.disable() 307 308 @unittest.skipUnless(threading, "test meaningless on builds without threads") 309 def test_trashcan_threads(self): 310 # Issue #13992: trashcan mechanism should be thread-safe 311 NESTING = 60 312 N_THREADS = 2 313 314 def sleeper_gen(): 315 """A generator that releases the GIL when closed or dealloc'ed.""" 316 try: 317 yield 318 finally: 319 time.sleep(0.000001) 320 321 class C(list): 322 # Appending to a list is atomic, which avoids the use of a lock. 323 inits = [] 324 dels = [] 325 def __init__(self, alist): 326 self[:] = alist 327 C.inits.append(None) 328 def __del__(self): 329 # This __del__ is called by subtype_dealloc(). 330 C.dels.append(None) 331 # `g` will release the GIL when garbage-collected. This 332 # helps assert subtype_dealloc's behaviour when threads 333 # switch in the middle of it. 334 g = sleeper_gen() 335 next(g) 336 # Now that __del__ is finished, subtype_dealloc will proceed 337 # to call list_dealloc, which also uses the trashcan mechanism. 338 339 def make_nested(): 340 """Create a sufficiently nested container object so that the 341 trashcan mechanism is invoked when deallocating it.""" 342 x = C([]) 343 for i in range(NESTING): 344 x = [C([x])] 345 del x 346 347 def run_thread(): 348 """Exercise make_nested() in a loop.""" 349 while not exit: 350 make_nested() 351 352 old_checkinterval = sys.getcheckinterval() 353 sys.setcheckinterval(3) 354 try: 355 exit = [] 356 threads = [] 357 for i in range(N_THREADS): 358 t = threading.Thread(target=run_thread) 359 threads.append(t) 360 with start_threads(threads, lambda: exit.append(1)): 361 time.sleep(1.0) 362 finally: 363 sys.setcheckinterval(old_checkinterval) 364 gc.collect() 365 self.assertEqual(len(C.inits), len(C.dels)) 366 367 def test_boom(self): 368 class Boom: 369 def __getattr__(self, someattribute): 370 del self.attr 371 raise AttributeError 372 373 a = Boom() 374 b = Boom() 375 a.attr = b 376 b.attr = a 377 378 gc.collect() 379 garbagelen = len(gc.garbage) 380 del a, b 381 # a<->b are in a trash cycle now. Collection will invoke 382 # Boom.__getattr__ (to see whether a and b have __del__ methods), and 383 # __getattr__ deletes the internal "attr" attributes as a side effect. 384 # That causes the trash cycle to get reclaimed via refcounts falling to 385 # 0, thus mutating the trash graph as a side effect of merely asking 386 # whether __del__ exists. This used to (before 2.3b1) crash Python. 387 # Now __getattr__ isn't called. 388 self.assertEqual(gc.collect(), 4) 389 self.assertEqual(len(gc.garbage), garbagelen) 390 391 def test_boom2(self): 392 class Boom2: 393 def __init__(self): 394 self.x = 0 395 396 def __getattr__(self, someattribute): 397 self.x += 1 398 if self.x > 1: 399 del self.attr 400 raise AttributeError 401 402 a = Boom2() 403 b = Boom2() 404 a.attr = b 405 b.attr = a 406 407 gc.collect() 408 garbagelen = len(gc.garbage) 409 del a, b 410 # Much like test_boom(), except that __getattr__ doesn't break the 411 # cycle until the second time gc checks for __del__. As of 2.3b1, 412 # there isn't a second time, so this simply cleans up the trash cycle. 413 # We expect a, b, a.__dict__ and b.__dict__ (4 objects) to get 414 # reclaimed this way. 415 self.assertEqual(gc.collect(), 4) 416 self.assertEqual(len(gc.garbage), garbagelen) 417 418 def test_boom_new(self): 419 # boom__new and boom2_new are exactly like boom and boom2, except use 420 # new-style classes. 421 422 class Boom_New(object): 423 def __getattr__(self, someattribute): 424 del self.attr 425 raise AttributeError 426 427 a = Boom_New() 428 b = Boom_New() 429 a.attr = b 430 b.attr = a 431 432 gc.collect() 433 garbagelen = len(gc.garbage) 434 del a, b 435 self.assertEqual(gc.collect(), 4) 436 self.assertEqual(len(gc.garbage), garbagelen) 437 438 def test_boom2_new(self): 439 class Boom2_New(object): 440 def __init__(self): 441 self.x = 0 442 443 def __getattr__(self, someattribute): 444 self.x += 1 445 if self.x > 1: 446 del self.attr 447 raise AttributeError 448 449 a = Boom2_New() 450 b = Boom2_New() 451 a.attr = b 452 b.attr = a 453 454 gc.collect() 455 garbagelen = len(gc.garbage) 456 del a, b 457 self.assertEqual(gc.collect(), 4) 458 self.assertEqual(len(gc.garbage), garbagelen) 459 460 def test_get_referents(self): 461 alist = [1, 3, 5] 462 got = gc.get_referents(alist) 463 got.sort() 464 self.assertEqual(got, alist) 465 466 atuple = tuple(alist) 467 got = gc.get_referents(atuple) 468 got.sort() 469 self.assertEqual(got, alist) 470 471 adict = {1: 3, 5: 7} 472 expected = [1, 3, 5, 7] 473 got = gc.get_referents(adict) 474 got.sort() 475 self.assertEqual(got, expected) 476 477 got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0)) 478 got.sort() 479 self.assertEqual(got, [0, 0] + range(5)) 480 481 self.assertEqual(gc.get_referents(1, 'a', 4j), []) 482 483 def test_is_tracked(self): 484 # Atomic built-in types are not tracked, user-defined objects and 485 # mutable containers are. 486 # NOTE: types with special optimizations (e.g. tuple) have tests 487 # in their own test files instead. 488 self.assertFalse(gc.is_tracked(None)) 489 self.assertFalse(gc.is_tracked(1)) 490 self.assertFalse(gc.is_tracked(1.0)) 491 self.assertFalse(gc.is_tracked(1.0 + 5.0j)) 492 self.assertFalse(gc.is_tracked(True)) 493 self.assertFalse(gc.is_tracked(False)) 494 self.assertFalse(gc.is_tracked("a")) 495 self.assertFalse(gc.is_tracked(u"a")) 496 self.assertFalse(gc.is_tracked(bytearray("a"))) 497 self.assertFalse(gc.is_tracked(type)) 498 self.assertFalse(gc.is_tracked(int)) 499 self.assertFalse(gc.is_tracked(object)) 500 self.assertFalse(gc.is_tracked(object())) 501 502 class OldStyle: 503 pass 504 class NewStyle(object): 505 pass 506 self.assertTrue(gc.is_tracked(gc)) 507 self.assertTrue(gc.is_tracked(OldStyle)) 508 self.assertTrue(gc.is_tracked(OldStyle())) 509 self.assertTrue(gc.is_tracked(NewStyle)) 510 self.assertTrue(gc.is_tracked(NewStyle())) 511 self.assertTrue(gc.is_tracked([])) 512 self.assertTrue(gc.is_tracked(set())) 513 514 def test_bug1055820b(self): 515 # Corresponds to temp2b.py in the bug report. 516 517 ouch = [] 518 def callback(ignored): 519 ouch[:] = [wr() for wr in WRs] 520 521 Cs = [C1055820(i) for i in range(2)] 522 WRs = [weakref.ref(c, callback) for c in Cs] 523 c = None 524 525 gc.collect() 526 self.assertEqual(len(ouch), 0) 527 # Make the two instances trash, and collect again. The bug was that 528 # the callback materialized a strong reference to an instance, but gc 529 # cleared the instance's dict anyway. 530 Cs = None 531 gc.collect() 532 self.assertEqual(len(ouch), 2) # else the callbacks didn't run 533 for x in ouch: 534 # If the callback resurrected one of these guys, the instance 535 # would be damaged, with an empty __dict__. 536 self.assertEqual(x, None) 537 538class GCTogglingTests(unittest.TestCase): 539 def setUp(self): 540 gc.enable() 541 542 def tearDown(self): 543 gc.disable() 544 545 def test_bug1055820c(self): 546 # Corresponds to temp2c.py in the bug report. This is pretty 547 # elaborate. 548 549 c0 = C1055820(0) 550 # Move c0 into generation 2. 551 gc.collect() 552 553 c1 = C1055820(1) 554 c1.keep_c0_alive = c0 555 del c0.loop # now only c1 keeps c0 alive 556 557 c2 = C1055820(2) 558 c2wr = weakref.ref(c2) # no callback! 559 560 ouch = [] 561 def callback(ignored): 562 ouch[:] = [c2wr()] 563 564 # The callback gets associated with a wr on an object in generation 2. 565 c0wr = weakref.ref(c0, callback) 566 567 c0 = c1 = c2 = None 568 569 # What we've set up: c0, c1, and c2 are all trash now. c0 is in 570 # generation 2. The only thing keeping it alive is that c1 points to 571 # it. c1 and c2 are in generation 0, and are in self-loops. There's a 572 # global weakref to c2 (c2wr), but that weakref has no callback. 573 # There's also a global weakref to c0 (c0wr), and that does have a 574 # callback, and that callback references c2 via c2wr(). 575 # 576 # c0 has a wr with callback, which references c2wr 577 # ^ 578 # | 579 # | Generation 2 above dots 580 #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . . 581 # | Generation 0 below dots 582 # | 583 # | 584 # ^->c1 ^->c2 has a wr but no callback 585 # | | | | 586 # <--v <--v 587 # 588 # So this is the nightmare: when generation 0 gets collected, we see 589 # that c2 has a callback-free weakref, and c1 doesn't even have a 590 # weakref. Collecting generation 0 doesn't see c0 at all, and c0 is 591 # the only object that has a weakref with a callback. gc clears c1 592 # and c2. Clearing c1 has the side effect of dropping the refcount on 593 # c0 to 0, so c0 goes away (despite that it's in an older generation) 594 # and c0's wr callback triggers. That in turn materializes a reference 595 # to c2 via c2wr(), but c2 gets cleared anyway by gc. 596 597 # We want to let gc happen "naturally", to preserve the distinction 598 # between generations. 599 junk = [] 600 i = 0 601 detector = GC_Detector() 602 while not detector.gc_happened: 603 i += 1 604 if i > 10000: 605 self.fail("gc didn't happen after 10000 iterations") 606 self.assertEqual(len(ouch), 0) 607 junk.append([]) # this will eventually trigger gc 608 609 self.assertEqual(len(ouch), 1) # else the callback wasn't invoked 610 for x in ouch: 611 # If the callback resurrected c2, the instance would be damaged, 612 # with an empty __dict__. 613 self.assertEqual(x, None) 614 615 def test_bug1055820d(self): 616 # Corresponds to temp2d.py in the bug report. This is very much like 617 # test_bug1055820c, but uses a __del__ method instead of a weakref 618 # callback to sneak in a resurrection of cyclic trash. 619 620 ouch = [] 621 class D(C1055820): 622 def __del__(self): 623 ouch[:] = [c2wr()] 624 625 d0 = D(0) 626 # Move all the above into generation 2. 627 gc.collect() 628 629 c1 = C1055820(1) 630 c1.keep_d0_alive = d0 631 del d0.loop # now only c1 keeps d0 alive 632 633 c2 = C1055820(2) 634 c2wr = weakref.ref(c2) # no callback! 635 636 d0 = c1 = c2 = None 637 638 # What we've set up: d0, c1, and c2 are all trash now. d0 is in 639 # generation 2. The only thing keeping it alive is that c1 points to 640 # it. c1 and c2 are in generation 0, and are in self-loops. There's 641 # a global weakref to c2 (c2wr), but that weakref has no callback. 642 # There are no other weakrefs. 643 # 644 # d0 has a __del__ method that references c2wr 645 # ^ 646 # | 647 # | Generation 2 above dots 648 #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . . 649 # | Generation 0 below dots 650 # | 651 # | 652 # ^->c1 ^->c2 has a wr but no callback 653 # | | | | 654 # <--v <--v 655 # 656 # So this is the nightmare: when generation 0 gets collected, we see 657 # that c2 has a callback-free weakref, and c1 doesn't even have a 658 # weakref. Collecting generation 0 doesn't see d0 at all. gc clears 659 # c1 and c2. Clearing c1 has the side effect of dropping the refcount 660 # on d0 to 0, so d0 goes away (despite that it's in an older 661 # generation) and d0's __del__ triggers. That in turn materializes 662 # a reference to c2 via c2wr(), but c2 gets cleared anyway by gc. 663 664 # We want to let gc happen "naturally", to preserve the distinction 665 # between generations. 666 detector = GC_Detector() 667 junk = [] 668 i = 0 669 while not detector.gc_happened: 670 i += 1 671 if i > 10000: 672 self.fail("gc didn't happen after 10000 iterations") 673 self.assertEqual(len(ouch), 0) 674 junk.append([]) # this will eventually trigger gc 675 676 self.assertEqual(len(ouch), 1) # else __del__ wasn't invoked 677 for x in ouch: 678 # If __del__ resurrected c2, the instance would be damaged, with an 679 # empty __dict__. 680 self.assertEqual(x, None) 681 682def test_main(): 683 enabled = gc.isenabled() 684 gc.disable() 685 assert not gc.isenabled() 686 debug = gc.get_debug() 687 gc.set_debug(debug & ~gc.DEBUG_LEAK) # this test is supposed to leak 688 689 try: 690 gc.collect() # Delete 2nd generation garbage 691 run_unittest(GCTests, GCTogglingTests) 692 finally: 693 gc.set_debug(debug) 694 # test gc.enable() even if GC is disabled by default 695 if verbose: 696 print "restoring automatic collection" 697 # make sure to always test gc.enable() 698 gc.enable() 699 assert gc.isenabled() 700 if not enabled: 701 gc.disable() 702 703if __name__ == "__main__": 704 test_main() 705