1import unittest 2from test.support import (verbose, run_unittest, start_threads, 3 requires_type_collecting) 4import sys 5import time 6import gc 7import weakref 8 9try: 10 import threading 11except ImportError: 12 threading = None 13 14### Support code 15############################################################################### 16 17# Bug 1055820 has several tests of longstanding bugs involving weakrefs and 18# cyclic gc. 19 20# An instance of C1055820 has a self-loop, so becomes cyclic trash when 21# unreachable. 22class C1055820(object): 23 def __init__(self, i): 24 self.i = i 25 self.loop = self 26 27class GC_Detector(object): 28 # Create an instance I. Then gc hasn't happened again so long as 29 # I.gc_happened is false. 30 31 def __init__(self): 32 self.gc_happened = False 33 34 def it_happened(ignored): 35 self.gc_happened = True 36 37 # Create a piece of cyclic trash that triggers it_happened when 38 # gc collects it. 39 self.wr = weakref.ref(C1055820(666), it_happened) 40 41 42### Tests 43############################################################################### 44 45class GCTests(unittest.TestCase): 46 def test_list(self): 47 l = [] 48 l.append(l) 49 gc.collect() 50 del l 51 self.assertEqual(gc.collect(), 1) 52 53 def test_dict(self): 54 d = {} 55 d[1] = d 56 gc.collect() 57 del d 58 self.assertEqual(gc.collect(), 1) 59 60 def test_tuple(self): 61 # since tuples are immutable we close the loop with a list 62 l = [] 63 t = (l,) 64 l.append(t) 65 gc.collect() 66 del t 67 del l 68 self.assertEqual(gc.collect(), 2) 69 70 def test_class(self): 71 class A: 72 pass 73 A.a = A 74 gc.collect() 75 del A 76 self.assertNotEqual(gc.collect(), 0) 77 78 def test_newstyleclass(self): 79 class A(object): 80 pass 81 gc.collect() 82 del A 83 self.assertNotEqual(gc.collect(), 0) 84 85 def test_instance(self): 86 class A: 87 pass 88 a = A() 89 a.a = a 90 gc.collect() 91 del a 92 self.assertNotEqual(gc.collect(), 0) 93 94 @requires_type_collecting 95 def test_newinstance(self): 96 class A(object): 97 pass 98 a = A() 99 a.a = a 100 gc.collect() 101 del a 102 self.assertNotEqual(gc.collect(), 0) 103 class B(list): 104 pass 105 class C(B, A): 106 pass 107 a = C() 108 a.a = a 109 gc.collect() 110 del a 111 self.assertNotEqual(gc.collect(), 0) 112 del B, C 113 self.assertNotEqual(gc.collect(), 0) 114 A.a = A() 115 del A 116 self.assertNotEqual(gc.collect(), 0) 117 self.assertEqual(gc.collect(), 0) 118 119 def test_method(self): 120 # Tricky: self.__init__ is a bound method, it references the instance. 121 class A: 122 def __init__(self): 123 self.init = self.__init__ 124 a = A() 125 gc.collect() 126 del a 127 self.assertNotEqual(gc.collect(), 0) 128 129 def test_finalizer(self): 130 # A() is uncollectable if it is part of a cycle, make sure it shows up 131 # in gc.garbage. 132 class A: 133 def __del__(self): pass 134 class B: 135 pass 136 a = A() 137 a.a = a 138 id_a = id(a) 139 b = B() 140 b.b = b 141 gc.collect() 142 del a 143 del b 144 self.assertNotEqual(gc.collect(), 0) 145 for obj in gc.garbage: 146 if id(obj) == id_a: 147 del obj.a 148 break 149 else: 150 self.fail("didn't find obj in garbage (finalizer)") 151 gc.garbage.remove(obj) 152 153 def test_finalizer_newclass(self): 154 # A() is uncollectable if it is part of a cycle, make sure it shows up 155 # in gc.garbage. 156 class A(object): 157 def __del__(self): pass 158 class B(object): 159 pass 160 a = A() 161 a.a = a 162 id_a = id(a) 163 b = B() 164 b.b = b 165 gc.collect() 166 del a 167 del b 168 self.assertNotEqual(gc.collect(), 0) 169 for obj in gc.garbage: 170 if id(obj) == id_a: 171 del obj.a 172 break 173 else: 174 self.fail("didn't find obj in garbage (finalizer)") 175 gc.garbage.remove(obj) 176 177 def test_function(self): 178 # Tricky: f -> d -> f, code should call d.clear() after the exec to 179 # break the cycle. 180 d = {} 181 exec("def f(): pass\n") in d 182 gc.collect() 183 del d 184 self.assertEqual(gc.collect(), 2) 185 186 def test_frame(self): 187 def f(): 188 frame = sys._getframe() 189 gc.collect() 190 f() 191 self.assertEqual(gc.collect(), 1) 192 193 def test_saveall(self): 194 # Verify that cyclic garbage like lists show up in gc.garbage if the 195 # SAVEALL option is enabled. 196 197 # First make sure we don't save away other stuff that just happens to 198 # be waiting for collection. 199 gc.collect() 200 # if this fails, someone else created immortal trash 201 self.assertEqual(gc.garbage, []) 202 203 L = [] 204 L.append(L) 205 id_L = id(L) 206 207 debug = gc.get_debug() 208 gc.set_debug(debug | gc.DEBUG_SAVEALL) 209 del L 210 gc.collect() 211 gc.set_debug(debug) 212 213 self.assertEqual(len(gc.garbage), 1) 214 obj = gc.garbage.pop() 215 self.assertEqual(id(obj), id_L) 216 217 def test_del(self): 218 # __del__ methods can trigger collection, make this to happen 219 thresholds = gc.get_threshold() 220 gc.enable() 221 gc.set_threshold(1) 222 223 class A: 224 def __del__(self): 225 dir(self) 226 a = A() 227 del a 228 229 gc.disable() 230 gc.set_threshold(*thresholds) 231 232 def test_del_newclass(self): 233 # __del__ methods can trigger collection, make this to happen 234 thresholds = gc.get_threshold() 235 gc.enable() 236 gc.set_threshold(1) 237 238 class A(object): 239 def __del__(self): 240 dir(self) 241 a = A() 242 del a 243 244 gc.disable() 245 gc.set_threshold(*thresholds) 246 247 # The following two tests are fragile: 248 # They precisely count the number of allocations, 249 # which is highly implementation-dependent. 250 # For example: 251 # - disposed tuples are not freed, but reused 252 # - the call to assertEqual somehow avoids building its args tuple 253 def test_get_count(self): 254 # Avoid future allocation of method object 255 assertEqual = self._baseAssertEqual 256 gc.collect() 257 assertEqual(gc.get_count(), (0, 0, 0)) 258 a = dict() 259 # since gc.collect(), we created two objects: 260 # the dict, and the tuple returned by get_count() 261 assertEqual(gc.get_count(), (2, 0, 0)) 262 263 def test_collect_generations(self): 264 # Avoid future allocation of method object 265 assertEqual = self.assertEqual 266 gc.collect() 267 a = dict() 268 gc.collect(0) 269 assertEqual(gc.get_count(), (0, 1, 0)) 270 gc.collect(1) 271 assertEqual(gc.get_count(), (0, 0, 1)) 272 gc.collect(2) 273 assertEqual(gc.get_count(), (0, 0, 0)) 274 275 def test_trashcan(self): 276 class Ouch: 277 n = 0 278 def __del__(self): 279 Ouch.n = Ouch.n + 1 280 if Ouch.n % 17 == 0: 281 gc.collect() 282 283 # "trashcan" is a hack to prevent stack overflow when deallocating 284 # very deeply nested tuples etc. It works in part by abusing the 285 # type pointer and refcount fields, and that can yield horrible 286 # problems when gc tries to traverse the structures. 287 # If this test fails (as it does in 2.0, 2.1 and 2.2), it will 288 # most likely die via segfault. 289 290 # Note: In 2.3 the possibility for compiling without cyclic gc was 291 # removed, and that in turn allows the trashcan mechanism to work 292 # via much simpler means (e.g., it never abuses the type pointer or 293 # refcount fields anymore). Since it's much less likely to cause a 294 # problem now, the various constants in this expensive (we force a lot 295 # of full collections) test are cut back from the 2.2 version. 296 gc.enable() 297 N = 150 298 for count in range(2): 299 t = [] 300 for i in range(N): 301 t = [t, Ouch()] 302 u = [] 303 for i in range(N): 304 u = [u, Ouch()] 305 v = {} 306 for i in range(N): 307 v = {1: v, 2: Ouch()} 308 gc.disable() 309 310 @unittest.skipUnless(threading, "test meaningless on builds without threads") 311 def test_trashcan_threads(self): 312 # Issue #13992: trashcan mechanism should be thread-safe 313 NESTING = 60 314 N_THREADS = 2 315 316 def sleeper_gen(): 317 """A generator that releases the GIL when closed or dealloc'ed.""" 318 try: 319 yield 320 finally: 321 time.sleep(0.000001) 322 323 class C(list): 324 # Appending to a list is atomic, which avoids the use of a lock. 325 inits = [] 326 dels = [] 327 def __init__(self, alist): 328 self[:] = alist 329 C.inits.append(None) 330 def __del__(self): 331 # This __del__ is called by subtype_dealloc(). 332 C.dels.append(None) 333 # `g` will release the GIL when garbage-collected. This 334 # helps assert subtype_dealloc's behaviour when threads 335 # switch in the middle of it. 336 g = sleeper_gen() 337 next(g) 338 # Now that __del__ is finished, subtype_dealloc will proceed 339 # to call list_dealloc, which also uses the trashcan mechanism. 340 341 def make_nested(): 342 """Create a sufficiently nested container object so that the 343 trashcan mechanism is invoked when deallocating it.""" 344 x = C([]) 345 for i in range(NESTING): 346 x = [C([x])] 347 del x 348 349 def run_thread(): 350 """Exercise make_nested() in a loop.""" 351 while not exit: 352 make_nested() 353 354 old_checkinterval = sys.getcheckinterval() 355 sys.setcheckinterval(3) 356 try: 357 exit = [] 358 threads = [] 359 for i in range(N_THREADS): 360 t = threading.Thread(target=run_thread) 361 threads.append(t) 362 with start_threads(threads, lambda: exit.append(1)): 363 time.sleep(1.0) 364 finally: 365 sys.setcheckinterval(old_checkinterval) 366 gc.collect() 367 self.assertEqual(len(C.inits), len(C.dels)) 368 369 def test_boom(self): 370 class Boom: 371 def __getattr__(self, someattribute): 372 del self.attr 373 raise AttributeError 374 375 a = Boom() 376 b = Boom() 377 a.attr = b 378 b.attr = a 379 380 gc.collect() 381 garbagelen = len(gc.garbage) 382 del a, b 383 # a<->b are in a trash cycle now. Collection will invoke 384 # Boom.__getattr__ (to see whether a and b have __del__ methods), and 385 # __getattr__ deletes the internal "attr" attributes as a side effect. 386 # That causes the trash cycle to get reclaimed via refcounts falling to 387 # 0, thus mutating the trash graph as a side effect of merely asking 388 # whether __del__ exists. This used to (before 2.3b1) crash Python. 389 # Now __getattr__ isn't called. 390 self.assertEqual(gc.collect(), 4) 391 self.assertEqual(len(gc.garbage), garbagelen) 392 393 def test_boom2(self): 394 class Boom2: 395 def __init__(self): 396 self.x = 0 397 398 def __getattr__(self, someattribute): 399 self.x += 1 400 if self.x > 1: 401 del self.attr 402 raise AttributeError 403 404 a = Boom2() 405 b = Boom2() 406 a.attr = b 407 b.attr = a 408 409 gc.collect() 410 garbagelen = len(gc.garbage) 411 del a, b 412 # Much like test_boom(), except that __getattr__ doesn't break the 413 # cycle until the second time gc checks for __del__. As of 2.3b1, 414 # there isn't a second time, so this simply cleans up the trash cycle. 415 # We expect a, b, a.__dict__ and b.__dict__ (4 objects) to get 416 # reclaimed this way. 417 self.assertEqual(gc.collect(), 4) 418 self.assertEqual(len(gc.garbage), garbagelen) 419 420 def test_boom_new(self): 421 # boom__new and boom2_new are exactly like boom and boom2, except use 422 # new-style classes. 423 424 class Boom_New(object): 425 def __getattr__(self, someattribute): 426 del self.attr 427 raise AttributeError 428 429 a = Boom_New() 430 b = Boom_New() 431 a.attr = b 432 b.attr = a 433 434 gc.collect() 435 garbagelen = len(gc.garbage) 436 del a, b 437 self.assertEqual(gc.collect(), 4) 438 self.assertEqual(len(gc.garbage), garbagelen) 439 440 def test_boom2_new(self): 441 class Boom2_New(object): 442 def __init__(self): 443 self.x = 0 444 445 def __getattr__(self, someattribute): 446 self.x += 1 447 if self.x > 1: 448 del self.attr 449 raise AttributeError 450 451 a = Boom2_New() 452 b = Boom2_New() 453 a.attr = b 454 b.attr = a 455 456 gc.collect() 457 garbagelen = len(gc.garbage) 458 del a, b 459 self.assertEqual(gc.collect(), 4) 460 self.assertEqual(len(gc.garbage), garbagelen) 461 462 def test_get_referents(self): 463 alist = [1, 3, 5] 464 got = gc.get_referents(alist) 465 got.sort() 466 self.assertEqual(got, alist) 467 468 atuple = tuple(alist) 469 got = gc.get_referents(atuple) 470 got.sort() 471 self.assertEqual(got, alist) 472 473 adict = {1: 3, 5: 7} 474 expected = [1, 3, 5, 7] 475 got = gc.get_referents(adict) 476 got.sort() 477 self.assertEqual(got, expected) 478 479 got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0)) 480 got.sort() 481 self.assertEqual(got, [0, 0] + range(5)) 482 483 self.assertEqual(gc.get_referents(1, 'a', 4j), []) 484 485 def test_is_tracked(self): 486 # Atomic built-in types are not tracked, user-defined objects and 487 # mutable containers are. 488 # NOTE: types with special optimizations (e.g. tuple) have tests 489 # in their own test files instead. 490 self.assertFalse(gc.is_tracked(None)) 491 self.assertFalse(gc.is_tracked(1)) 492 self.assertFalse(gc.is_tracked(1.0)) 493 self.assertFalse(gc.is_tracked(1.0 + 5.0j)) 494 self.assertFalse(gc.is_tracked(True)) 495 self.assertFalse(gc.is_tracked(False)) 496 self.assertFalse(gc.is_tracked("a")) 497 self.assertFalse(gc.is_tracked(u"a")) 498 self.assertFalse(gc.is_tracked(bytearray("a"))) 499 self.assertFalse(gc.is_tracked(type)) 500 self.assertFalse(gc.is_tracked(int)) 501 self.assertFalse(gc.is_tracked(object)) 502 self.assertFalse(gc.is_tracked(object())) 503 504 class OldStyle: 505 pass 506 class NewStyle(object): 507 pass 508 self.assertTrue(gc.is_tracked(gc)) 509 self.assertTrue(gc.is_tracked(OldStyle)) 510 self.assertTrue(gc.is_tracked(OldStyle())) 511 self.assertTrue(gc.is_tracked(NewStyle)) 512 self.assertTrue(gc.is_tracked(NewStyle())) 513 self.assertTrue(gc.is_tracked([])) 514 self.assertTrue(gc.is_tracked(set())) 515 516 def test_bug1055820b(self): 517 # Corresponds to temp2b.py in the bug report. 518 519 ouch = [] 520 def callback(ignored): 521 ouch[:] = [wr() for wr in WRs] 522 523 Cs = [C1055820(i) for i in range(2)] 524 WRs = [weakref.ref(c, callback) for c in Cs] 525 c = None 526 527 gc.collect() 528 self.assertEqual(len(ouch), 0) 529 # Make the two instances trash, and collect again. The bug was that 530 # the callback materialized a strong reference to an instance, but gc 531 # cleared the instance's dict anyway. 532 Cs = None 533 gc.collect() 534 self.assertEqual(len(ouch), 2) # else the callbacks didn't run 535 for x in ouch: 536 # If the callback resurrected one of these guys, the instance 537 # would be damaged, with an empty __dict__. 538 self.assertEqual(x, None) 539 540class GCTogglingTests(unittest.TestCase): 541 def setUp(self): 542 gc.enable() 543 544 def tearDown(self): 545 gc.disable() 546 547 def test_bug1055820c(self): 548 # Corresponds to temp2c.py in the bug report. This is pretty 549 # elaborate. 550 551 c0 = C1055820(0) 552 # Move c0 into generation 2. 553 gc.collect() 554 555 c1 = C1055820(1) 556 c1.keep_c0_alive = c0 557 del c0.loop # now only c1 keeps c0 alive 558 559 c2 = C1055820(2) 560 c2wr = weakref.ref(c2) # no callback! 561 562 ouch = [] 563 def callback(ignored): 564 ouch[:] = [c2wr()] 565 566 # The callback gets associated with a wr on an object in generation 2. 567 c0wr = weakref.ref(c0, callback) 568 569 c0 = c1 = c2 = None 570 571 # What we've set up: c0, c1, and c2 are all trash now. c0 is in 572 # generation 2. The only thing keeping it alive is that c1 points to 573 # it. c1 and c2 are in generation 0, and are in self-loops. There's a 574 # global weakref to c2 (c2wr), but that weakref has no callback. 575 # There's also a global weakref to c0 (c0wr), and that does have a 576 # callback, and that callback references c2 via c2wr(). 577 # 578 # c0 has a wr with callback, which references c2wr 579 # ^ 580 # | 581 # | Generation 2 above dots 582 #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . . 583 # | Generation 0 below dots 584 # | 585 # | 586 # ^->c1 ^->c2 has a wr but no callback 587 # | | | | 588 # <--v <--v 589 # 590 # So this is the nightmare: when generation 0 gets collected, we see 591 # that c2 has a callback-free weakref, and c1 doesn't even have a 592 # weakref. Collecting generation 0 doesn't see c0 at all, and c0 is 593 # the only object that has a weakref with a callback. gc clears c1 594 # and c2. Clearing c1 has the side effect of dropping the refcount on 595 # c0 to 0, so c0 goes away (despite that it's in an older generation) 596 # and c0's wr callback triggers. That in turn materializes a reference 597 # to c2 via c2wr(), but c2 gets cleared anyway by gc. 598 599 # We want to let gc happen "naturally", to preserve the distinction 600 # between generations. 601 junk = [] 602 i = 0 603 detector = GC_Detector() 604 while not detector.gc_happened: 605 i += 1 606 if i > 10000: 607 self.fail("gc didn't happen after 10000 iterations") 608 self.assertEqual(len(ouch), 0) 609 junk.append([]) # this will eventually trigger gc 610 611 self.assertEqual(len(ouch), 1) # else the callback wasn't invoked 612 for x in ouch: 613 # If the callback resurrected c2, the instance would be damaged, 614 # with an empty __dict__. 615 self.assertEqual(x, None) 616 617 def test_bug1055820d(self): 618 # Corresponds to temp2d.py in the bug report. This is very much like 619 # test_bug1055820c, but uses a __del__ method instead of a weakref 620 # callback to sneak in a resurrection of cyclic trash. 621 622 ouch = [] 623 class D(C1055820): 624 def __del__(self): 625 ouch[:] = [c2wr()] 626 627 d0 = D(0) 628 # Move all the above into generation 2. 629 gc.collect() 630 631 c1 = C1055820(1) 632 c1.keep_d0_alive = d0 633 del d0.loop # now only c1 keeps d0 alive 634 635 c2 = C1055820(2) 636 c2wr = weakref.ref(c2) # no callback! 637 638 d0 = c1 = c2 = None 639 640 # What we've set up: d0, c1, and c2 are all trash now. d0 is in 641 # generation 2. The only thing keeping it alive is that c1 points to 642 # it. c1 and c2 are in generation 0, and are in self-loops. There's 643 # a global weakref to c2 (c2wr), but that weakref has no callback. 644 # There are no other weakrefs. 645 # 646 # d0 has a __del__ method that references c2wr 647 # ^ 648 # | 649 # | Generation 2 above dots 650 #. . . . . . . .|. . . . . . . . . . . . . . . . . . . . . . . . 651 # | Generation 0 below dots 652 # | 653 # | 654 # ^->c1 ^->c2 has a wr but no callback 655 # | | | | 656 # <--v <--v 657 # 658 # So this is the nightmare: when generation 0 gets collected, we see 659 # that c2 has a callback-free weakref, and c1 doesn't even have a 660 # weakref. Collecting generation 0 doesn't see d0 at all. gc clears 661 # c1 and c2. Clearing c1 has the side effect of dropping the refcount 662 # on d0 to 0, so d0 goes away (despite that it's in an older 663 # generation) and d0's __del__ triggers. That in turn materializes 664 # a reference to c2 via c2wr(), but c2 gets cleared anyway by gc. 665 666 # We want to let gc happen "naturally", to preserve the distinction 667 # between generations. 668 detector = GC_Detector() 669 junk = [] 670 i = 0 671 while not detector.gc_happened: 672 i += 1 673 if i > 10000: 674 self.fail("gc didn't happen after 10000 iterations") 675 self.assertEqual(len(ouch), 0) 676 junk.append([]) # this will eventually trigger gc 677 678 self.assertEqual(len(ouch), 1) # else __del__ wasn't invoked 679 for x in ouch: 680 # If __del__ resurrected c2, the instance would be damaged, with an 681 # empty __dict__. 682 self.assertEqual(x, None) 683 684def test_main(): 685 enabled = gc.isenabled() 686 gc.disable() 687 assert not gc.isenabled() 688 debug = gc.get_debug() 689 gc.set_debug(debug & ~gc.DEBUG_LEAK) # this test is supposed to leak 690 691 try: 692 gc.collect() # Delete 2nd generation garbage 693 run_unittest(GCTests, GCTogglingTests) 694 finally: 695 gc.set_debug(debug) 696 # test gc.enable() even if GC is disabled by default 697 if verbose: 698 print "restoring automatic collection" 699 # make sure to always test gc.enable() 700 gc.enable() 701 assert gc.isenabled() 702 if not enabled: 703 gc.disable() 704 705if __name__ == "__main__": 706 test_main() 707