• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import os
2import re
3import sys
4import warnings
5from inspect import isabstract
6from test import support
7from test.support import os_helper
8from test.libregrtest.utils import clear_caches
9
10try:
11    from _abc import _get_dump
12except ImportError:
13    import weakref
14
15    def _get_dump(cls):
16        # Reimplement _get_dump() for pure-Python implementation of
17        # the abc module (Lib/_py_abc.py)
18        registry_weakrefs = set(weakref.ref(obj) for obj in cls._abc_registry)
19        return (registry_weakrefs, cls._abc_cache,
20                cls._abc_negative_cache, cls._abc_negative_cache_version)
21
22
23def dash_R(ns, test_name, test_func):
24    """Run a test multiple times, looking for reference leaks.
25
26    Returns:
27        False if the test didn't leak references; True if we detected refleaks.
28    """
29    # This code is hackish and inelegant, but it seems to do the job.
30    import copyreg
31    import collections.abc
32
33    if not hasattr(sys, 'gettotalrefcount'):
34        raise Exception("Tracking reference leaks requires a debug build "
35                        "of Python")
36
37    # Avoid false positives due to various caches
38    # filling slowly with random data:
39    warm_caches()
40
41    # Save current values for dash_R_cleanup() to restore.
42    fs = warnings.filters[:]
43    ps = copyreg.dispatch_table.copy()
44    pic = sys.path_importer_cache.copy()
45    try:
46        import zipimport
47    except ImportError:
48        zdc = None # Run unmodified on platforms without zipimport support
49    else:
50        zdc = zipimport._zip_directory_cache.copy()
51    abcs = {}
52    for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
53        if not isabstract(abc):
54            continue
55        for obj in abc.__subclasses__() + [abc]:
56            abcs[obj] = _get_dump(obj)[0]
57
58    # bpo-31217: Integer pool to get a single integer object for the same
59    # value. The pool is used to prevent false alarm when checking for memory
60    # block leaks. Fill the pool with values in -1000..1000 which are the most
61    # common (reference, memory block, file descriptor) differences.
62    int_pool = {value: value for value in range(-1000, 1000)}
63    def get_pooled_int(value):
64        return int_pool.setdefault(value, value)
65
66    nwarmup, ntracked, fname = ns.huntrleaks
67    fname = os.path.join(os_helper.SAVEDCWD, fname)
68    repcount = nwarmup + ntracked
69
70    # Pre-allocate to ensure that the loop doesn't allocate anything new
71    rep_range = list(range(repcount))
72    rc_deltas = [0] * repcount
73    alloc_deltas = [0] * repcount
74    fd_deltas = [0] * repcount
75    getallocatedblocks = sys.getallocatedblocks
76    gettotalrefcount = sys.gettotalrefcount
77    fd_count = os_helper.fd_count
78
79    # initialize variables to make pyflakes quiet
80    rc_before = alloc_before = fd_before = 0
81
82    if not ns.quiet:
83        print("beginning", repcount, "repetitions", file=sys.stderr)
84        print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
85              flush=True)
86
87    dash_R_cleanup(fs, ps, pic, zdc, abcs)
88
89    for i in rep_range:
90        test_func()
91        dash_R_cleanup(fs, ps, pic, zdc, abcs)
92
93        # dash_R_cleanup() ends with collecting cyclic trash:
94        # read memory statistics immediately after.
95        alloc_after = getallocatedblocks()
96        rc_after = gettotalrefcount()
97        fd_after = fd_count()
98
99        if not ns.quiet:
100            print('.', end='', file=sys.stderr, flush=True)
101
102        rc_deltas[i] = get_pooled_int(rc_after - rc_before)
103        alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
104        fd_deltas[i] = get_pooled_int(fd_after - fd_before)
105
106        alloc_before = alloc_after
107        rc_before = rc_after
108        fd_before = fd_after
109
110    if not ns.quiet:
111        print(file=sys.stderr)
112
113    # These checkers return False on success, True on failure
114    def check_rc_deltas(deltas):
115        # Checker for reference counters and memory blocks.
116        #
117        # bpo-30776: Try to ignore false positives:
118        #
119        #   [3, 0, 0]
120        #   [0, 1, 0]
121        #   [8, -8, 1]
122        #
123        # Expected leaks:
124        #
125        #   [5, 5, 6]
126        #   [10, 1, 1]
127        return all(delta >= 1 for delta in deltas)
128
129    def check_fd_deltas(deltas):
130        return any(deltas)
131
132    failed = False
133    for deltas, item_name, checker in [
134        (rc_deltas, 'references', check_rc_deltas),
135        (alloc_deltas, 'memory blocks', check_rc_deltas),
136        (fd_deltas, 'file descriptors', check_fd_deltas)
137    ]:
138        # ignore warmup runs
139        deltas = deltas[nwarmup:]
140        if checker(deltas):
141            msg = '%s leaked %s %s, sum=%s' % (
142                test_name, deltas, item_name, sum(deltas))
143            print(msg, file=sys.stderr, flush=True)
144            with open(fname, "a") as refrep:
145                print(msg, file=refrep)
146                refrep.flush()
147            failed = True
148    return failed
149
150
151def dash_R_cleanup(fs, ps, pic, zdc, abcs):
152    import copyreg
153    import collections.abc
154
155    # Restore some original values.
156    warnings.filters[:] = fs
157    copyreg.dispatch_table.clear()
158    copyreg.dispatch_table.update(ps)
159    sys.path_importer_cache.clear()
160    sys.path_importer_cache.update(pic)
161    try:
162        import zipimport
163    except ImportError:
164        pass # Run unmodified on platforms without zipimport support
165    else:
166        zipimport._zip_directory_cache.clear()
167        zipimport._zip_directory_cache.update(zdc)
168
169    # clear type cache
170    sys._clear_type_cache()
171
172    # Clear ABC registries, restoring previously saved ABC registries.
173    abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
174    abs_classes = filter(isabstract, abs_classes)
175    for abc in abs_classes:
176        for obj in abc.__subclasses__() + [abc]:
177            for ref in abcs.get(obj, set()):
178                if ref() is not None:
179                    obj.register(ref())
180            obj._abc_caches_clear()
181
182    clear_caches()
183
184
185def warm_caches():
186    # char cache
187    s = bytes(range(256))
188    for i in range(256):
189        s[i:i+1]
190    # unicode cache
191    [chr(i) for i in range(256)]
192    # int cache
193    list(range(-5, 257))
194