• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import os
2import sys
3import warnings
4from inspect import isabstract
5from typing import Any
6
7from test import support
8from test.support import os_helper
9from test.support import refleak_helper
10
11from .runtests import HuntRefleak
12from .utils import clear_caches
13
14try:
15    from _abc import _get_dump
16except ImportError:
17    import weakref
18
19    def _get_dump(cls):
20        # Reimplement _get_dump() for pure-Python implementation of
21        # the abc module (Lib/_py_abc.py)
22        registry_weakrefs = set(weakref.ref(obj) for obj in cls._abc_registry)
23        return (registry_weakrefs, cls._abc_cache,
24                cls._abc_negative_cache, cls._abc_negative_cache_version)
25
26
27def save_support_xml(filename):
28    if support.junit_xml_list is None:
29        return
30
31    import pickle
32    with open(filename, 'xb') as fp:
33        pickle.dump(support.junit_xml_list, fp)
34    support.junit_xml_list = None
35
36
37def restore_support_xml(filename):
38    try:
39        fp = open(filename, 'rb')
40    except FileNotFoundError:
41        return
42
43    import pickle
44    with fp:
45        xml_list = pickle.load(fp)
46    os.unlink(filename)
47
48    support.junit_xml_list = xml_list
49
50
51def runtest_refleak(test_name, test_func,
52                    hunt_refleak: HuntRefleak,
53                    quiet: bool):
54    """Run a test multiple times, looking for reference leaks.
55
56    Returns:
57        False if the test didn't leak references; True if we detected refleaks.
58    """
59    # This code is hackish and inelegant, but it seems to do the job.
60    import copyreg
61    import collections.abc
62
63    if not hasattr(sys, 'gettotalrefcount'):
64        raise Exception("Tracking reference leaks requires a debug build "
65                        "of Python")
66
67    # Avoid false positives due to various caches
68    # filling slowly with random data:
69    warm_caches()
70
71    # Save current values for dash_R_cleanup() to restore.
72    fs = warnings.filters[:]
73    ps = copyreg.dispatch_table.copy()
74    pic = sys.path_importer_cache.copy()
75    zdc: dict[str, Any] | None
76    try:
77        import zipimport
78    except ImportError:
79        zdc = None # Run unmodified on platforms without zipimport support
80    else:
81        # private attribute that mypy doesn't know about:
82        zdc = zipimport._zip_directory_cache.copy()  # type: ignore[attr-defined]
83    abcs = {}
84    for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
85        if not isabstract(abc):
86            continue
87        for obj in abc.__subclasses__() + [abc]:
88            abcs[obj] = _get_dump(obj)[0]
89
90    # bpo-31217: Integer pool to get a single integer object for the same
91    # value. The pool is used to prevent false alarm when checking for memory
92    # block leaks. Fill the pool with values in -1000..1000 which are the most
93    # common (reference, memory block, file descriptor) differences.
94    int_pool = {value: value for value in range(-1000, 1000)}
95    def get_pooled_int(value):
96        return int_pool.setdefault(value, value)
97
98    warmups = hunt_refleak.warmups
99    runs = hunt_refleak.runs
100    filename = hunt_refleak.filename
101    repcount = warmups + runs
102
103    # Pre-allocate to ensure that the loop doesn't allocate anything new
104    rep_range = list(range(repcount))
105    rc_deltas = [0] * repcount
106    alloc_deltas = [0] * repcount
107    fd_deltas = [0] * repcount
108    getallocatedblocks = sys.getallocatedblocks
109    gettotalrefcount = sys.gettotalrefcount
110    getunicodeinternedsize = sys.getunicodeinternedsize
111    fd_count = os_helper.fd_count
112    # initialize variables to make pyflakes quiet
113    rc_before = alloc_before = fd_before = interned_immortal_before = 0
114
115    if not quiet:
116        print("beginning", repcount, "repetitions. Showing number of leaks "
117                "(. for 0 or less, X for 10 or more)",
118              file=sys.stderr)
119        numbers = ("1234567890"*(repcount//10 + 1))[:repcount]
120        numbers = numbers[:warmups] + ':' + numbers[warmups:]
121        print(numbers, file=sys.stderr, flush=True)
122
123    xml_filename = 'refleak-xml.tmp'
124    result = None
125    dash_R_cleanup(fs, ps, pic, zdc, abcs)
126    support.gc_collect()
127
128    for i in rep_range:
129        current = refleak_helper._hunting_for_refleaks
130        refleak_helper._hunting_for_refleaks = True
131        try:
132            result = test_func()
133        finally:
134            refleak_helper._hunting_for_refleaks = current
135
136        save_support_xml(xml_filename)
137        dash_R_cleanup(fs, ps, pic, zdc, abcs)
138        support.gc_collect()
139
140        # Read memory statistics immediately after the garbage collection.
141        # Also, readjust the reference counts and alloc blocks by ignoring
142        # any strings that might have been interned during test_func. These
143        # strings will be deallocated at runtime shutdown
144        interned_immortal_after = getunicodeinternedsize(
145            # Use an internal-only keyword argument that mypy doesn't know yet
146            _only_immortal=True)  # type: ignore[call-arg]
147        alloc_after = getallocatedblocks() - interned_immortal_after
148        rc_after = gettotalrefcount()
149        fd_after = fd_count()
150
151        rc_deltas[i] = get_pooled_int(rc_after - rc_before)
152        alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
153        fd_deltas[i] = get_pooled_int(fd_after - fd_before)
154
155        if not quiet:
156            # use max, not sum, so total_leaks is one of the pooled ints
157            total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i])
158            if total_leaks <= 0:
159                symbol = '.'
160            elif total_leaks < 10:
161                symbol = (
162                    '.', '1', '2', '3', '4', '5', '6', '7', '8', '9',
163                    )[total_leaks]
164            else:
165                symbol = 'X'
166            if i == warmups:
167                print(' ', end='', file=sys.stderr, flush=True)
168            print(symbol, end='', file=sys.stderr, flush=True)
169            del total_leaks
170            del symbol
171
172        alloc_before = alloc_after
173        rc_before = rc_after
174        fd_before = fd_after
175        interned_immortal_before = interned_immortal_after
176
177        restore_support_xml(xml_filename)
178
179    if not quiet:
180        print(file=sys.stderr)
181
182    # These checkers return False on success, True on failure
183    def check_rc_deltas(deltas):
184        # Checker for reference counters and memory blocks.
185        #
186        # bpo-30776: Try to ignore false positives:
187        #
188        #   [3, 0, 0]
189        #   [0, 1, 0]
190        #   [8, -8, 1]
191        #
192        # Expected leaks:
193        #
194        #   [5, 5, 6]
195        #   [10, 1, 1]
196        return all(delta >= 1 for delta in deltas)
197
198    def check_fd_deltas(deltas):
199        return any(deltas)
200
201    failed = False
202    for deltas, item_name, checker in [
203        (rc_deltas, 'references', check_rc_deltas),
204        (alloc_deltas, 'memory blocks', check_rc_deltas),
205        (fd_deltas, 'file descriptors', check_fd_deltas)
206    ]:
207        # ignore warmup runs
208        deltas = deltas[warmups:]
209        failing = checker(deltas)
210        suspicious = any(deltas)
211        if failing or suspicious:
212            msg = '%s leaked %s %s, sum=%s' % (
213                test_name, deltas, item_name, sum(deltas))
214            print(msg, end='', file=sys.stderr)
215            if failing:
216                print(file=sys.stderr, flush=True)
217                with open(filename, "a", encoding="utf-8") as refrep:
218                    print(msg, file=refrep)
219                    refrep.flush()
220                failed = True
221            else:
222                print(' (this is fine)', file=sys.stderr, flush=True)
223    return (failed, result)
224
225
226def dash_R_cleanup(fs, ps, pic, zdc, abcs):
227    import copyreg
228    import collections.abc
229
230    # Restore some original values.
231    warnings.filters[:] = fs
232    copyreg.dispatch_table.clear()
233    copyreg.dispatch_table.update(ps)
234    sys.path_importer_cache.clear()
235    sys.path_importer_cache.update(pic)
236    try:
237        import zipimport
238    except ImportError:
239        pass # Run unmodified on platforms without zipimport support
240    else:
241        zipimport._zip_directory_cache.clear()
242        zipimport._zip_directory_cache.update(zdc)
243
244    # Clear ABC registries, restoring previously saved ABC registries.
245    # ignore deprecation warning for collections.abc.ByteString
246    abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
247    abs_classes = filter(isabstract, abs_classes)
248    for abc in abs_classes:
249        for obj in abc.__subclasses__() + [abc]:
250            refs = abcs.get(obj, None)
251            if refs is not None:
252                obj._abc_registry_clear()
253                for ref in refs:
254                    subclass = ref()
255                    if subclass is not None:
256                        obj.register(subclass)
257            obj._abc_caches_clear()
258
259    # Clear caches
260    clear_caches()
261
262    # Clear other caches last (previous function calls can re-populate them):
263    sys._clear_internal_caches()
264
265
266def warm_caches():
267    # char cache
268    s = bytes(range(256))
269    for i in range(256):
270        s[i:i+1]
271    # unicode cache
272    [chr(i) for i in range(256)]
273    # int cache
274    list(range(-5, 257))
275