• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import os
2import re
3import sys
4import warnings
5from inspect import isabstract
6from test import support
7try:
8    from _abc import _get_dump
9except ImportError:
10    import weakref
11
12    def _get_dump(cls):
13        # Reimplement _get_dump() for pure-Python implementation of
14        # the abc module (Lib/_py_abc.py)
15        registry_weakrefs = set(weakref.ref(obj) for obj in cls._abc_registry)
16        return (registry_weakrefs, cls._abc_cache,
17                cls._abc_negative_cache, cls._abc_negative_cache_version)
18
19
20def dash_R(ns, test_name, test_func):
21    """Run a test multiple times, looking for reference leaks.
22
23    Returns:
24        False if the test didn't leak references; True if we detected refleaks.
25    """
26    # This code is hackish and inelegant, but it seems to do the job.
27    import copyreg
28    import collections.abc
29
30    if not hasattr(sys, 'gettotalrefcount'):
31        raise Exception("Tracking reference leaks requires a debug build "
32                        "of Python")
33
34    # Avoid false positives due to various caches
35    # filling slowly with random data:
36    warm_caches()
37
38    # Save current values for dash_R_cleanup() to restore.
39    fs = warnings.filters[:]
40    ps = copyreg.dispatch_table.copy()
41    pic = sys.path_importer_cache.copy()
42    try:
43        import zipimport
44    except ImportError:
45        zdc = None # Run unmodified on platforms without zipimport support
46    else:
47        zdc = zipimport._zip_directory_cache.copy()
48    abcs = {}
49    for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
50        if not isabstract(abc):
51            continue
52        for obj in abc.__subclasses__() + [abc]:
53            abcs[obj] = _get_dump(obj)[0]
54
55    # bpo-31217: Integer pool to get a single integer object for the same
56    # value. The pool is used to prevent false alarm when checking for memory
57    # block leaks. Fill the pool with values in -1000..1000 which are the most
58    # common (reference, memory block, file descriptor) differences.
59    int_pool = {value: value for value in range(-1000, 1000)}
60    def get_pooled_int(value):
61        return int_pool.setdefault(value, value)
62
63    nwarmup, ntracked, fname = ns.huntrleaks
64    fname = os.path.join(support.SAVEDCWD, fname)
65    repcount = nwarmup + ntracked
66
67    # Pre-allocate to ensure that the loop doesn't allocate anything new
68    rep_range = list(range(repcount))
69    rc_deltas = [0] * repcount
70    alloc_deltas = [0] * repcount
71    fd_deltas = [0] * repcount
72    getallocatedblocks = sys.getallocatedblocks
73    gettotalrefcount = sys.gettotalrefcount
74    fd_count = support.fd_count
75
76    # initialize variables to make pyflakes quiet
77    rc_before = alloc_before = fd_before = 0
78
79    if not ns.quiet:
80        print("beginning", repcount, "repetitions", file=sys.stderr)
81        print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
82              flush=True)
83
84    dash_R_cleanup(fs, ps, pic, zdc, abcs)
85
86    for i in rep_range:
87        test_func()
88        dash_R_cleanup(fs, ps, pic, zdc, abcs)
89
90        # dash_R_cleanup() ends with collecting cyclic trash:
91        # read memory statistics immediately after.
92        alloc_after = getallocatedblocks()
93        rc_after = gettotalrefcount()
94        fd_after = fd_count()
95
96        if not ns.quiet:
97            print('.', end='', file=sys.stderr, flush=True)
98
99        rc_deltas[i] = get_pooled_int(rc_after - rc_before)
100        alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
101        fd_deltas[i] = get_pooled_int(fd_after - fd_before)
102
103        alloc_before = alloc_after
104        rc_before = rc_after
105        fd_before = fd_after
106
107    if not ns.quiet:
108        print(file=sys.stderr)
109
110    # These checkers return False on success, True on failure
111    def check_rc_deltas(deltas):
112        # Checker for reference counters and memomry blocks.
113        #
114        # bpo-30776: Try to ignore false positives:
115        #
116        #   [3, 0, 0]
117        #   [0, 1, 0]
118        #   [8, -8, 1]
119        #
120        # Expected leaks:
121        #
122        #   [5, 5, 6]
123        #   [10, 1, 1]
124        return all(delta >= 1 for delta in deltas)
125
126    def check_fd_deltas(deltas):
127        return any(deltas)
128
129    failed = False
130    for deltas, item_name, checker in [
131        (rc_deltas, 'references', check_rc_deltas),
132        (alloc_deltas, 'memory blocks', check_rc_deltas),
133        (fd_deltas, 'file descriptors', check_fd_deltas)
134    ]:
135        # ignore warmup runs
136        deltas = deltas[nwarmup:]
137        if checker(deltas):
138            msg = '%s leaked %s %s, sum=%s' % (
139                test_name, deltas, item_name, sum(deltas))
140            print(msg, file=sys.stderr, flush=True)
141            with open(fname, "a") as refrep:
142                print(msg, file=refrep)
143                refrep.flush()
144            failed = True
145    return failed
146
147
148def dash_R_cleanup(fs, ps, pic, zdc, abcs):
149    import copyreg
150    import collections.abc
151
152    # Restore some original values.
153    warnings.filters[:] = fs
154    copyreg.dispatch_table.clear()
155    copyreg.dispatch_table.update(ps)
156    sys.path_importer_cache.clear()
157    sys.path_importer_cache.update(pic)
158    try:
159        import zipimport
160    except ImportError:
161        pass # Run unmodified on platforms without zipimport support
162    else:
163        zipimport._zip_directory_cache.clear()
164        zipimport._zip_directory_cache.update(zdc)
165
166    # clear type cache
167    sys._clear_type_cache()
168
169    # Clear ABC registries, restoring previously saved ABC registries.
170    abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
171    abs_classes = filter(isabstract, abs_classes)
172    for abc in abs_classes:
173        for obj in abc.__subclasses__() + [abc]:
174            for ref in abcs.get(obj, set()):
175                if ref() is not None:
176                    obj.register(ref())
177            obj._abc_caches_clear()
178
179    clear_caches()
180
181
182def clear_caches():
183    # Clear the warnings registry, so they can be displayed again
184    for mod in sys.modules.values():
185        if hasattr(mod, '__warningregistry__'):
186            del mod.__warningregistry__
187
188    # Flush standard output, so that buffered data is sent to the OS and
189    # associated Python objects are reclaimed.
190    for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__):
191        if stream is not None:
192            stream.flush()
193
194    # Clear assorted module caches.
195    # Don't worry about resetting the cache if the module is not loaded
196    try:
197        distutils_dir_util = sys.modules['distutils.dir_util']
198    except KeyError:
199        pass
200    else:
201        distutils_dir_util._path_created.clear()
202    re.purge()
203
204    try:
205        _strptime = sys.modules['_strptime']
206    except KeyError:
207        pass
208    else:
209        _strptime._regex_cache.clear()
210
211    try:
212        urllib_parse = sys.modules['urllib.parse']
213    except KeyError:
214        pass
215    else:
216        urllib_parse.clear_cache()
217
218    try:
219        urllib_request = sys.modules['urllib.request']
220    except KeyError:
221        pass
222    else:
223        urllib_request.urlcleanup()
224
225    try:
226        linecache = sys.modules['linecache']
227    except KeyError:
228        pass
229    else:
230        linecache.clearcache()
231
232    try:
233        mimetypes = sys.modules['mimetypes']
234    except KeyError:
235        pass
236    else:
237        mimetypes._default_mime_types()
238
239    try:
240        filecmp = sys.modules['filecmp']
241    except KeyError:
242        pass
243    else:
244        filecmp._cache.clear()
245
246    try:
247        struct = sys.modules['struct']
248    except KeyError:
249        pass
250    else:
251        struct._clearcache()
252
253    try:
254        doctest = sys.modules['doctest']
255    except KeyError:
256        pass
257    else:
258        doctest.master = None
259
260    try:
261        ctypes = sys.modules['ctypes']
262    except KeyError:
263        pass
264    else:
265        ctypes._reset_cache()
266
267    try:
268        typing = sys.modules['typing']
269    except KeyError:
270        pass
271    else:
272        for f in typing._cleanups:
273            f()
274
275    support.gc_collect()
276
277
278def warm_caches():
279    # char cache
280    s = bytes(range(256))
281    for i in range(256):
282        s[i:i+1]
283    # unicode cache
284    [chr(i) for i in range(256)]
285    # int cache
286    list(range(-5, 257))
287