• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import errno
2import os
3import re
4import sys
5import warnings
6from inspect import isabstract
7from test import support
8try:
9    from _abc import _get_dump
10except ImportError:
11    def _get_dump(cls):
12        # For legacy Python version
13        return (cls._abc_registry, cls._abc_cache,
14                cls._abc_negative_cache, cls._abc_negative_cache_version)
15
16
17def dash_R(the_module, test, indirect_test, huntrleaks):
18    """Run a test multiple times, looking for reference leaks.
19
20    Returns:
21        False if the test didn't leak references; True if we detected refleaks.
22    """
23    # This code is hackish and inelegant, but it seems to do the job.
24    import copyreg
25    import collections.abc
26
27    if not hasattr(sys, 'gettotalrefcount'):
28        raise Exception("Tracking reference leaks requires a debug build "
29                        "of Python")
30
31    # Save current values for dash_R_cleanup() to restore.
32    fs = warnings.filters[:]
33    ps = copyreg.dispatch_table.copy()
34    pic = sys.path_importer_cache.copy()
35    try:
36        import zipimport
37    except ImportError:
38        zdc = None # Run unmodified on platforms without zipimport support
39    else:
40        zdc = zipimport._zip_directory_cache.copy()
41    abcs = {}
42    for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
43        if not isabstract(abc):
44            continue
45        for obj in abc.__subclasses__() + [abc]:
46            abcs[obj] = _get_dump(obj)[0]
47
48    # bpo-31217: Integer pool to get a single integer object for the same
49    # value. The pool is used to prevent false alarm when checking for memory
50    # block leaks. Fill the pool with values in -1000..1000 which are the most
51    # common (reference, memory block, file descriptor) differences.
52    int_pool = {value: value for value in range(-1000, 1000)}
53    def get_pooled_int(value):
54        return int_pool.setdefault(value, value)
55
56    nwarmup, ntracked, fname = huntrleaks
57    fname = os.path.join(support.SAVEDCWD, fname)
58    repcount = nwarmup + ntracked
59    rc_deltas = [0] * repcount
60    alloc_deltas = [0] * repcount
61    fd_deltas = [0] * repcount
62
63    print("beginning", repcount, "repetitions", file=sys.stderr)
64    print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
65          flush=True)
66    # initialize variables to make pyflakes quiet
67    rc_before = alloc_before = fd_before = 0
68    for i in range(repcount):
69        indirect_test()
70        alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc,
71                                                         abcs)
72        print('.', end='', file=sys.stderr, flush=True)
73        if i >= nwarmup:
74            rc_deltas[i] = get_pooled_int(rc_after - rc_before)
75            alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
76            fd_deltas[i] = get_pooled_int(fd_after - fd_before)
77        alloc_before = alloc_after
78        rc_before = rc_after
79        fd_before = fd_after
80    print(file=sys.stderr)
81
82    # These checkers return False on success, True on failure
83    def check_rc_deltas(deltas):
84        # Checker for reference counters and memomry blocks.
85        #
86        # bpo-30776: Try to ignore false positives:
87        #
88        #   [3, 0, 0]
89        #   [0, 1, 0]
90        #   [8, -8, 1]
91        #
92        # Expected leaks:
93        #
94        #   [5, 5, 6]
95        #   [10, 1, 1]
96        return all(delta >= 1 for delta in deltas)
97
98    def check_fd_deltas(deltas):
99        return any(deltas)
100
101    failed = False
102    for deltas, item_name, checker in [
103        (rc_deltas, 'references', check_rc_deltas),
104        (alloc_deltas, 'memory blocks', check_rc_deltas),
105        (fd_deltas, 'file descriptors', check_fd_deltas)
106    ]:
107        # ignore warmup runs
108        deltas = deltas[nwarmup:]
109        if checker(deltas):
110            msg = '%s leaked %s %s, sum=%s' % (
111                test, deltas, item_name, sum(deltas))
112            print(msg, file=sys.stderr, flush=True)
113            with open(fname, "a") as refrep:
114                print(msg, file=refrep)
115                refrep.flush()
116            failed = True
117    return failed
118
119
120def dash_R_cleanup(fs, ps, pic, zdc, abcs):
121    import gc, copyreg
122    import collections.abc
123
124    # Restore some original values.
125    warnings.filters[:] = fs
126    copyreg.dispatch_table.clear()
127    copyreg.dispatch_table.update(ps)
128    sys.path_importer_cache.clear()
129    sys.path_importer_cache.update(pic)
130    try:
131        import zipimport
132    except ImportError:
133        pass # Run unmodified on platforms without zipimport support
134    else:
135        zipimport._zip_directory_cache.clear()
136        zipimport._zip_directory_cache.update(zdc)
137
138    # clear type cache
139    sys._clear_type_cache()
140
141    # Clear ABC registries, restoring previously saved ABC registries.
142    abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
143    abs_classes = filter(isabstract, abs_classes)
144    for abc in abs_classes:
145        for obj in abc.__subclasses__() + [abc]:
146            for ref in abcs.get(obj, set()):
147                if ref() is not None:
148                    obj.register(ref())
149            obj._abc_caches_clear()
150
151    clear_caches()
152
153    # Collect cyclic trash and read memory statistics immediately after.
154    func1 = sys.getallocatedblocks
155    func2 = sys.gettotalrefcount
156    gc.collect()
157    return func1(), func2(), support.fd_count()
158
159
160def clear_caches():
161    import gc
162
163    # Clear the warnings registry, so they can be displayed again
164    for mod in sys.modules.values():
165        if hasattr(mod, '__warningregistry__'):
166            del mod.__warningregistry__
167
168    # Flush standard output, so that buffered data is sent to the OS and
169    # associated Python objects are reclaimed.
170    for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__):
171        if stream is not None:
172            stream.flush()
173
174    # Clear assorted module caches.
175    # Don't worry about resetting the cache if the module is not loaded
176    try:
177        distutils_dir_util = sys.modules['distutils.dir_util']
178    except KeyError:
179        pass
180    else:
181        distutils_dir_util._path_created.clear()
182    re.purge()
183
184    try:
185        _strptime = sys.modules['_strptime']
186    except KeyError:
187        pass
188    else:
189        _strptime._regex_cache.clear()
190
191    try:
192        urllib_parse = sys.modules['urllib.parse']
193    except KeyError:
194        pass
195    else:
196        urllib_parse.clear_cache()
197
198    try:
199        urllib_request = sys.modules['urllib.request']
200    except KeyError:
201        pass
202    else:
203        urllib_request.urlcleanup()
204
205    try:
206        linecache = sys.modules['linecache']
207    except KeyError:
208        pass
209    else:
210        linecache.clearcache()
211
212    try:
213        mimetypes = sys.modules['mimetypes']
214    except KeyError:
215        pass
216    else:
217        mimetypes._default_mime_types()
218
219    try:
220        filecmp = sys.modules['filecmp']
221    except KeyError:
222        pass
223    else:
224        filecmp._cache.clear()
225
226    try:
227        struct = sys.modules['struct']
228    except KeyError:
229        pass
230    else:
231        struct._clearcache()
232
233    try:
234        doctest = sys.modules['doctest']
235    except KeyError:
236        pass
237    else:
238        doctest.master = None
239
240    try:
241        ctypes = sys.modules['ctypes']
242    except KeyError:
243        pass
244    else:
245        ctypes._reset_cache()
246
247    try:
248        typing = sys.modules['typing']
249    except KeyError:
250        pass
251    else:
252        for f in typing._cleanups:
253            f()
254
255    gc.collect()
256
257
258def warm_caches():
259    # char cache
260    s = bytes(range(256))
261    for i in range(256):
262        s[i:i+1]
263    # unicode cache
264    [chr(i) for i in range(256)]
265    # int cache
266    list(range(-5, 257))
267