1 import os
2 import sys
3 import warnings
4 from inspect import isabstract
5 from test import support
6 from test.support import os_helper
7 from test.libregrtest.utils import clear_caches
8
9 try:
10 from _abc import _get_dump
11 except ImportError:
12 import weakref
13
14 def _get_dump(cls):
15 # Reimplement _get_dump() for pure-Python implementation of
16 # the abc module (Lib/_py_abc.py)
17 registry_weakrefs = set(weakref.ref(obj) for obj in cls._abc_registry)
18 return (registry_weakrefs, cls._abc_cache,
19 cls._abc_negative_cache, cls._abc_negative_cache_version)
20
21
22 def dash_R(ns, test_name, test_func):
23 """Run a test multiple times, looking for reference leaks.
24
25 Returns:
26 False if the test didn't leak references; True if we detected refleaks.
27 """
28 # This code is hackish and inelegant, but it seems to do the job.
29 import copyreg
30 import collections.abc
31
32 if not hasattr(sys, 'gettotalrefcount'):
33 raise Exception("Tracking reference leaks requires a debug build "
34 "of Python")
35
36 # Avoid false positives due to various caches
37 # filling slowly with random data:
38 warm_caches()
39
40 # Save current values for dash_R_cleanup() to restore.
41 fs = warnings.filters[:]
42 ps = copyreg.dispatch_table.copy()
43 pic = sys.path_importer_cache.copy()
44 try:
45 import zipimport
46 except ImportError:
47 zdc = None # Run unmodified on platforms without zipimport support
48 else:
49 zdc = zipimport._zip_directory_cache.copy()
50 abcs = {}
51 for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
52 if not isabstract(abc):
53 continue
54 for obj in abc.__subclasses__() + [abc]:
55 abcs[obj] = _get_dump(obj)[0]
56
57 # bpo-31217: Integer pool to get a single integer object for the same
58 # value. The pool is used to prevent false alarm when checking for memory
59 # block leaks. Fill the pool with values in -1000..1000 which are the most
60 # common (reference, memory block, file descriptor) differences.
61 int_pool = {value: value for value in range(-1000, 1000)}
62 def get_pooled_int(value):
63 return int_pool.setdefault(value, value)
64
65 nwarmup, ntracked, fname = ns.huntrleaks
66 fname = os.path.join(os_helper.SAVEDCWD, fname)
67 repcount = nwarmup + ntracked
68
69 # Pre-allocate to ensure that the loop doesn't allocate anything new
70 rep_range = list(range(repcount))
71 rc_deltas = [0] * repcount
72 alloc_deltas = [0] * repcount
73 fd_deltas = [0] * repcount
74 getallocatedblocks = sys.getallocatedblocks
75 gettotalrefcount = sys.gettotalrefcount
76 getunicodeinternedsize = sys.getunicodeinternedsize
77 fd_count = os_helper.fd_count
78 # initialize variables to make pyflakes quiet
79 rc_before = alloc_before = fd_before = interned_before = 0
80
81 if not ns.quiet:
82 print("beginning", repcount, "repetitions", file=sys.stderr)
83 print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
84 flush=True)
85
86 results = None
87 dash_R_cleanup(fs, ps, pic, zdc, abcs)
88 support.gc_collect()
89
90 for i in rep_range:
91 results = test_func()
92
93 dash_R_cleanup(fs, ps, pic, zdc, abcs)
94 support.gc_collect()
95
96 # Read memory statistics immediately after the garbage collection.
97 # Also, readjust the reference counts and alloc blocks by ignoring
98 # any strings that might have been interned during test_func. These
99 # strings will be deallocated at runtime shutdown
100 interned_after = getunicodeinternedsize()
101 alloc_after = getallocatedblocks() - interned_after
102 rc_after = gettotalrefcount() - interned_after * 2
103 fd_after = fd_count()
104
105 if not ns.quiet:
106 print('.', end='', file=sys.stderr, flush=True)
107
108 rc_deltas[i] = get_pooled_int(rc_after - rc_before)
109 alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
110 fd_deltas[i] = get_pooled_int(fd_after - fd_before)
111
112 alloc_before = alloc_after
113 rc_before = rc_after
114 fd_before = fd_after
115 interned_before = interned_after
116
117 if not ns.quiet:
118 print(file=sys.stderr)
119
120 # These checkers return False on success, True on failure
121 def check_rc_deltas(deltas):
122 # Checker for reference counters and memory blocks.
123 #
124 # bpo-30776: Try to ignore false positives:
125 #
126 # [3, 0, 0]
127 # [0, 1, 0]
128 # [8, -8, 1]
129 #
130 # Expected leaks:
131 #
132 # [5, 5, 6]
133 # [10, 1, 1]
134 return all(delta >= 1 for delta in deltas)
135
136 def check_fd_deltas(deltas):
137 return any(deltas)
138
139 failed = False
140 for deltas, item_name, checker in [
141 (rc_deltas, 'references', check_rc_deltas),
142 (alloc_deltas, 'memory blocks', check_rc_deltas),
143 (fd_deltas, 'file descriptors', check_fd_deltas)
144 ]:
145 # ignore warmup runs
146 deltas = deltas[nwarmup:]
147 if checker(deltas):
148 msg = '%s leaked %s %s, sum=%s' % (
149 test_name, deltas, item_name, sum(deltas))
150 print(msg, file=sys.stderr, flush=True)
151 with open(fname, "a", encoding="utf-8") as refrep:
152 print(msg, file=refrep)
153 refrep.flush()
154 failed = True
155 return (failed, results)
156
157
158 def dash_R_cleanup(fs, ps, pic, zdc, abcs):
159 import copyreg
160 import collections.abc
161
162 # Restore some original values.
163 warnings.filters[:] = fs
164 copyreg.dispatch_table.clear()
165 copyreg.dispatch_table.update(ps)
166 sys.path_importer_cache.clear()
167 sys.path_importer_cache.update(pic)
168 try:
169 import zipimport
170 except ImportError:
171 pass # Run unmodified on platforms without zipimport support
172 else:
173 zipimport._zip_directory_cache.clear()
174 zipimport._zip_directory_cache.update(zdc)
175
176 # Clear ABC registries, restoring previously saved ABC registries.
177 # ignore deprecation warning for collections.abc.ByteString
178 abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
179 abs_classes = filter(isabstract, abs_classes)
180 for abc in abs_classes:
181 for obj in abc.__subclasses__() + [abc]:
182 for ref in abcs.get(obj, set()):
183 if ref() is not None:
184 obj.register(ref())
185 obj._abc_caches_clear()
186
187 # Clear caches
188 clear_caches()
189
190 # Clear type cache at the end: previous function calls can modify types
191 sys._clear_type_cache()
192
193
194 def warm_caches():
195 # char cache
196 s = bytes(range(256))
197 for i in range(256):
198 s[i:i+1]
199 # unicode cache
200 [chr(i) for i in range(256)]
201 # int cache
202 list(range(-5, 257))