1 import sys
2 import warnings
3 from inspect import isabstract
4 from typing import Any
5
6 from test import support
7 from test.support import os_helper
8
9 from .runtests import HuntRefleak
10 from .utils import clear_caches
11
12 try:
13 from _abc import _get_dump
14 except ImportError:
15 import weakref
16
17 def _get_dump(cls):
18 # Reimplement _get_dump() for pure-Python implementation of
19 # the abc module (Lib/_py_abc.py)
20 registry_weakrefs = set(weakref.ref(obj) for obj in cls._abc_registry)
21 return (registry_weakrefs, cls._abc_cache,
22 cls._abc_negative_cache, cls._abc_negative_cache_version)
23
24
25 def runtest_refleak(test_name, test_func,
26 hunt_refleak: HuntRefleak,
27 quiet: bool):
28 """Run a test multiple times, looking for reference leaks.
29
30 Returns:
31 False if the test didn't leak references; True if we detected refleaks.
32 """
33 # This code is hackish and inelegant, but it seems to do the job.
34 import copyreg
35 import collections.abc
36
37 if not hasattr(sys, 'gettotalrefcount'):
38 raise Exception("Tracking reference leaks requires a debug build "
39 "of Python")
40
41 # Avoid false positives due to various caches
42 # filling slowly with random data:
43 warm_caches()
44
45 # Save current values for dash_R_cleanup() to restore.
46 fs = warnings.filters[:]
47 ps = copyreg.dispatch_table.copy()
48 pic = sys.path_importer_cache.copy()
49 zdc: dict[str, Any] | None
50 try:
51 import zipimport
52 except ImportError:
53 zdc = None # Run unmodified on platforms without zipimport support
54 else:
55 # private attribute that mypy doesn't know about:
56 zdc = zipimport._zip_directory_cache.copy() # type: ignore[attr-defined]
57 abcs = {}
58 for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
59 if not isabstract(abc):
60 continue
61 for obj in abc.__subclasses__() + [abc]:
62 abcs[obj] = _get_dump(obj)[0]
63
64 # bpo-31217: Integer pool to get a single integer object for the same
65 # value. The pool is used to prevent false alarm when checking for memory
66 # block leaks. Fill the pool with values in -1000..1000 which are the most
67 # common (reference, memory block, file descriptor) differences.
68 int_pool = {value: value for value in range(-1000, 1000)}
69 def get_pooled_int(value):
70 return int_pool.setdefault(value, value)
71
72 warmups = hunt_refleak.warmups
73 runs = hunt_refleak.runs
74 filename = hunt_refleak.filename
75 repcount = warmups + runs
76
77 # Pre-allocate to ensure that the loop doesn't allocate anything new
78 rep_range = list(range(repcount))
79 rc_deltas = [0] * repcount
80 alloc_deltas = [0] * repcount
81 fd_deltas = [0] * repcount
82 getallocatedblocks = sys.getallocatedblocks
83 gettotalrefcount = sys.gettotalrefcount
84 fd_count = os_helper.fd_count
85 # initialize variables to make pyflakes quiet
86 rc_before = alloc_before = fd_before = 0
87
88 if not quiet:
89 print("beginning", repcount, "repetitions", file=sys.stderr)
90 print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
91 flush=True)
92
93 results = None
94 dash_R_cleanup(fs, ps, pic, zdc, abcs)
95 support.gc_collect()
96
97 for i in rep_range:
98 results = test_func()
99
100 dash_R_cleanup(fs, ps, pic, zdc, abcs)
101 support.gc_collect()
102
103 # Read memory statistics immediately after the garbage collection.
104 alloc_after = getallocatedblocks()
105 rc_after = gettotalrefcount()
106 fd_after = fd_count()
107
108 if not quiet:
109 print('.', end='', file=sys.stderr, flush=True)
110
111 rc_deltas[i] = get_pooled_int(rc_after - rc_before)
112 alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
113 fd_deltas[i] = get_pooled_int(fd_after - fd_before)
114
115 alloc_before = alloc_after
116 rc_before = rc_after
117 fd_before = fd_after
118
119 if not quiet:
120 print(file=sys.stderr)
121
122 # These checkers return False on success, True on failure
123 def check_rc_deltas(deltas):
124 # Checker for reference counters and memory blocks.
125 #
126 # bpo-30776: Try to ignore false positives:
127 #
128 # [3, 0, 0]
129 # [0, 1, 0]
130 # [8, -8, 1]
131 #
132 # Expected leaks:
133 #
134 # [5, 5, 6]
135 # [10, 1, 1]
136 return all(delta >= 1 for delta in deltas)
137
138 def check_fd_deltas(deltas):
139 return any(deltas)
140
141 failed = False
142 for deltas, item_name, checker in [
143 (rc_deltas, 'references', check_rc_deltas),
144 (alloc_deltas, 'memory blocks', check_rc_deltas),
145 (fd_deltas, 'file descriptors', check_fd_deltas)
146 ]:
147 # ignore warmup runs
148 deltas = deltas[warmups:]
149 if checker(deltas):
150 msg = '%s leaked %s %s, sum=%s' % (
151 test_name, deltas, item_name, sum(deltas))
152 print(msg, file=sys.stderr, flush=True)
153 with open(filename, "a", encoding="utf-8") as refrep:
154 print(msg, file=refrep)
155 refrep.flush()
156 failed = True
157 return (failed, results)
158
159
160 def dash_R_cleanup(fs, ps, pic, zdc, abcs):
161 import copyreg
162 import collections.abc
163
164 # Restore some original values.
165 warnings.filters[:] = fs
166 copyreg.dispatch_table.clear()
167 copyreg.dispatch_table.update(ps)
168 sys.path_importer_cache.clear()
169 sys.path_importer_cache.update(pic)
170 try:
171 import zipimport
172 except ImportError:
173 pass # Run unmodified on platforms without zipimport support
174 else:
175 zipimport._zip_directory_cache.clear()
176 zipimport._zip_directory_cache.update(zdc)
177
178 # Clear ABC registries, restoring previously saved ABC registries.
179 # ignore deprecation warning for collections.abc.ByteString
180 abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
181 abs_classes = filter(isabstract, abs_classes)
182 for abc in abs_classes:
183 for obj in abc.__subclasses__() + [abc]:
184 for ref in abcs.get(obj, set()):
185 if ref() is not None:
186 obj.register(ref())
187 obj._abc_caches_clear()
188
189 # Clear caches
190 clear_caches()
191
192 # Clear type cache at the end: previous function calls can modify types
193 sys._clear_type_cache()
194
195
196 def warm_caches():
197 # char cache
198 s = bytes(range(256))
199 for i in range(256):
200 s[i:i+1]
201 # unicode cache
202 [chr(i) for i in range(256)]
203 # int cache
204 list(range(-5, 257))