1 from collections.abc import Sequence, Iterable
2 from functools import total_ordering
3 import fnmatch
4 import linecache
5 import os.path
6 import pickle
7
8 # Import types and functions implemented in C
9 from _tracemalloc import *
10 from _tracemalloc import _get_object_traceback, _get_traces
11
12
13 def _format_size(size, sign):
14 for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'):
15 if abs(size) < 100 and unit != 'B':
16 # 3 digits (xx.x UNIT)
17 if sign:
18 return "%+.1f %s" % (size, unit)
19 else:
20 return "%.1f %s" % (size, unit)
21 if abs(size) < 10 * 1024 or unit == 'TiB':
22 # 4 or 5 digits (xxxx UNIT)
23 if sign:
24 return "%+.0f %s" % (size, unit)
25 else:
26 return "%.0f %s" % (size, unit)
27 size /= 1024
28
29
30 class ESC[4;38;5;81mStatistic:
31 """
32 Statistic difference on memory allocations between two Snapshot instance.
33 """
34
35 __slots__ = ('traceback', 'size', 'count')
36
37 def __init__(self, traceback, size, count):
38 self.traceback = traceback
39 self.size = size
40 self.count = count
41
42 def __hash__(self):
43 return hash((self.traceback, self.size, self.count))
44
45 def __eq__(self, other):
46 if not isinstance(other, Statistic):
47 return NotImplemented
48 return (self.traceback == other.traceback
49 and self.size == other.size
50 and self.count == other.count)
51
52 def __str__(self):
53 text = ("%s: size=%s, count=%i"
54 % (self.traceback,
55 _format_size(self.size, False),
56 self.count))
57 if self.count:
58 average = self.size / self.count
59 text += ", average=%s" % _format_size(average, False)
60 return text
61
62 def __repr__(self):
63 return ('<Statistic traceback=%r size=%i count=%i>'
64 % (self.traceback, self.size, self.count))
65
66 def _sort_key(self):
67 return (self.size, self.count, self.traceback)
68
69
70 class ESC[4;38;5;81mStatisticDiff:
71 """
72 Statistic difference on memory allocations between an old and a new
73 Snapshot instance.
74 """
75 __slots__ = ('traceback', 'size', 'size_diff', 'count', 'count_diff')
76
77 def __init__(self, traceback, size, size_diff, count, count_diff):
78 self.traceback = traceback
79 self.size = size
80 self.size_diff = size_diff
81 self.count = count
82 self.count_diff = count_diff
83
84 def __hash__(self):
85 return hash((self.traceback, self.size, self.size_diff,
86 self.count, self.count_diff))
87
88 def __eq__(self, other):
89 if not isinstance(other, StatisticDiff):
90 return NotImplemented
91 return (self.traceback == other.traceback
92 and self.size == other.size
93 and self.size_diff == other.size_diff
94 and self.count == other.count
95 and self.count_diff == other.count_diff)
96
97 def __str__(self):
98 text = ("%s: size=%s (%s), count=%i (%+i)"
99 % (self.traceback,
100 _format_size(self.size, False),
101 _format_size(self.size_diff, True),
102 self.count,
103 self.count_diff))
104 if self.count:
105 average = self.size / self.count
106 text += ", average=%s" % _format_size(average, False)
107 return text
108
109 def __repr__(self):
110 return ('<StatisticDiff traceback=%r size=%i (%+i) count=%i (%+i)>'
111 % (self.traceback, self.size, self.size_diff,
112 self.count, self.count_diff))
113
114 def _sort_key(self):
115 return (abs(self.size_diff), self.size,
116 abs(self.count_diff), self.count,
117 self.traceback)
118
119
120 def _compare_grouped_stats(old_group, new_group):
121 statistics = []
122 for traceback, stat in new_group.items():
123 previous = old_group.pop(traceback, None)
124 if previous is not None:
125 stat = StatisticDiff(traceback,
126 stat.size, stat.size - previous.size,
127 stat.count, stat.count - previous.count)
128 else:
129 stat = StatisticDiff(traceback,
130 stat.size, stat.size,
131 stat.count, stat.count)
132 statistics.append(stat)
133
134 for traceback, stat in old_group.items():
135 stat = StatisticDiff(traceback, 0, -stat.size, 0, -stat.count)
136 statistics.append(stat)
137 return statistics
138
139
140 @total_ordering
141 class ESC[4;38;5;81mFrame:
142 """
143 Frame of a traceback.
144 """
145 __slots__ = ("_frame",)
146
147 def __init__(self, frame):
148 # frame is a tuple: (filename: str, lineno: int)
149 self._frame = frame
150
151 @property
152 def filename(self):
153 return self._frame[0]
154
155 @property
156 def lineno(self):
157 return self._frame[1]
158
159 def __eq__(self, other):
160 if not isinstance(other, Frame):
161 return NotImplemented
162 return (self._frame == other._frame)
163
164 def __lt__(self, other):
165 if not isinstance(other, Frame):
166 return NotImplemented
167 return (self._frame < other._frame)
168
169 def __hash__(self):
170 return hash(self._frame)
171
172 def __str__(self):
173 return "%s:%s" % (self.filename, self.lineno)
174
175 def __repr__(self):
176 return "<Frame filename=%r lineno=%r>" % (self.filename, self.lineno)
177
178
179 @total_ordering
180 class ESC[4;38;5;81mTraceback(ESC[4;38;5;149mSequence):
181 """
182 Sequence of Frame instances sorted from the oldest frame
183 to the most recent frame.
184 """
185 __slots__ = ("_frames", '_total_nframe')
186
187 def __init__(self, frames, total_nframe=None):
188 Sequence.__init__(self)
189 # frames is a tuple of frame tuples: see Frame constructor for the
190 # format of a frame tuple; it is reversed, because _tracemalloc
191 # returns frames sorted from most recent to oldest, but the
192 # Python API expects oldest to most recent
193 self._frames = tuple(reversed(frames))
194 self._total_nframe = total_nframe
195
196 @property
197 def total_nframe(self):
198 return self._total_nframe
199
200 def __len__(self):
201 return len(self._frames)
202
203 def __getitem__(self, index):
204 if isinstance(index, slice):
205 return tuple(Frame(trace) for trace in self._frames[index])
206 else:
207 return Frame(self._frames[index])
208
209 def __contains__(self, frame):
210 return frame._frame in self._frames
211
212 def __hash__(self):
213 return hash(self._frames)
214
215 def __eq__(self, other):
216 if not isinstance(other, Traceback):
217 return NotImplemented
218 return (self._frames == other._frames)
219
220 def __lt__(self, other):
221 if not isinstance(other, Traceback):
222 return NotImplemented
223 return (self._frames < other._frames)
224
225 def __str__(self):
226 return str(self[0])
227
228 def __repr__(self):
229 s = f"<Traceback {tuple(self)}"
230 if self._total_nframe is None:
231 s += ">"
232 else:
233 s += f" total_nframe={self.total_nframe}>"
234 return s
235
236 def format(self, limit=None, most_recent_first=False):
237 lines = []
238 if limit is not None:
239 if limit > 0:
240 frame_slice = self[-limit:]
241 else:
242 frame_slice = self[:limit]
243 else:
244 frame_slice = self
245
246 if most_recent_first:
247 frame_slice = reversed(frame_slice)
248 for frame in frame_slice:
249 lines.append(' File "%s", line %s'
250 % (frame.filename, frame.lineno))
251 line = linecache.getline(frame.filename, frame.lineno).strip()
252 if line:
253 lines.append(' %s' % line)
254 return lines
255
256
257 def get_object_traceback(obj):
258 """
259 Get the traceback where the Python object *obj* was allocated.
260 Return a Traceback instance.
261
262 Return None if the tracemalloc module is not tracing memory allocations or
263 did not trace the allocation of the object.
264 """
265 frames = _get_object_traceback(obj)
266 if frames is not None:
267 return Traceback(frames)
268 else:
269 return None
270
271
272 class ESC[4;38;5;81mTrace:
273 """
274 Trace of a memory block.
275 """
276 __slots__ = ("_trace",)
277
278 def __init__(self, trace):
279 # trace is a tuple: (domain: int, size: int, traceback: tuple).
280 # See Traceback constructor for the format of the traceback tuple.
281 self._trace = trace
282
283 @property
284 def domain(self):
285 return self._trace[0]
286
287 @property
288 def size(self):
289 return self._trace[1]
290
291 @property
292 def traceback(self):
293 return Traceback(*self._trace[2:])
294
295 def __eq__(self, other):
296 if not isinstance(other, Trace):
297 return NotImplemented
298 return (self._trace == other._trace)
299
300 def __hash__(self):
301 return hash(self._trace)
302
303 def __str__(self):
304 return "%s: %s" % (self.traceback, _format_size(self.size, False))
305
306 def __repr__(self):
307 return ("<Trace domain=%s size=%s, traceback=%r>"
308 % (self.domain, _format_size(self.size, False), self.traceback))
309
310
311 class ESC[4;38;5;81m_Traces(ESC[4;38;5;149mSequence):
312 def __init__(self, traces):
313 Sequence.__init__(self)
314 # traces is a tuple of trace tuples: see Trace constructor
315 self._traces = traces
316
317 def __len__(self):
318 return len(self._traces)
319
320 def __getitem__(self, index):
321 if isinstance(index, slice):
322 return tuple(Trace(trace) for trace in self._traces[index])
323 else:
324 return Trace(self._traces[index])
325
326 def __contains__(self, trace):
327 return trace._trace in self._traces
328
329 def __eq__(self, other):
330 if not isinstance(other, _Traces):
331 return NotImplemented
332 return (self._traces == other._traces)
333
334 def __repr__(self):
335 return "<Traces len=%s>" % len(self)
336
337
338 def _normalize_filename(filename):
339 filename = os.path.normcase(filename)
340 if filename.endswith('.pyc'):
341 filename = filename[:-1]
342 return filename
343
344
345 class ESC[4;38;5;81mBaseFilter:
346 def __init__(self, inclusive):
347 self.inclusive = inclusive
348
349 def _match(self, trace):
350 raise NotImplementedError
351
352
353 class ESC[4;38;5;81mFilter(ESC[4;38;5;149mBaseFilter):
354 def __init__(self, inclusive, filename_pattern,
355 lineno=None, all_frames=False, domain=None):
356 super().__init__(inclusive)
357 self.inclusive = inclusive
358 self._filename_pattern = _normalize_filename(filename_pattern)
359 self.lineno = lineno
360 self.all_frames = all_frames
361 self.domain = domain
362
363 @property
364 def filename_pattern(self):
365 return self._filename_pattern
366
367 def _match_frame_impl(self, filename, lineno):
368 filename = _normalize_filename(filename)
369 if not fnmatch.fnmatch(filename, self._filename_pattern):
370 return False
371 if self.lineno is None:
372 return True
373 else:
374 return (lineno == self.lineno)
375
376 def _match_frame(self, filename, lineno):
377 return self._match_frame_impl(filename, lineno) ^ (not self.inclusive)
378
379 def _match_traceback(self, traceback):
380 if self.all_frames:
381 if any(self._match_frame_impl(filename, lineno)
382 for filename, lineno in traceback):
383 return self.inclusive
384 else:
385 return (not self.inclusive)
386 else:
387 filename, lineno = traceback[0]
388 return self._match_frame(filename, lineno)
389
390 def _match(self, trace):
391 domain, size, traceback, total_nframe = trace
392 res = self._match_traceback(traceback)
393 if self.domain is not None:
394 if self.inclusive:
395 return res and (domain == self.domain)
396 else:
397 return res or (domain != self.domain)
398 return res
399
400
401 class ESC[4;38;5;81mDomainFilter(ESC[4;38;5;149mBaseFilter):
402 def __init__(self, inclusive, domain):
403 super().__init__(inclusive)
404 self._domain = domain
405
406 @property
407 def domain(self):
408 return self._domain
409
410 def _match(self, trace):
411 domain, size, traceback, total_nframe = trace
412 return (domain == self.domain) ^ (not self.inclusive)
413
414
415 class ESC[4;38;5;81mSnapshot:
416 """
417 Snapshot of traces of memory blocks allocated by Python.
418 """
419
420 def __init__(self, traces, traceback_limit):
421 # traces is a tuple of trace tuples: see _Traces constructor for
422 # the exact format
423 self.traces = _Traces(traces)
424 self.traceback_limit = traceback_limit
425
426 def dump(self, filename):
427 """
428 Write the snapshot into a file.
429 """
430 with open(filename, "wb") as fp:
431 pickle.dump(self, fp, pickle.HIGHEST_PROTOCOL)
432
433 @staticmethod
434 def load(filename):
435 """
436 Load a snapshot from a file.
437 """
438 with open(filename, "rb") as fp:
439 return pickle.load(fp)
440
441 def _filter_trace(self, include_filters, exclude_filters, trace):
442 if include_filters:
443 if not any(trace_filter._match(trace)
444 for trace_filter in include_filters):
445 return False
446 if exclude_filters:
447 if any(not trace_filter._match(trace)
448 for trace_filter in exclude_filters):
449 return False
450 return True
451
452 def filter_traces(self, filters):
453 """
454 Create a new Snapshot instance with a filtered traces sequence, filters
455 is a list of Filter or DomainFilter instances. If filters is an empty
456 list, return a new Snapshot instance with a copy of the traces.
457 """
458 if not isinstance(filters, Iterable):
459 raise TypeError("filters must be a list of filters, not %s"
460 % type(filters).__name__)
461 if filters:
462 include_filters = []
463 exclude_filters = []
464 for trace_filter in filters:
465 if trace_filter.inclusive:
466 include_filters.append(trace_filter)
467 else:
468 exclude_filters.append(trace_filter)
469 new_traces = [trace for trace in self.traces._traces
470 if self._filter_trace(include_filters,
471 exclude_filters,
472 trace)]
473 else:
474 new_traces = self.traces._traces.copy()
475 return Snapshot(new_traces, self.traceback_limit)
476
477 def _group_by(self, key_type, cumulative):
478 if key_type not in ('traceback', 'filename', 'lineno'):
479 raise ValueError("unknown key_type: %r" % (key_type,))
480 if cumulative and key_type not in ('lineno', 'filename'):
481 raise ValueError("cumulative mode cannot by used "
482 "with key type %r" % key_type)
483
484 stats = {}
485 tracebacks = {}
486 if not cumulative:
487 for trace in self.traces._traces:
488 domain, size, trace_traceback, total_nframe = trace
489 try:
490 traceback = tracebacks[trace_traceback]
491 except KeyError:
492 if key_type == 'traceback':
493 frames = trace_traceback
494 elif key_type == 'lineno':
495 frames = trace_traceback[:1]
496 else: # key_type == 'filename':
497 frames = ((trace_traceback[0][0], 0),)
498 traceback = Traceback(frames)
499 tracebacks[trace_traceback] = traceback
500 try:
501 stat = stats[traceback]
502 stat.size += size
503 stat.count += 1
504 except KeyError:
505 stats[traceback] = Statistic(traceback, size, 1)
506 else:
507 # cumulative statistics
508 for trace in self.traces._traces:
509 domain, size, trace_traceback, total_nframe = trace
510 for frame in trace_traceback:
511 try:
512 traceback = tracebacks[frame]
513 except KeyError:
514 if key_type == 'lineno':
515 frames = (frame,)
516 else: # key_type == 'filename':
517 frames = ((frame[0], 0),)
518 traceback = Traceback(frames)
519 tracebacks[frame] = traceback
520 try:
521 stat = stats[traceback]
522 stat.size += size
523 stat.count += 1
524 except KeyError:
525 stats[traceback] = Statistic(traceback, size, 1)
526 return stats
527
528 def statistics(self, key_type, cumulative=False):
529 """
530 Group statistics by key_type. Return a sorted list of Statistic
531 instances.
532 """
533 grouped = self._group_by(key_type, cumulative)
534 statistics = list(grouped.values())
535 statistics.sort(reverse=True, key=Statistic._sort_key)
536 return statistics
537
538 def compare_to(self, old_snapshot, key_type, cumulative=False):
539 """
540 Compute the differences with an old snapshot old_snapshot. Get
541 statistics as a sorted list of StatisticDiff instances, grouped by
542 group_by.
543 """
544 new_group = self._group_by(key_type, cumulative)
545 old_group = old_snapshot._group_by(key_type, cumulative)
546 statistics = _compare_grouped_stats(old_group, new_group)
547 statistics.sort(reverse=True, key=StatisticDiff._sort_key)
548 return statistics
549
550
551 def take_snapshot():
552 """
553 Take a snapshot of traces of memory blocks allocated by Python.
554 """
555 if not is_tracing():
556 raise RuntimeError("the tracemalloc module must be tracing memory "
557 "allocations to take a snapshot")
558 traces = _get_traces()
559 traceback_limit = get_traceback_limit()
560 return Snapshot(traces, traceback_limit)