1 from collections import namedtuple
2 import logging
3 import os
4 import os.path
5 import re
6 import textwrap
7
8 from c_common.tables import build_table, resolve_columns
9 from c_parser.parser._regexes import _ind
10 from ._files import iter_header_files
11 from . import REPO_ROOT
12
13
14 logger = logging.getLogger(__name__)
15
16
17 INCLUDE_ROOT = os.path.join(REPO_ROOT, 'Include')
18 INCLUDE_CPYTHON = os.path.join(INCLUDE_ROOT, 'cpython')
19 INCLUDE_INTERNAL = os.path.join(INCLUDE_ROOT, 'internal')
20
21 _MAYBE_NESTED_PARENS = textwrap.dedent(r'''
22 (?:
23 (?: [^(]* [(] [^()]* [)] )* [^(]*
24 )
25 ''')
26
27 CAPI_FUNC = textwrap.dedent(rf'''
28 (?:
29 ^
30 \s*
31 PyAPI_FUNC \s*
32 [(]
33 {_ind(_MAYBE_NESTED_PARENS, 2)}
34 [)] \s*
35 (\w+) # <func>
36 \s* [(]
37 )
38 ''')
39 CAPI_DATA = textwrap.dedent(rf'''
40 (?:
41 ^
42 \s*
43 PyAPI_DATA \s*
44 [(]
45 {_ind(_MAYBE_NESTED_PARENS, 2)}
46 [)] \s*
47 (\w+) # <data>
48 \b [^(]
49 )
50 ''')
51 CAPI_INLINE = textwrap.dedent(r'''
52 (?:
53 ^
54 \s*
55 static \s+ inline \s+
56 .*?
57 \s+
58 ( \w+ ) # <inline>
59 \s* [(]
60 )
61 ''')
62 CAPI_MACRO = textwrap.dedent(r'''
63 (?:
64 (\w+) # <macro>
65 [(]
66 )
67 ''')
68 CAPI_CONSTANT = textwrap.dedent(r'''
69 (?:
70 (\w+) # <constant>
71 \s+ [^(]
72 )
73 ''')
74 CAPI_DEFINE = textwrap.dedent(rf'''
75 (?:
76 ^
77 \s* [#] \s* define \s+
78 (?:
79 {_ind(CAPI_MACRO, 3)}
80 |
81 {_ind(CAPI_CONSTANT, 3)}
82 |
83 (?:
84 # ignored
85 \w+ # <defined_name>
86 \s*
87 $
88 )
89 )
90 )
91 ''')
92 CAPI_RE = re.compile(textwrap.dedent(rf'''
93 (?:
94 {_ind(CAPI_FUNC, 2)}
95 |
96 {_ind(CAPI_DATA, 2)}
97 |
98 {_ind(CAPI_INLINE, 2)}
99 |
100 {_ind(CAPI_DEFINE, 2)}
101 )
102 '''), re.VERBOSE)
103
104 KINDS = [
105 'func',
106 'data',
107 'inline',
108 'macro',
109 'constant',
110 ]
111
112
113 def _parse_line(line, prev=None):
114 last = line
115 if prev:
116 if not prev.endswith(os.linesep):
117 prev += os.linesep
118 line = prev + line
119 m = CAPI_RE.match(line)
120 if not m:
121 if not prev and line.startswith('static inline '):
122 return line # the new "prev"
123 #if 'PyAPI_' in line or '#define ' in line or ' define ' in line:
124 # print(line)
125 return None
126 results = zip(KINDS, m.groups())
127 for kind, name in results:
128 if name:
129 clean = last.split('//')[0].rstrip()
130 if clean.endswith('*/'):
131 clean = clean.split('/*')[0].rstrip()
132
133 if kind == 'macro' or kind == 'constant':
134 if not clean.endswith('\\'):
135 return name, kind
136 elif kind == 'inline':
137 if clean.endswith('}'):
138 if not prev or clean == '}':
139 return name, kind
140 elif kind == 'func' or kind == 'data':
141 if clean.endswith(';'):
142 return name, kind
143 else:
144 # This should not be reached.
145 raise NotImplementedError
146 return line # the new "prev"
147 # It was a plain #define.
148 return None
149
150
151 LEVELS = [
152 'stable',
153 'cpython',
154 'private',
155 'internal',
156 ]
157
158 def _get_level(filename, name, *,
159 _cpython=INCLUDE_CPYTHON + os.path.sep,
160 _internal=INCLUDE_INTERNAL + os.path.sep,
161 ):
162 if filename.startswith(_internal):
163 return 'internal'
164 elif name.startswith('_'):
165 return 'private'
166 elif os.path.dirname(filename) == INCLUDE_ROOT:
167 return 'stable'
168 elif filename.startswith(_cpython):
169 return 'cpython'
170 else:
171 raise NotImplementedError
172 #return '???'
173
174
175 GROUPINGS = {
176 'kind': KINDS,
177 'level': LEVELS,
178 }
179
180
181 class ESC[4;38;5;81mCAPIItem(ESC[4;38;5;149mnamedtuple('CAPIItem', 'file lno name kind level')):
182
183 @classmethod
184 def from_line(cls, line, filename, lno, prev=None):
185 parsed = _parse_line(line, prev)
186 if not parsed:
187 return None, None
188 if isinstance(parsed, str):
189 # incomplete
190 return None, parsed
191 name, kind = parsed
192 level = _get_level(filename, name)
193 self = cls(filename, lno, name, kind, level)
194 if prev:
195 self._text = (prev + line).rstrip().splitlines()
196 else:
197 self._text = [line.rstrip()]
198 return self, None
199
200 @property
201 def relfile(self):
202 return self.file[len(REPO_ROOT) + 1:]
203
204 @property
205 def text(self):
206 try:
207 return self._text
208 except AttributeError:
209 # XXX Actually ready the text from disk?.
210 self._text = []
211 if self.kind == 'data':
212 self._text = [
213 f'PyAPI_DATA(...) {self.name}',
214 ]
215 elif self.kind == 'func':
216 self._text = [
217 f'PyAPI_FUNC(...) {self.name}(...);',
218 ]
219 elif self.kind == 'inline':
220 self._text = [
221 f'static inline {self.name}(...);',
222 ]
223 elif self.kind == 'macro':
224 self._text = [
225 f'#define {self.name}(...) \\',
226 f' ...',
227 ]
228 elif self.kind == 'constant':
229 self._text = [
230 f'#define {self.name} ...',
231 ]
232 else:
233 raise NotImplementedError
234
235 return self._text
236
237
238 def _parse_groupby(raw):
239 if not raw:
240 raw = 'kind'
241
242 if isinstance(raw, str):
243 groupby = raw.replace(',', ' ').strip().split()
244 else:
245 raise NotImplementedError
246
247 if not all(v in GROUPINGS for v in groupby):
248 raise ValueError(f'invalid groupby value {raw!r}')
249 return groupby
250
251
252 def _resolve_full_groupby(groupby):
253 if isinstance(groupby, str):
254 groupby = [groupby]
255 groupings = []
256 for grouping in groupby + list(GROUPINGS):
257 if grouping not in groupings:
258 groupings.append(grouping)
259 return groupings
260
261
262 def summarize(items, *, groupby='kind', includeempty=True, minimize=None):
263 if minimize is None:
264 if includeempty is None:
265 minimize = True
266 includeempty = False
267 else:
268 minimize = includeempty
269 elif includeempty is None:
270 includeempty = minimize
271 elif minimize and includeempty:
272 raise ValueError(f'cannot minimize and includeempty at the same time')
273
274 groupby = _parse_groupby(groupby)[0]
275 _outer, _inner = _resolve_full_groupby(groupby)
276 outers = GROUPINGS[_outer]
277 inners = GROUPINGS[_inner]
278
279 summary = {
280 'totals': {
281 'all': 0,
282 'subs': {o: 0 for o in outers},
283 'bygroup': {o: {i: 0 for i in inners}
284 for o in outers},
285 },
286 }
287
288 for item in items:
289 outer = getattr(item, _outer)
290 inner = getattr(item, _inner)
291 # Update totals.
292 summary['totals']['all'] += 1
293 summary['totals']['subs'][outer] += 1
294 summary['totals']['bygroup'][outer][inner] += 1
295
296 if not includeempty:
297 subtotals = summary['totals']['subs']
298 bygroup = summary['totals']['bygroup']
299 for outer in outers:
300 if subtotals[outer] == 0:
301 del subtotals[outer]
302 del bygroup[outer]
303 continue
304
305 for inner in inners:
306 if bygroup[outer][inner] == 0:
307 del bygroup[outer][inner]
308 if minimize:
309 if len(bygroup[outer]) == 1:
310 del bygroup[outer]
311
312 return summary
313
314
315 def _parse_capi(lines, filename):
316 if isinstance(lines, str):
317 lines = lines.splitlines()
318 prev = None
319 for lno, line in enumerate(lines, 1):
320 parsed, prev = CAPIItem.from_line(line, filename, lno, prev)
321 if parsed:
322 yield parsed
323 if prev:
324 parsed, prev = CAPIItem.from_line('', filename, lno, prev)
325 if parsed:
326 yield parsed
327 if prev:
328 print('incomplete match:')
329 print(filename)
330 print(prev)
331 raise Exception
332
333
334 def iter_capi(filenames=None):
335 for filename in iter_header_files(filenames):
336 with open(filename) as infile:
337 for item in _parse_capi(infile, filename):
338 yield item
339
340
341 def resolve_filter(ignored):
342 if not ignored:
343 return None
344 ignored = set(_resolve_ignored(ignored))
345 def filter(item, *, log=None):
346 if item.name not in ignored:
347 return True
348 if log is not None:
349 log(f'ignored {item.name!r}')
350 return False
351 return filter
352
353
354 def _resolve_ignored(ignored):
355 if isinstance(ignored, str):
356 ignored = [ignored]
357 for raw in ignored:
358 if isinstance(raw, str):
359 if raw.startswith('|'):
360 yield raw[1:]
361 elif raw.startswith('<') and raw.endswith('>'):
362 filename = raw[1:-1]
363 try:
364 infile = open(filename)
365 except Exception as exc:
366 logger.error(f'ignore file failed: {exc}')
367 continue
368 logger.log(1, f'reading ignored names from {filename!r}')
369 with infile:
370 for line in infile:
371 if not line:
372 continue
373 if line[0].isspace():
374 continue
375 line = line.partition('#')[0].rstrip()
376 if line:
377 # XXX Recurse?
378 yield line
379 else:
380 raw = raw.strip()
381 if raw:
382 yield raw
383 else:
384 raise NotImplementedError
385
386
387 def _collate(items, groupby, includeempty):
388 groupby = _parse_groupby(groupby)[0]
389 maxfilename = maxname = maxkind = maxlevel = 0
390
391 collated = {}
392 groups = GROUPINGS[groupby]
393 for group in groups:
394 collated[group] = []
395
396 for item in items:
397 key = getattr(item, groupby)
398 collated[key].append(item)
399 maxfilename = max(len(item.relfile), maxfilename)
400 maxname = max(len(item.name), maxname)
401 maxkind = max(len(item.kind), maxkind)
402 maxlevel = max(len(item.level), maxlevel)
403 if not includeempty:
404 for group in groups:
405 if not collated[group]:
406 del collated[group]
407 maxextra = {
408 'kind': maxkind,
409 'level': maxlevel,
410 }
411 return collated, groupby, maxfilename, maxname, maxextra
412
413
414 def _get_sortkey(sort, _groupby, _columns):
415 if sort is True or sort is None:
416 # For now:
417 def sortkey(item):
418 return (
419 item.level == 'private',
420 LEVELS.index(item.level),
421 KINDS.index(item.kind),
422 os.path.dirname(item.file),
423 os.path.basename(item.file),
424 item.name,
425 )
426 return sortkey
427
428 sortfields = 'not-private level kind dirname basename name'.split()
429 elif isinstance(sort, str):
430 sortfields = sort.replace(',', ' ').strip().split()
431 elif callable(sort):
432 return sort
433 else:
434 raise NotImplementedError
435
436 # XXX Build a sortkey func from sortfields.
437 raise NotImplementedError
438
439
440 ##################################
441 # CLI rendering
442
443 _MARKERS = {
444 'level': {
445 'S': 'stable',
446 'C': 'cpython',
447 'P': 'private',
448 'I': 'internal',
449 },
450 'kind': {
451 'F': 'func',
452 'D': 'data',
453 'I': 'inline',
454 'M': 'macro',
455 'C': 'constant',
456 },
457 }
458
459
460 def resolve_format(format):
461 if not format:
462 return 'table'
463 elif isinstance(format, str) and format in _FORMATS:
464 return format
465 else:
466 return resolve_columns(format)
467
468
469 def get_renderer(format):
470 format = resolve_format(format)
471 if isinstance(format, str):
472 try:
473 return _FORMATS[format]
474 except KeyError:
475 raise ValueError(f'unsupported format {format!r}')
476 else:
477 def render(items, **kwargs):
478 return render_table(items, columns=format, **kwargs)
479 return render
480
481
482 def render_table(items, *,
483 columns=None,
484 groupby='kind',
485 sort=True,
486 showempty=False,
487 verbose=False,
488 ):
489 if groupby is None:
490 groupby = 'kind'
491 if showempty is None:
492 showempty = False
493
494 if groupby:
495 (collated, groupby, maxfilename, maxname, maxextra,
496 ) = _collate(items, groupby, showempty)
497 for grouping in GROUPINGS:
498 maxextra[grouping] = max(len(g) for g in GROUPINGS[grouping])
499
500 _, extra = _resolve_full_groupby(groupby)
501 extras = [extra]
502 markers = {extra: _MARKERS[extra]}
503
504 groups = GROUPINGS[groupby]
505 else:
506 # XXX Support no grouping?
507 raise NotImplementedError
508
509 if columns:
510 def get_extra(item):
511 return {extra: getattr(item, extra)
512 for extra in ('kind', 'level')}
513 else:
514 if verbose:
515 extracols = [f'{extra}:{maxextra[extra]}'
516 for extra in extras]
517 def get_extra(item):
518 return {extra: getattr(item, extra)
519 for extra in extras}
520 elif len(extras) == 1:
521 extra, = extras
522 extracols = [f'{m}:1' for m in markers[extra]]
523 def get_extra(item):
524 return {m: m if getattr(item, extra) == markers[extra][m] else ''
525 for m in markers[extra]}
526 else:
527 raise NotImplementedError
528 #extracols = [[f'{m}:1' for m in markers[extra]]
529 # for extra in extras]
530 #def get_extra(item):
531 # values = {}
532 # for extra in extras:
533 # cur = markers[extra]
534 # for m in cur:
535 # values[m] = m if getattr(item, m) == cur[m] else ''
536 # return values
537 columns = [
538 f'filename:{maxfilename}',
539 f'name:{maxname}',
540 *extracols,
541 ]
542 header, div, fmt = build_table(columns)
543
544 if sort:
545 sortkey = _get_sortkey(sort, groupby, columns)
546
547 total = 0
548 for group, grouped in collated.items():
549 if not showempty and group not in collated:
550 continue
551 yield ''
552 yield f' === {group} ==='
553 yield ''
554 yield header
555 yield div
556 if grouped:
557 if sort:
558 grouped = sorted(grouped, key=sortkey)
559 for item in grouped:
560 yield fmt.format(
561 filename=item.relfile,
562 name=item.name,
563 **get_extra(item),
564 )
565 yield div
566 subtotal = len(grouped)
567 yield f' sub-total: {subtotal}'
568 total += subtotal
569 yield ''
570 yield f'total: {total}'
571
572
573 def render_full(items, *,
574 groupby='kind',
575 sort=None,
576 showempty=None,
577 verbose=False,
578 ):
579 if groupby is None:
580 groupby = 'kind'
581 if showempty is None:
582 showempty = False
583
584 if sort:
585 sortkey = _get_sortkey(sort, groupby, None)
586
587 if groupby:
588 collated, groupby, _, _, _ = _collate(items, groupby, showempty)
589 for group, grouped in collated.items():
590 yield '#' * 25
591 yield f'# {group} ({len(grouped)})'
592 yield '#' * 25
593 yield ''
594 if not grouped:
595 continue
596 if sort:
597 grouped = sorted(grouped, key=sortkey)
598 for item in grouped:
599 yield from _render_item_full(item, groupby, verbose)
600 yield ''
601 else:
602 if sort:
603 items = sorted(items, key=sortkey)
604 for item in items:
605 yield from _render_item_full(item, None, verbose)
606 yield ''
607
608
609 def _render_item_full(item, groupby, verbose):
610 yield item.name
611 yield f' {"filename:":10} {item.relfile}'
612 for extra in ('kind', 'level'):
613 yield f' {extra+":":10} {getattr(item, extra)}'
614 if verbose:
615 print(' ---------------------------------------')
616 for lno, line in enumerate(item.text, item.lno):
617 print(f' | {lno:3} {line}')
618 print(' ---------------------------------------')
619
620
621 def render_summary(items, *,
622 groupby='kind',
623 sort=None,
624 showempty=None,
625 verbose=False,
626 ):
627 if groupby is None:
628 groupby = 'kind'
629 summary = summarize(
630 items,
631 groupby=groupby,
632 includeempty=showempty,
633 minimize=None if showempty else not verbose,
634 )
635
636 subtotals = summary['totals']['subs']
637 bygroup = summary['totals']['bygroup']
638 for outer, subtotal in subtotals.items():
639 if bygroup:
640 subtotal = f'({subtotal})'
641 yield f'{outer + ":":20} {subtotal:>8}'
642 else:
643 yield f'{outer + ":":10} {subtotal:>8}'
644 if outer in bygroup:
645 for inner, count in bygroup[outer].items():
646 yield f' {inner + ":":9} {count}'
647 total = f'*{summary["totals"]["all"]}*'
648 label = '*total*:'
649 if bygroup:
650 yield f'{label:20} {total:>8}'
651 else:
652 yield f'{label:10} {total:>9}'
653
654
655 _FORMATS = {
656 'table': render_table,
657 'full': render_full,
658 'summary': render_summary,
659 }