1 import io
2 import logging
3 import os
4 import os.path
5 import re
6 import sys
7
8 from c_common import fsutil
9 from c_common.logging import VERBOSITY, Printer
10 from c_common.scriptutil import (
11 add_verbosity_cli,
12 add_traceback_cli,
13 add_sepval_cli,
14 add_progress_cli,
15 add_files_cli,
16 add_commands_cli,
17 process_args_by_key,
18 configure_logger,
19 get_prog,
20 filter_filenames,
21 iter_marks,
22 )
23 from c_parser.info import KIND
24 from c_parser.match import is_type_decl
25 from .match import filter_forward
26 from . import (
27 analyze as _analyze,
28 datafiles as _datafiles,
29 check_all as _check_all,
30 )
31
32
33 KINDS = [
34 KIND.TYPEDEF,
35 KIND.STRUCT,
36 KIND.UNION,
37 KIND.ENUM,
38 KIND.FUNCTION,
39 KIND.VARIABLE,
40 KIND.STATEMENT,
41 ]
42
43 logger = logging.getLogger(__name__)
44
45
46 #######################################
47 # table helpers
48
49 TABLE_SECTIONS = {
50 'types': (
51 ['kind', 'name', 'data', 'file'],
52 KIND.is_type_decl,
53 (lambda v: (v.kind.value, v.filename or '', v.name)),
54 ),
55 'typedefs': 'types',
56 'structs': 'types',
57 'unions': 'types',
58 'enums': 'types',
59 'functions': (
60 ['name', 'data', 'file'],
61 (lambda kind: kind is KIND.FUNCTION),
62 (lambda v: (v.filename or '', v.name)),
63 ),
64 'variables': (
65 ['name', 'parent', 'data', 'file'],
66 (lambda kind: kind is KIND.VARIABLE),
67 (lambda v: (v.filename or '', str(v.parent) if v.parent else '', v.name)),
68 ),
69 'statements': (
70 ['file', 'parent', 'data'],
71 (lambda kind: kind is KIND.STATEMENT),
72 (lambda v: (v.filename or '', str(v.parent) if v.parent else '', v.name)),
73 ),
74 KIND.TYPEDEF: 'typedefs',
75 KIND.STRUCT: 'structs',
76 KIND.UNION: 'unions',
77 KIND.ENUM: 'enums',
78 KIND.FUNCTION: 'functions',
79 KIND.VARIABLE: 'variables',
80 KIND.STATEMENT: 'statements',
81 }
82
83
84 def _render_table(items, columns, relroot=None):
85 # XXX improve this
86 header = '\t'.join(columns)
87 div = '--------------------'
88 yield header
89 yield div
90 total = 0
91 for item in items:
92 rowdata = item.render_rowdata(columns)
93 row = [rowdata[c] for c in columns]
94 if relroot and 'file' in columns:
95 index = columns.index('file')
96 row[index] = os.path.relpath(row[index], relroot)
97 yield '\t'.join(row)
98 total += 1
99 yield div
100 yield f'total: {total}'
101
102
103 def build_section(name, groupitems, *, relroot=None):
104 info = TABLE_SECTIONS[name]
105 while type(info) is not tuple:
106 if name in KINDS:
107 name = info
108 info = TABLE_SECTIONS[info]
109
110 columns, match_kind, sortkey = info
111 items = (v for v in groupitems if match_kind(v.kind))
112 items = sorted(items, key=sortkey)
113 def render():
114 yield ''
115 yield f'{name}:'
116 yield ''
117 for line in _render_table(items, columns, relroot):
118 yield line
119 return items, render
120
121
122 #######################################
123 # the checks
124
125 CHECKS = {
126 #'globals': _check_globals,
127 }
128
129
130 def add_checks_cli(parser, checks=None, *, add_flags=None):
131 default = False
132 if not checks:
133 checks = list(CHECKS)
134 default = True
135 elif isinstance(checks, str):
136 checks = [checks]
137 if (add_flags is None and len(checks) > 1) or default:
138 add_flags = True
139
140 process_checks = add_sepval_cli(parser, '--check', 'checks', checks)
141 if add_flags:
142 for check in checks:
143 parser.add_argument(f'--{check}', dest='checks',
144 action='append_const', const=check)
145 return [
146 process_checks,
147 ]
148
149
150 def _get_check_handlers(fmt, printer, verbosity=VERBOSITY):
151 div = None
152 def handle_after():
153 pass
154 if not fmt:
155 div = ''
156 def handle_failure(failure, data):
157 data = repr(data)
158 if verbosity >= 3:
159 logger.info(f'failure: {failure}')
160 logger.info(f'data: {data}')
161 else:
162 logger.warn(f'failure: {failure} (data: {data})')
163 elif fmt == 'raw':
164 def handle_failure(failure, data):
165 print(f'{failure!r} {data!r}')
166 elif fmt == 'brief':
167 def handle_failure(failure, data):
168 parent = data.parent or ''
169 funcname = parent if isinstance(parent, str) else parent.name
170 name = f'({funcname}).{data.name}' if funcname else data.name
171 failure = failure.split('\t')[0]
172 print(f'{data.filename}:{name} - {failure}')
173 elif fmt == 'summary':
174 def handle_failure(failure, data):
175 print(_fmt_one_summary(data, failure))
176 elif fmt == 'full':
177 div = ''
178 def handle_failure(failure, data):
179 name = data.shortkey if data.kind is KIND.VARIABLE else data.name
180 parent = data.parent or ''
181 funcname = parent if isinstance(parent, str) else parent.name
182 known = 'yes' if data.is_known else '*** NO ***'
183 print(f'{data.kind.value} {name!r} failed ({failure})')
184 print(f' file: {data.filename}')
185 print(f' func: {funcname or "-"}')
186 print(f' name: {data.name}')
187 print(f' data: ...')
188 print(f' type unknown: {known}')
189 else:
190 if fmt in FORMATS:
191 raise NotImplementedError(fmt)
192 raise ValueError(f'unsupported fmt {fmt!r}')
193 return handle_failure, handle_after, div
194
195
196 #######################################
197 # the formats
198
199 def fmt_raw(analysis):
200 for item in analysis:
201 yield from item.render('raw')
202
203
204 def fmt_brief(analysis):
205 # XXX Support sorting.
206 items = sorted(analysis)
207 for kind in KINDS:
208 if kind is KIND.STATEMENT:
209 continue
210 for item in items:
211 if item.kind is not kind:
212 continue
213 yield from item.render('brief')
214 yield f' total: {len(items)}'
215
216
217 def fmt_summary(analysis):
218 # XXX Support sorting and grouping.
219 items = list(analysis)
220 total = len(items)
221
222 def section(name):
223 _, render = build_section(name, items)
224 yield from render()
225
226 yield from section('types')
227 yield from section('functions')
228 yield from section('variables')
229 yield from section('statements')
230
231 yield ''
232 # yield f'grand total: {len(supported) + len(unsupported)}'
233 yield f'grand total: {total}'
234
235
236 def _fmt_one_summary(item, extra=None):
237 parent = item.parent or ''
238 funcname = parent if isinstance(parent, str) else parent.name
239 if extra:
240 return f'{item.filename:35}\t{funcname or "-":35}\t{item.name:40}\t{extra}'
241 else:
242 return f'{item.filename:35}\t{funcname or "-":35}\t{item.name}'
243
244
245 def fmt_full(analysis):
246 # XXX Support sorting.
247 items = sorted(analysis, key=lambda v: v.key)
248 yield ''
249 for item in items:
250 yield from item.render('full')
251 yield ''
252 yield f'total: {len(items)}'
253
254
255 FORMATS = {
256 'raw': fmt_raw,
257 'brief': fmt_brief,
258 'summary': fmt_summary,
259 'full': fmt_full,
260 }
261
262
263 def add_output_cli(parser, *, default='summary'):
264 parser.add_argument('--format', dest='fmt', default=default, choices=tuple(FORMATS))
265
266 def process_args(args, *, argv=None):
267 pass
268 return process_args
269
270
271 #######################################
272 # the commands
273
274 def _cli_check(parser, checks=None, **kwargs):
275 if isinstance(checks, str):
276 checks = [checks]
277 if checks is False:
278 process_checks = None
279 elif checks is None:
280 process_checks = add_checks_cli(parser)
281 elif len(checks) == 1 and type(checks) is not dict and re.match(r'^<.*>$', checks[0]):
282 check = checks[0][1:-1]
283 def process_checks(args, *, argv=None):
284 args.checks = [check]
285 else:
286 process_checks = add_checks_cli(parser, checks=checks)
287 process_progress = add_progress_cli(parser)
288 process_output = add_output_cli(parser, default=None)
289 process_files = add_files_cli(parser, **kwargs)
290 return [
291 process_checks,
292 process_progress,
293 process_output,
294 process_files,
295 ]
296
297
298 def cmd_check(filenames, *,
299 checks=None,
300 ignored=None,
301 fmt=None,
302 failfast=False,
303 iter_filenames=None,
304 relroot=fsutil.USE_CWD,
305 track_progress=None,
306 verbosity=VERBOSITY,
307 _analyze=_analyze,
308 _CHECKS=CHECKS,
309 **kwargs
310 ):
311 if not checks:
312 checks = _CHECKS
313 elif isinstance(checks, str):
314 checks = [checks]
315 checks = [_CHECKS[c] if isinstance(c, str) else c
316 for c in checks]
317 printer = Printer(verbosity)
318 (handle_failure, handle_after, div
319 ) = _get_check_handlers(fmt, printer, verbosity)
320
321 filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
322 filenames = filter_filenames(filenames, iter_filenames, relroot)
323 if track_progress:
324 filenames = track_progress(filenames)
325
326 logger.info('analyzing files...')
327 analyzed = _analyze(filenames, **kwargs)
328 analyzed.fix_filenames(relroot, normalize=False)
329 decls = filter_forward(analyzed, markpublic=True)
330
331 logger.info('checking analysis results...')
332 failed = []
333 for data, failure in _check_all(decls, checks, failfast=failfast):
334 if data is None:
335 printer.info('stopping after one failure')
336 break
337 if div is not None and len(failed) > 0:
338 printer.info(div)
339 failed.append(data)
340 handle_failure(failure, data)
341 handle_after()
342
343 printer.info('-------------------------')
344 logger.info(f'total failures: {len(failed)}')
345 logger.info('done checking')
346
347 if fmt == 'summary':
348 print('Categorized by storage:')
349 print()
350 from .match import group_by_storage
351 grouped = group_by_storage(failed, ignore_non_match=False)
352 for group, decls in grouped.items():
353 print()
354 print(group)
355 for decl in decls:
356 print(' ', _fmt_one_summary(decl))
357 print(f'subtotal: {len(decls)}')
358
359 if len(failed) > 0:
360 sys.exit(len(failed))
361
362
363 def _cli_analyze(parser, **kwargs):
364 process_progress = add_progress_cli(parser)
365 process_output = add_output_cli(parser)
366 process_files = add_files_cli(parser, **kwargs)
367 return [
368 process_progress,
369 process_output,
370 process_files,
371 ]
372
373
374 # XXX Support filtering by kind.
375 def cmd_analyze(filenames, *,
376 fmt=None,
377 iter_filenames=None,
378 relroot=fsutil.USE_CWD,
379 track_progress=None,
380 verbosity=None,
381 _analyze=_analyze,
382 formats=FORMATS,
383 **kwargs
384 ):
385 verbosity = verbosity if verbosity is not None else 3
386
387 try:
388 do_fmt = formats[fmt]
389 except KeyError:
390 raise ValueError(f'unsupported fmt {fmt!r}')
391
392 filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
393 filenames = filter_filenames(filenames, iter_filenames, relroot)
394 if track_progress:
395 filenames = track_progress(filenames)
396
397 logger.info('analyzing files...')
398 analyzed = _analyze(filenames, **kwargs)
399 analyzed.fix_filenames(relroot, normalize=False)
400 decls = filter_forward(analyzed, markpublic=True)
401
402 for line in do_fmt(decls):
403 print(line)
404
405
406 def _cli_data(parser, filenames=None, known=None):
407 ArgumentParser = type(parser)
408 common = ArgumentParser(add_help=False)
409 # These flags will get processed by the top-level parse_args().
410 add_verbosity_cli(common)
411 add_traceback_cli(common)
412
413 subs = parser.add_subparsers(dest='datacmd')
414
415 sub = subs.add_parser('show', parents=[common])
416 if known is None:
417 sub.add_argument('--known', required=True)
418 if filenames is None:
419 sub.add_argument('filenames', metavar='FILE', nargs='+')
420
421 sub = subs.add_parser('dump', parents=[common])
422 if known is None:
423 sub.add_argument('--known')
424 sub.add_argument('--show', action='store_true')
425 process_progress = add_progress_cli(sub)
426
427 sub = subs.add_parser('check', parents=[common])
428 if known is None:
429 sub.add_argument('--known', required=True)
430
431 def process_args(args, *, argv):
432 if args.datacmd == 'dump':
433 process_progress(args, argv)
434 return process_args
435
436
437 def cmd_data(datacmd, filenames, known=None, *,
438 _analyze=_analyze,
439 formats=FORMATS,
440 extracolumns=None,
441 relroot=fsutil.USE_CWD,
442 track_progress=None,
443 **kwargs
444 ):
445 kwargs.pop('verbosity', None)
446 usestdout = kwargs.pop('show', None)
447 if datacmd == 'show':
448 do_fmt = formats['summary']
449 if isinstance(known, str):
450 known, _ = _datafiles.get_known(known, extracolumns, relroot)
451 for line in do_fmt(known):
452 print(line)
453 elif datacmd == 'dump':
454 filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
455 if track_progress:
456 filenames = track_progress(filenames)
457 analyzed = _analyze(filenames, **kwargs)
458 analyzed.fix_filenames(relroot, normalize=False)
459 if known is None or usestdout:
460 outfile = io.StringIO()
461 _datafiles.write_known(analyzed, outfile, extracolumns,
462 relroot=relroot)
463 print(outfile.getvalue())
464 else:
465 _datafiles.write_known(analyzed, known, extracolumns,
466 relroot=relroot)
467 elif datacmd == 'check':
468 raise NotImplementedError(datacmd)
469 else:
470 raise ValueError(f'unsupported data command {datacmd!r}')
471
472
473 COMMANDS = {
474 'check': (
475 'analyze and fail if the given C source/header files have any problems',
476 [_cli_check],
477 cmd_check,
478 ),
479 'analyze': (
480 'report on the state of the given C source/header files',
481 [_cli_analyze],
482 cmd_analyze,
483 ),
484 'data': (
485 'check/manage local data (e.g. known types, ignored vars, caches)',
486 [_cli_data],
487 cmd_data,
488 ),
489 }
490
491
492 #######################################
493 # the script
494
495 def parse_args(argv=sys.argv[1:], prog=sys.argv[0], *, subset=None):
496 import argparse
497 parser = argparse.ArgumentParser(
498 prog=prog or get_prog(),
499 )
500
501 processors = add_commands_cli(
502 parser,
503 commands={k: v[1] for k, v in COMMANDS.items()},
504 commonspecs=[
505 add_verbosity_cli,
506 add_traceback_cli,
507 ],
508 subset=subset,
509 )
510
511 args = parser.parse_args(argv)
512 ns = vars(args)
513
514 cmd = ns.pop('cmd')
515
516 verbosity, traceback_cm = process_args_by_key(
517 args,
518 argv,
519 processors[cmd],
520 ['verbosity', 'traceback_cm'],
521 )
522 # "verbosity" is sent to the commands, so we put it back.
523 args.verbosity = verbosity
524
525 return cmd, ns, verbosity, traceback_cm
526
527
528 def main(cmd, cmd_kwargs):
529 try:
530 run_cmd = COMMANDS[cmd][0]
531 except KeyError:
532 raise ValueError(f'unsupported cmd {cmd!r}')
533 run_cmd(**cmd_kwargs)
534
535
536 if __name__ == '__main__':
537 cmd, cmd_kwargs, verbosity, traceback_cm = parse_args()
538 configure_logger(verbosity)
539 with traceback_cm:
540 main(cmd, cmd_kwargs)