1 import logging
2 import sys
3 import textwrap
4
5 from c_common.scriptutil import (
6 VERBOSITY,
7 add_verbosity_cli,
8 add_traceback_cli,
9 add_commands_cli,
10 add_kind_filtering_cli,
11 add_files_cli,
12 add_progress_cli,
13 process_args_by_key,
14 configure_logger,
15 get_prog,
16 )
17 from c_parser.info import KIND
18 import c_parser.__main__ as c_parser
19 import c_analyzer.__main__ as c_analyzer
20 import c_analyzer as _c_analyzer
21 from c_analyzer.info import UNKNOWN
22 from . import _analyzer, _builtin_types, _capi, _files, _parser, REPO_ROOT
23
24
25 logger = logging.getLogger(__name__)
26
27
28 CHECK_EXPLANATION = textwrap.dedent('''
29 -------------------------
30
31 Non-constant global variables are generally not supported
32 in the CPython repo. We use a tool to analyze the C code
33 and report if any unsupported globals are found. The tool
34 may be run manually with:
35
36 ./python Tools/c-analyzer/check-c-globals.py --format summary [FILE]
37
38 Occasionally the tool is unable to parse updated code.
39 If this happens then add the file to the "EXCLUDED" list
40 in Tools/c-analyzer/cpython/_parser.py and create a new
41 issue for fixing the tool (and CC ericsnowcurrently
42 on the issue).
43
44 If the tool reports an unsupported global variable and
45 it is actually const (and thus supported) then first try
46 fixing the declaration appropriately in the code. If that
47 doesn't work then add the variable to the "should be const"
48 section of Tools/c-analyzer/cpython/ignored.tsv.
49
50 If the tool otherwise reports an unsupported global variable
51 then first try to make it non-global, possibly adding to
52 PyInterpreterState (for core code) or module state (for
53 extension modules). In an emergency, you can add the
54 variable to Tools/c-analyzer/cpython/globals-to-fix.tsv
55 to get CI passing, but doing so should be avoided. If
56 this course it taken, be sure to create an issue for
57 eliminating the global (and CC ericsnowcurrently).
58 ''')
59
60
61 def _resolve_filenames(filenames):
62 if filenames:
63 resolved = (_files.resolve_filename(f) for f in filenames)
64 else:
65 resolved = _files.iter_filenames()
66 return resolved
67
68
69 #######################################
70 # the formats
71
72 def fmt_summary(analysis):
73 # XXX Support sorting and grouping.
74 supported = []
75 unsupported = []
76 for item in analysis:
77 if item.supported:
78 supported.append(item)
79 else:
80 unsupported.append(item)
81 total = 0
82
83 def section(name, groupitems):
84 nonlocal total
85 items, render = c_analyzer.build_section(name, groupitems,
86 relroot=REPO_ROOT)
87 yield from render()
88 total += len(items)
89
90 yield ''
91 yield '===================='
92 yield 'supported'
93 yield '===================='
94
95 yield from section('types', supported)
96 yield from section('variables', supported)
97
98 yield ''
99 yield '===================='
100 yield 'unsupported'
101 yield '===================='
102
103 yield from section('types', unsupported)
104 yield from section('variables', unsupported)
105
106 yield ''
107 yield f'grand total: {total}'
108
109
110 #######################################
111 # the checks
112
113 CHECKS = dict(c_analyzer.CHECKS, **{
114 'globals': _analyzer.check_globals,
115 })
116
117 #######################################
118 # the commands
119
120 FILES_KWARGS = dict(excluded=_parser.EXCLUDED, nargs='*')
121
122
123 def _cli_parse(parser):
124 process_output = c_parser.add_output_cli(parser)
125 process_kind = add_kind_filtering_cli(parser)
126 process_preprocessor = c_parser.add_preprocessor_cli(
127 parser,
128 get_preprocessor=_parser.get_preprocessor,
129 )
130 process_files = add_files_cli(parser, **FILES_KWARGS)
131 return [
132 process_output,
133 process_kind,
134 process_preprocessor,
135 process_files,
136 ]
137
138
139 def cmd_parse(filenames=None, **kwargs):
140 filenames = _resolve_filenames(filenames)
141 if 'get_file_preprocessor' not in kwargs:
142 kwargs['get_file_preprocessor'] = _parser.get_preprocessor()
143 c_parser.cmd_parse(
144 filenames,
145 relroot=REPO_ROOT,
146 file_maxsizes=_parser.MAX_SIZES,
147 **kwargs
148 )
149
150
151 def _cli_check(parser, **kwargs):
152 return c_analyzer._cli_check(parser, CHECKS, **kwargs, **FILES_KWARGS)
153
154
155 def cmd_check(filenames=None, **kwargs):
156 filenames = _resolve_filenames(filenames)
157 kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
158 try:
159 c_analyzer.cmd_check(
160 filenames,
161 relroot=REPO_ROOT,
162 _analyze=_analyzer.analyze,
163 _CHECKS=CHECKS,
164 file_maxsizes=_parser.MAX_SIZES,
165 **kwargs
166 )
167 except SystemExit as exc:
168 num_failed = exc.args[0] if getattr(exc, 'args', None) else None
169 if isinstance(num_failed, int):
170 if num_failed > 0:
171 sys.stderr.flush()
172 print(CHECK_EXPLANATION, flush=True)
173 raise # re-raise
174 except Exception:
175 sys.stderr.flush()
176 print(CHECK_EXPLANATION, flush=True)
177 raise # re-raise
178
179
180 def cmd_analyze(filenames=None, **kwargs):
181 formats = dict(c_analyzer.FORMATS)
182 formats['summary'] = fmt_summary
183 filenames = _resolve_filenames(filenames)
184 kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
185 c_analyzer.cmd_analyze(
186 filenames,
187 relroot=REPO_ROOT,
188 _analyze=_analyzer.analyze,
189 formats=formats,
190 file_maxsizes=_parser.MAX_SIZES,
191 **kwargs
192 )
193
194
195 def _cli_data(parser):
196 filenames = False
197 known = True
198 return c_analyzer._cli_data(parser, filenames, known)
199
200
201 def cmd_data(datacmd, **kwargs):
202 formats = dict(c_analyzer.FORMATS)
203 formats['summary'] = fmt_summary
204 filenames = (file
205 for file in _resolve_filenames(None)
206 if file not in _parser.EXCLUDED)
207 kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
208 if datacmd == 'show':
209 types = _analyzer.read_known()
210 results = []
211 for decl, info in types.items():
212 if info is UNKNOWN:
213 if decl.kind in (KIND.STRUCT, KIND.UNION):
214 extra = {'unsupported': ['type unknown'] * len(decl.members)}
215 else:
216 extra = {'unsupported': ['type unknown']}
217 info = (info, extra)
218 results.append((decl, info))
219 if decl.shortkey == 'struct _object':
220 tempinfo = info
221 known = _analyzer.Analysis.from_results(results)
222 analyze = None
223 elif datacmd == 'dump':
224 known = _analyzer.KNOWN_FILE
225 def analyze(files, **kwargs):
226 decls = []
227 for decl in _analyzer.iter_decls(files, **kwargs):
228 if not KIND.is_type_decl(decl.kind):
229 continue
230 if not decl.filename.endswith('.h'):
231 if decl.shortkey not in _analyzer.KNOWN_IN_DOT_C:
232 continue
233 decls.append(decl)
234 results = _c_analyzer.analyze_decls(
235 decls,
236 known={},
237 analyze_resolved=_analyzer.analyze_resolved,
238 )
239 return _analyzer.Analysis.from_results(results)
240 else: # check
241 known = _analyzer.read_known()
242 def analyze(files, **kwargs):
243 return _analyzer.iter_decls(files, **kwargs)
244 extracolumns = None
245 c_analyzer.cmd_data(
246 datacmd,
247 filenames,
248 known,
249 _analyze=analyze,
250 formats=formats,
251 extracolumns=extracolumns,
252 relroot=REPO_ROOT,
253 **kwargs
254 )
255
256
257 def _cli_capi(parser):
258 parser.add_argument('--levels', action='append', metavar='LEVEL[,...]')
259 parser.add_argument(f'--public', dest='levels',
260 action='append_const', const='public')
261 parser.add_argument(f'--no-public', dest='levels',
262 action='append_const', const='no-public')
263 for level in _capi.LEVELS:
264 parser.add_argument(f'--{level}', dest='levels',
265 action='append_const', const=level)
266 def process_levels(args, *, argv=None):
267 levels = []
268 for raw in args.levels or ():
269 for level in raw.replace(',', ' ').strip().split():
270 if level == 'public':
271 levels.append('stable')
272 levels.append('cpython')
273 elif level == 'no-public':
274 levels.append('private')
275 levels.append('internal')
276 elif level in _capi.LEVELS:
277 levels.append(level)
278 else:
279 parser.error(f'expected LEVEL to be one of {sorted(_capi.LEVELS)}, got {level!r}')
280 args.levels = set(levels)
281
282 parser.add_argument('--kinds', action='append', metavar='KIND[,...]')
283 for kind in _capi.KINDS:
284 parser.add_argument(f'--{kind}', dest='kinds',
285 action='append_const', const=kind)
286 def process_kinds(args, *, argv=None):
287 kinds = []
288 for raw in args.kinds or ():
289 for kind in raw.replace(',', ' ').strip().split():
290 if kind in _capi.KINDS:
291 kinds.append(kind)
292 else:
293 parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}')
294 args.kinds = set(kinds)
295
296 parser.add_argument('--group-by', dest='groupby',
297 choices=['level', 'kind'])
298
299 parser.add_argument('--format', default='table')
300 parser.add_argument('--summary', dest='format',
301 action='store_const', const='summary')
302 def process_format(args, *, argv=None):
303 orig = args.format
304 args.format = _capi.resolve_format(args.format)
305 if isinstance(args.format, str):
306 if args.format not in _capi._FORMATS:
307 parser.error(f'unsupported format {orig!r}')
308
309 parser.add_argument('--show-empty', dest='showempty', action='store_true')
310 parser.add_argument('--no-show-empty', dest='showempty', action='store_false')
311 parser.set_defaults(showempty=None)
312
313 # XXX Add --sort-by, --sort and --no-sort.
314
315 parser.add_argument('--ignore', dest='ignored', action='append')
316 def process_ignored(args, *, argv=None):
317 ignored = []
318 for raw in args.ignored or ():
319 ignored.extend(raw.replace(',', ' ').strip().split())
320 args.ignored = ignored or None
321
322 parser.add_argument('filenames', nargs='*', metavar='FILENAME')
323 process_progress = add_progress_cli(parser)
324
325 return [
326 process_levels,
327 process_kinds,
328 process_format,
329 process_ignored,
330 process_progress,
331 ]
332
333
334 def cmd_capi(filenames=None, *,
335 levels=None,
336 kinds=None,
337 groupby='kind',
338 format='table',
339 showempty=None,
340 ignored=None,
341 track_progress=None,
342 verbosity=VERBOSITY,
343 **kwargs
344 ):
345 render = _capi.get_renderer(format)
346
347 filenames = _files.iter_header_files(filenames, levels=levels)
348 #filenames = (file for file, _ in main_for_filenames(filenames))
349 if track_progress:
350 filenames = track_progress(filenames)
351 items = _capi.iter_capi(filenames)
352 if levels:
353 items = (item for item in items if item.level in levels)
354 if kinds:
355 items = (item for item in items if item.kind in kinds)
356
357 filter = _capi.resolve_filter(ignored)
358 if filter:
359 items = (item for item in items if filter(item, log=lambda msg: logger.log(1, msg)))
360
361 lines = render(
362 items,
363 groupby=groupby,
364 showempty=showempty,
365 verbose=verbosity > VERBOSITY,
366 )
367 print()
368 for line in lines:
369 print(line)
370
371
372 def _cli_builtin_types(parser):
373 parser.add_argument('--format', dest='fmt', default='table')
374 # parser.add_argument('--summary', dest='format',
375 # action='store_const', const='summary')
376 def process_format(args, *, argv=None):
377 orig = args.fmt
378 args.fmt = _builtin_types.resolve_format(args.fmt)
379 if isinstance(args.fmt, str):
380 if args.fmt not in _builtin_types._FORMATS:
381 parser.error(f'unsupported format {orig!r}')
382
383 parser.add_argument('--include-modules', dest='showmodules',
384 action='store_true')
385 def process_modules(args, *, argv=None):
386 pass
387
388 return [
389 process_format,
390 process_modules,
391 ]
392
393
394 def cmd_builtin_types(fmt, *,
395 showmodules=False,
396 verbosity=VERBOSITY,
397 ):
398 render = _builtin_types.get_renderer(fmt)
399 types = _builtin_types.iter_builtin_types()
400 match = _builtin_types.resolve_matcher(showmodules)
401 if match:
402 types = (t for t in types if match(t, log=lambda msg: logger.log(1, msg)))
403
404 lines = render(
405 types,
406 # verbose=verbosity > VERBOSITY,
407 )
408 print()
409 for line in lines:
410 print(line)
411
412
413 # We do not define any other cmd_*() handlers here,
414 # favoring those defined elsewhere.
415
416 COMMANDS = {
417 'check': (
418 'analyze and fail if the CPython source code has any problems',
419 [_cli_check],
420 cmd_check,
421 ),
422 'analyze': (
423 'report on the state of the CPython source code',
424 [(lambda p: c_analyzer._cli_analyze(p, **FILES_KWARGS))],
425 cmd_analyze,
426 ),
427 'parse': (
428 'parse the CPython source files',
429 [_cli_parse],
430 cmd_parse,
431 ),
432 'data': (
433 'check/manage local data (e.g. known types, ignored vars, caches)',
434 [_cli_data],
435 cmd_data,
436 ),
437 'capi': (
438 'inspect the C-API',
439 [_cli_capi],
440 cmd_capi,
441 ),
442 'builtin-types': (
443 'show the builtin types',
444 [_cli_builtin_types],
445 cmd_builtin_types,
446 ),
447 }
448
449
450 #######################################
451 # the script
452
453 def parse_args(argv=sys.argv[1:], prog=None, *, subset=None):
454 import argparse
455 parser = argparse.ArgumentParser(
456 prog=prog or get_prog(),
457 )
458
459 # if subset == 'check' or subset == ['check']:
460 # if checks is not None:
461 # commands = dict(COMMANDS)
462 # commands['check'] = list(commands['check'])
463 # cli = commands['check'][1][0]
464 # commands['check'][1][0] = (lambda p: cli(p, checks=checks))
465 processors = add_commands_cli(
466 parser,
467 commands=COMMANDS,
468 commonspecs=[
469 add_verbosity_cli,
470 add_traceback_cli,
471 ],
472 subset=subset,
473 )
474
475 args = parser.parse_args(argv)
476 ns = vars(args)
477
478 cmd = ns.pop('cmd')
479
480 verbosity, traceback_cm = process_args_by_key(
481 args,
482 argv,
483 processors[cmd],
484 ['verbosity', 'traceback_cm'],
485 )
486 if cmd != 'parse':
487 # "verbosity" is sent to the commands, so we put it back.
488 args.verbosity = verbosity
489
490 return cmd, ns, verbosity, traceback_cm
491
492
493 def main(cmd, cmd_kwargs):
494 try:
495 run_cmd = COMMANDS[cmd][-1]
496 except KeyError:
497 raise ValueError(f'unsupported cmd {cmd!r}')
498 run_cmd(**cmd_kwargs)
499
500
501 if __name__ == '__main__':
502 cmd, cmd_kwargs, verbosity, traceback_cm = parse_args()
503 configure_logger(verbosity)
504 with traceback_cm:
505 main(cmd, cmd_kwargs)