1 import logging
2 import sys
3
4 from c_common.fsutil import expand_filenames, iter_files_by_suffix
5 from c_common.scriptutil import (
6 VERBOSITY,
7 add_verbosity_cli,
8 add_traceback_cli,
9 add_commands_cli,
10 add_kind_filtering_cli,
11 add_files_cli,
12 add_progress_cli,
13 main_for_filenames,
14 process_args_by_key,
15 configure_logger,
16 get_prog,
17 )
18 from c_parser.info import KIND
19 import c_parser.__main__ as c_parser
20 import c_analyzer.__main__ as c_analyzer
21 import c_analyzer as _c_analyzer
22 from c_analyzer.info import UNKNOWN
23 from . import _analyzer, _capi, _files, _parser, REPO_ROOT
24
25
26 logger = logging.getLogger(__name__)
27
28
29 def _resolve_filenames(filenames):
30 if filenames:
31 resolved = (_files.resolve_filename(f) for f in filenames)
32 else:
33 resolved = _files.iter_filenames()
34 return resolved
35
36
37 #######################################
38 # the formats
39
40 def fmt_summary(analysis):
41 # XXX Support sorting and grouping.
42 supported = []
43 unsupported = []
44 for item in analysis:
45 if item.supported:
46 supported.append(item)
47 else:
48 unsupported.append(item)
49 total = 0
50
51 def section(name, groupitems):
52 nonlocal total
53 items, render = c_analyzer.build_section(name, groupitems,
54 relroot=REPO_ROOT)
55 yield from render()
56 total += len(items)
57
58 yield ''
59 yield '===================='
60 yield 'supported'
61 yield '===================='
62
63 yield from section('types', supported)
64 yield from section('variables', supported)
65
66 yield ''
67 yield '===================='
68 yield 'unsupported'
69 yield '===================='
70
71 yield from section('types', unsupported)
72 yield from section('variables', unsupported)
73
74 yield ''
75 yield f'grand total: {total}'
76
77
78 #######################################
79 # the checks
80
81 CHECKS = dict(c_analyzer.CHECKS, **{
82 'globals': _analyzer.check_globals,
83 })
84
85 #######################################
86 # the commands
87
88 FILES_KWARGS = dict(excluded=_parser.EXCLUDED, nargs='*')
89
90
91 def _cli_parse(parser):
92 process_output = c_parser.add_output_cli(parser)
93 process_kind = add_kind_filtering_cli(parser)
94 process_preprocessor = c_parser.add_preprocessor_cli(
95 parser,
96 get_preprocessor=_parser.get_preprocessor,
97 )
98 process_files = add_files_cli(parser, **FILES_KWARGS)
99 return [
100 process_output,
101 process_kind,
102 process_preprocessor,
103 process_files,
104 ]
105
106
107 def cmd_parse(filenames=None, **kwargs):
108 filenames = _resolve_filenames(filenames)
109 if 'get_file_preprocessor' not in kwargs:
110 kwargs['get_file_preprocessor'] = _parser.get_preprocessor()
111 c_parser.cmd_parse(
112 filenames,
113 relroot=REPO_ROOT,
114 file_maxsizes=_parser.MAX_SIZES,
115 **kwargs
116 )
117
118
119 def _cli_check(parser, **kwargs):
120 return c_analyzer._cli_check(parser, CHECKS, **kwargs, **FILES_KWARGS)
121
122
123 def cmd_check(filenames=None, **kwargs):
124 filenames = _resolve_filenames(filenames)
125 kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
126 c_analyzer.cmd_check(
127 filenames,
128 relroot=REPO_ROOT,
129 _analyze=_analyzer.analyze,
130 _CHECKS=CHECKS,
131 file_maxsizes=_parser.MAX_SIZES,
132 **kwargs
133 )
134
135
136 def cmd_analyze(filenames=None, **kwargs):
137 formats = dict(c_analyzer.FORMATS)
138 formats['summary'] = fmt_summary
139 filenames = _resolve_filenames(filenames)
140 kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
141 c_analyzer.cmd_analyze(
142 filenames,
143 relroot=REPO_ROOT,
144 _analyze=_analyzer.analyze,
145 formats=formats,
146 file_maxsizes=_parser.MAX_SIZES,
147 **kwargs
148 )
149
150
151 def _cli_data(parser):
152 filenames = False
153 known = True
154 return c_analyzer._cli_data(parser, filenames, known)
155
156
157 def cmd_data(datacmd, **kwargs):
158 formats = dict(c_analyzer.FORMATS)
159 formats['summary'] = fmt_summary
160 filenames = (file
161 for file in _resolve_filenames(None)
162 if file not in _parser.EXCLUDED)
163 kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
164 if datacmd == 'show':
165 types = _analyzer.read_known()
166 results = []
167 for decl, info in types.items():
168 if info is UNKNOWN:
169 if decl.kind in (KIND.STRUCT, KIND.UNION):
170 extra = {'unsupported': ['type unknown'] * len(decl.members)}
171 else:
172 extra = {'unsupported': ['type unknown']}
173 info = (info, extra)
174 results.append((decl, info))
175 if decl.shortkey == 'struct _object':
176 tempinfo = info
177 known = _analyzer.Analysis.from_results(results)
178 analyze = None
179 elif datacmd == 'dump':
180 known = _analyzer.KNOWN_FILE
181 def analyze(files, **kwargs):
182 decls = []
183 for decl in _analyzer.iter_decls(files, **kwargs):
184 if not KIND.is_type_decl(decl.kind):
185 continue
186 if not decl.filename.endswith('.h'):
187 if decl.shortkey not in _analyzer.KNOWN_IN_DOT_C:
188 continue
189 decls.append(decl)
190 results = _c_analyzer.analyze_decls(
191 decls,
192 known={},
193 analyze_resolved=_analyzer.analyze_resolved,
194 )
195 return _analyzer.Analysis.from_results(results)
196 else: # check
197 known = _analyzer.read_known()
198 def analyze(files, **kwargs):
199 return _analyzer.iter_decls(files, **kwargs)
200 extracolumns = None
201 c_analyzer.cmd_data(
202 datacmd,
203 filenames,
204 known,
205 _analyze=analyze,
206 formats=formats,
207 extracolumns=extracolumns,
208 relroot=REPO_ROOT,
209 **kwargs
210 )
211
212
213 def _cli_capi(parser):
214 parser.add_argument('--levels', action='append', metavar='LEVEL[,...]')
215 parser.add_argument(f'--public', dest='levels',
216 action='append_const', const='public')
217 parser.add_argument(f'--no-public', dest='levels',
218 action='append_const', const='no-public')
219 for level in _capi.LEVELS:
220 parser.add_argument(f'--{level}', dest='levels',
221 action='append_const', const=level)
222 def process_levels(args, *, argv=None):
223 levels = []
224 for raw in args.levels or ():
225 for level in raw.replace(',', ' ').strip().split():
226 if level == 'public':
227 levels.append('stable')
228 levels.append('cpython')
229 elif level == 'no-public':
230 levels.append('private')
231 levels.append('internal')
232 elif level in _capi.LEVELS:
233 levels.append(level)
234 else:
235 parser.error(f'expected LEVEL to be one of {sorted(_capi.LEVELS)}, got {level!r}')
236 args.levels = set(levels)
237
238 parser.add_argument('--kinds', action='append', metavar='KIND[,...]')
239 for kind in _capi.KINDS:
240 parser.add_argument(f'--{kind}', dest='kinds',
241 action='append_const', const=kind)
242 def process_kinds(args, *, argv=None):
243 kinds = []
244 for raw in args.kinds or ():
245 for kind in raw.replace(',', ' ').strip().split():
246 if kind in _capi.KINDS:
247 kinds.append(kind)
248 else:
249 parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}')
250 args.kinds = set(kinds)
251
252 parser.add_argument('--group-by', dest='groupby',
253 choices=['level', 'kind'])
254
255 parser.add_argument('--format', default='table')
256 parser.add_argument('--summary', dest='format',
257 action='store_const', const='summary')
258 def process_format(args, *, argv=None):
259 orig = args.format
260 args.format = _capi.resolve_format(args.format)
261 if isinstance(args.format, str):
262 if args.format not in _capi._FORMATS:
263 parser.error(f'unsupported format {orig!r}')
264
265 parser.add_argument('--show-empty', dest='showempty', action='store_true')
266 parser.add_argument('--no-show-empty', dest='showempty', action='store_false')
267 parser.set_defaults(showempty=None)
268
269 # XXX Add --sort-by, --sort and --no-sort.
270
271 parser.add_argument('--ignore', dest='ignored', action='append')
272 def process_ignored(args, *, argv=None):
273 ignored = []
274 for raw in args.ignored or ():
275 ignored.extend(raw.replace(',', ' ').strip().split())
276 args.ignored = ignored or None
277
278 parser.add_argument('filenames', nargs='*', metavar='FILENAME')
279 process_progress = add_progress_cli(parser)
280
281 return [
282 process_levels,
283 process_kinds,
284 process_format,
285 process_ignored,
286 process_progress,
287 ]
288
289
290 def cmd_capi(filenames=None, *,
291 levels=None,
292 kinds=None,
293 groupby='kind',
294 format='table',
295 showempty=None,
296 ignored=None,
297 track_progress=None,
298 verbosity=VERBOSITY,
299 **kwargs
300 ):
301 render = _capi.get_renderer(format)
302
303 filenames = _files.iter_header_files(filenames, levels=levels)
304 #filenames = (file for file, _ in main_for_filenames(filenames))
305 if track_progress:
306 filenames = track_progress(filenames)
307 items = _capi.iter_capi(filenames)
308 if levels:
309 items = (item for item in items if item.level in levels)
310 if kinds:
311 items = (item for item in items if item.kind in kinds)
312
313 filter = _capi.resolve_filter(ignored)
314 if filter:
315 items = (item for item in items if filter(item, log=lambda msg: logger.log(1, msg)))
316
317 lines = render(
318 items,
319 groupby=groupby,
320 showempty=showempty,
321 verbose=verbosity > VERBOSITY,
322 )
323 print()
324 for line in lines:
325 print(line)
326
327
328 # We do not define any other cmd_*() handlers here,
329 # favoring those defined elsewhere.
330
331 COMMANDS = {
332 'check': (
333 'analyze and fail if the CPython source code has any problems',
334 [_cli_check],
335 cmd_check,
336 ),
337 'analyze': (
338 'report on the state of the CPython source code',
339 [(lambda p: c_analyzer._cli_analyze(p, **FILES_KWARGS))],
340 cmd_analyze,
341 ),
342 'parse': (
343 'parse the CPython source files',
344 [_cli_parse],
345 cmd_parse,
346 ),
347 'data': (
348 'check/manage local data (e.g. known types, ignored vars, caches)',
349 [_cli_data],
350 cmd_data,
351 ),
352 'capi': (
353 'inspect the C-API',
354 [_cli_capi],
355 cmd_capi,
356 ),
357 }
358
359
360 #######################################
361 # the script
362
363 def parse_args(argv=sys.argv[1:], prog=None, *, subset=None):
364 import argparse
365 parser = argparse.ArgumentParser(
366 prog=prog or get_prog(),
367 )
368
369 # if subset == 'check' or subset == ['check']:
370 # if checks is not None:
371 # commands = dict(COMMANDS)
372 # commands['check'] = list(commands['check'])
373 # cli = commands['check'][1][0]
374 # commands['check'][1][0] = (lambda p: cli(p, checks=checks))
375 processors = add_commands_cli(
376 parser,
377 commands=COMMANDS,
378 commonspecs=[
379 add_verbosity_cli,
380 add_traceback_cli,
381 ],
382 subset=subset,
383 )
384
385 args = parser.parse_args(argv)
386 ns = vars(args)
387
388 cmd = ns.pop('cmd')
389
390 verbosity, traceback_cm = process_args_by_key(
391 args,
392 argv,
393 processors[cmd],
394 ['verbosity', 'traceback_cm'],
395 )
396 if cmd != 'parse':
397 # "verbosity" is sent to the commands, so we put it back.
398 args.verbosity = verbosity
399
400 return cmd, ns, verbosity, traceback_cm
401
402
403 def main(cmd, cmd_kwargs):
404 try:
405 run_cmd = COMMANDS[cmd][-1]
406 except KeyError:
407 raise ValueError(f'unsupported cmd {cmd!r}')
408 run_cmd(**cmd_kwargs)
409
410
411 if __name__ == '__main__':
412 cmd, cmd_kwargs, verbosity, traceback_cm = parse_args()
413 configure_logger(verbosity)
414 with traceback_cm:
415 main(cmd, cmd_kwargs)