1 from collections import namedtuple
2 import enum
3 import os.path
4 import re
5
6 from c_common import fsutil
7 from c_common.clsutil import classonly
8 import c_common.misc as _misc
9 import c_common.strutil as _strutil
10 import c_common.tables as _tables
11 from .parser._regexes import SIMPLE_TYPE, _STORAGE
12
13
14 FIXED_TYPE = _misc.Labeled('FIXED_TYPE')
15
16 STORAGE = frozenset(_STORAGE)
17
18
19 #############################
20 # kinds
21
22 @enum.unique
23 class ESC[4;38;5;81mKIND(ESC[4;38;5;149menumESC[4;38;5;149m.ESC[4;38;5;149mEnum):
24
25 # XXX Use these in the raw parser code.
26 TYPEDEF = 'typedef'
27 STRUCT = 'struct'
28 UNION = 'union'
29 ENUM = 'enum'
30 FUNCTION = 'function'
31 VARIABLE = 'variable'
32 STATEMENT = 'statement'
33
34 @classonly
35 def _from_raw(cls, raw):
36 if raw is None:
37 return None
38 elif isinstance(raw, cls):
39 return raw
40 elif type(raw) is str:
41 # We could use cls[raw] for the upper-case form,
42 # but there's no need to go to the trouble.
43 return cls(raw.lower())
44 else:
45 raise NotImplementedError(raw)
46
47 @classonly
48 def by_priority(cls, group=None):
49 if group is None:
50 return cls._ALL_BY_PRIORITY.copy()
51 elif group == 'type':
52 return cls._TYPE_DECLS_BY_PRIORITY.copy()
53 elif group == 'decl':
54 return cls._ALL_DECLS_BY_PRIORITY.copy()
55 elif isinstance(group, str):
56 raise NotImplementedError(group)
57 else:
58 # XXX Treat group as a set of kinds & return in priority order?
59 raise NotImplementedError(group)
60
61 @classonly
62 def is_type_decl(cls, kind):
63 if kind in cls.TYPES:
64 return True
65 if not isinstance(kind, cls):
66 raise TypeError(f'expected KIND, got {kind!r}')
67 return False
68
69 @classonly
70 def is_decl(cls, kind):
71 if kind in cls.DECLS:
72 return True
73 if not isinstance(kind, cls):
74 raise TypeError(f'expected KIND, got {kind!r}')
75 return False
76
77 @classonly
78 def get_group(cls, kind, *, groups=None):
79 if not isinstance(kind, cls):
80 raise TypeError(f'expected KIND, got {kind!r}')
81 if groups is None:
82 groups = ['type']
83 elif not groups:
84 groups = ()
85 elif isinstance(groups, str):
86 group = groups
87 if group not in cls._GROUPS:
88 raise ValueError(f'unsupported group {group!r}')
89 groups = [group]
90 else:
91 unsupported = [g for g in groups if g not in cls._GROUPS]
92 if unsupported:
93 raise ValueError(f'unsupported groups {", ".join(repr(unsupported))}')
94 for group in groups:
95 if kind in cls._GROUPS[group]:
96 return group
97 else:
98 return kind.value
99
100 @classonly
101 def resolve_group(cls, group):
102 if isinstance(group, cls):
103 return {group}
104 elif isinstance(group, str):
105 try:
106 return cls._GROUPS[group].copy()
107 except KeyError:
108 raise ValueError(f'unsupported group {group!r}')
109 else:
110 resolved = set()
111 for gr in group:
112 resolve.update(cls.resolve_group(gr))
113 return resolved
114 #return {*cls.resolve_group(g) for g in group}
115
116
117 KIND._TYPE_DECLS_BY_PRIORITY = [
118 # These are in preferred order.
119 KIND.TYPEDEF,
120 KIND.STRUCT,
121 KIND.UNION,
122 KIND.ENUM,
123 ]
124 KIND._ALL_DECLS_BY_PRIORITY = [
125 # These are in preferred order.
126 *KIND._TYPE_DECLS_BY_PRIORITY,
127 KIND.FUNCTION,
128 KIND.VARIABLE,
129 ]
130 KIND._ALL_BY_PRIORITY = [
131 # These are in preferred order.
132 *KIND._ALL_DECLS_BY_PRIORITY,
133 KIND.STATEMENT,
134 ]
135
136 KIND.TYPES = frozenset(KIND._TYPE_DECLS_BY_PRIORITY)
137 KIND.DECLS = frozenset(KIND._ALL_DECLS_BY_PRIORITY)
138 KIND._GROUPS = {
139 'type': KIND.TYPES,
140 'decl': KIND.DECLS,
141 }
142 KIND._GROUPS.update((k.value, {k}) for k in KIND)
143
144
145 def get_kind_group(item):
146 return KIND.get_group(item.kind)
147
148
149 #############################
150 # low-level
151
152 def _fix_filename(filename, relroot, *,
153 formatted=True,
154 **kwargs):
155 if formatted:
156 fix = fsutil.format_filename
157 else:
158 fix = fsutil.fix_filename
159 return fix(filename, relroot=relroot, **kwargs)
160
161
162 class ESC[4;38;5;81mFileInfo(ESC[4;38;5;149mnamedtuple('FileInfo', 'filename lno')):
163 @classmethod
164 def from_raw(cls, raw):
165 if isinstance(raw, cls):
166 return raw
167 elif isinstance(raw, tuple):
168 return cls(*raw)
169 elif not raw:
170 return None
171 elif isinstance(raw, str):
172 return cls(raw, -1)
173 else:
174 raise TypeError(f'unsupported "raw": {raw:!r}')
175
176 def __str__(self):
177 return self.filename
178
179 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
180 filename = _fix_filename(self.filename, relroot, **kwargs)
181 if filename == self.filename:
182 return self
183 return self._replace(filename=filename)
184
185
186 class ESC[4;38;5;81mSourceLine(ESC[4;38;5;149mnamedtuple('Line', 'file kind data conditions')):
187 KINDS = (
188 #'directive', # data is ...
189 'source', # "data" is the line
190 #'comment', # "data" is the text, including comment markers
191 )
192
193 @property
194 def filename(self):
195 return self.file.filename
196
197 @property
198 def lno(self):
199 return self.file.lno
200
201
202 class ESC[4;38;5;81mDeclID(ESC[4;38;5;149mnamedtuple('DeclID', 'filename funcname name')):
203 """The globally-unique identifier for a declaration."""
204
205 @classmethod
206 def from_row(cls, row, **markers):
207 row = _tables.fix_row(row, **markers)
208 return cls(*row)
209
210 # We have to provde _make() becaose we implemented __new__().
211
212 @classmethod
213 def _make(cls, iterable):
214 try:
215 return cls(*iterable)
216 except Exception:
217 super()._make(iterable)
218 raise # re-raise
219
220 def __new__(cls, filename, funcname, name):
221 self = super().__new__(
222 cls,
223 filename=str(filename) if filename else None,
224 funcname=str(funcname) if funcname else None,
225 name=str(name) if name else None,
226 )
227 self._compare = tuple(v or '' for v in self)
228 return self
229
230 def __hash__(self):
231 return super().__hash__()
232
233 def __eq__(self, other):
234 try:
235 other = tuple(v or '' for v in other)
236 except TypeError:
237 return NotImplemented
238 return self._compare == other
239
240 def __gt__(self, other):
241 try:
242 other = tuple(v or '' for v in other)
243 except TypeError:
244 return NotImplemented
245 return self._compare > other
246
247 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
248 filename = _fix_filename(self.filename, relroot, **kwargs)
249 if filename == self.filename:
250 return self
251 return self._replace(filename=filename)
252
253
254 class ESC[4;38;5;81mParsedItem(ESC[4;38;5;149mnamedtuple('ParsedItem', 'file kind parent name data')):
255
256 @classmethod
257 def from_raw(cls, raw):
258 if isinstance(raw, cls):
259 return raw
260 elif isinstance(raw, tuple):
261 return cls(*raw)
262 else:
263 raise TypeError(f'unsupported "raw": {raw:!r}')
264
265 @classmethod
266 def from_row(cls, row, columns=None):
267 if not columns:
268 colnames = 'filename funcname name kind data'.split()
269 else:
270 colnames = list(columns)
271 for i, column in enumerate(colnames):
272 if column == 'file':
273 colnames[i] = 'filename'
274 elif column == 'funcname':
275 colnames[i] = 'parent'
276 if len(row) != len(set(colnames)):
277 raise NotImplementedError(columns, row)
278 kwargs = {}
279 for column, value in zip(colnames, row):
280 if column == 'filename':
281 kwargs['file'] = FileInfo.from_raw(value)
282 elif column == 'kind':
283 kwargs['kind'] = KIND(value)
284 elif column in cls._fields:
285 kwargs[column] = value
286 else:
287 raise NotImplementedError(column)
288 return cls(**kwargs)
289
290 @property
291 def id(self):
292 try:
293 return self._id
294 except AttributeError:
295 if self.kind is KIND.STATEMENT:
296 self._id = None
297 else:
298 self._id = DeclID(str(self.file), self.funcname, self.name)
299 return self._id
300
301 @property
302 def filename(self):
303 if not self.file:
304 return None
305 return self.file.filename
306
307 @property
308 def lno(self):
309 if not self.file:
310 return -1
311 return self.file.lno
312
313 @property
314 def funcname(self):
315 if not self.parent:
316 return None
317 if type(self.parent) is str:
318 return self.parent
319 else:
320 return self.parent.name
321
322 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
323 fixed = self.file.fix_filename(relroot, **kwargs)
324 if fixed == self.file:
325 return self
326 return self._replace(file=fixed)
327
328 def as_row(self, columns=None):
329 if not columns:
330 columns = self._fields
331 row = []
332 for column in columns:
333 if column == 'file':
334 value = self.filename
335 elif column == 'kind':
336 value = self.kind.value
337 elif column == 'data':
338 value = self._render_data()
339 else:
340 value = getattr(self, column)
341 row.append(value)
342 return row
343
344 def _render_data(self):
345 if not self.data:
346 return None
347 elif isinstance(self.data, str):
348 return self.data
349 else:
350 # XXX
351 raise NotImplementedError
352
353
354 def _get_vartype(data):
355 try:
356 vartype = dict(data['vartype'])
357 except KeyError:
358 vartype = dict(data)
359 storage = data.get('storage')
360 else:
361 storage = data.get('storage') or vartype.get('storage')
362 del vartype['storage']
363 return storage, vartype
364
365
366 def get_parsed_vartype(decl):
367 kind = getattr(decl, 'kind', None)
368 if isinstance(decl, ParsedItem):
369 storage, vartype = _get_vartype(decl.data)
370 typequal = vartype['typequal']
371 typespec = vartype['typespec']
372 abstract = vartype['abstract']
373 elif isinstance(decl, dict):
374 kind = decl.get('kind')
375 storage, vartype = _get_vartype(decl)
376 typequal = vartype['typequal']
377 typespec = vartype['typespec']
378 abstract = vartype['abstract']
379 elif isinstance(decl, VarType):
380 storage = None
381 typequal, typespec, abstract = decl
382 elif isinstance(decl, TypeDef):
383 storage = None
384 typequal, typespec, abstract = decl.vartype
385 elif isinstance(decl, Variable):
386 storage = decl.storage
387 typequal, typespec, abstract = decl.vartype
388 elif isinstance(decl, Function):
389 storage = decl.storage
390 typequal, typespec, abstract = decl.signature.returntype
391 elif isinstance(decl, str):
392 vartype, storage = VarType.from_str(decl)
393 typequal, typespec, abstract = vartype
394 else:
395 raise NotImplementedError(decl)
396 return kind, storage, typequal, typespec, abstract
397
398
399 def get_default_storage(decl):
400 if decl.kind not in (KIND.VARIABLE, KIND.FUNCTION):
401 return None
402 return 'extern' if decl.parent is None else 'auto'
403
404
405 def get_effective_storage(decl, *, default=None):
406 # Note that "static" limits access to just that C module
407 # and "extern" (the default for module-level) allows access
408 # outside the C module.
409 if default is None:
410 default = get_default_storage(decl)
411 if default is None:
412 return None
413 try:
414 storage = decl.storage
415 except AttributeError:
416 storage, _ = _get_vartype(decl.data)
417 return storage or default
418
419
420 #############################
421 # high-level
422
423 class ESC[4;38;5;81mHighlevelParsedItem:
424
425 kind = None
426
427 FIELDS = ('file', 'parent', 'name', 'data')
428
429 @classmethod
430 def from_parsed(cls, parsed):
431 if parsed.kind is not cls.kind:
432 raise TypeError(f'kind mismatch ({parsed.kind.value} != {cls.kind.value})')
433 data, extra = cls._resolve_data(parsed.data)
434 self = cls(
435 cls._resolve_file(parsed),
436 parsed.name,
437 data,
438 cls._resolve_parent(parsed) if parsed.parent else None,
439 **extra or {}
440 )
441 self._parsed = parsed
442 return self
443
444 @classmethod
445 def _resolve_file(cls, parsed):
446 fileinfo = FileInfo.from_raw(parsed.file)
447 if not fileinfo:
448 raise NotImplementedError(parsed)
449 return fileinfo
450
451 @classmethod
452 def _resolve_data(cls, data):
453 return data, None
454
455 @classmethod
456 def _raw_data(cls, data, extra):
457 if isinstance(data, str):
458 return data
459 else:
460 raise NotImplementedError(data)
461
462 @classmethod
463 def _data_as_row(cls, data, extra, colnames):
464 row = {}
465 for colname in colnames:
466 if colname in row:
467 continue
468 rendered = cls._render_data_row_item(colname, data, extra)
469 if rendered is iter(rendered):
470 rendered, = rendered
471 row[colname] = rendered
472 return row
473
474 @classmethod
475 def _render_data_row_item(cls, colname, data, extra):
476 if colname == 'data':
477 return str(data)
478 else:
479 return None
480
481 @classmethod
482 def _render_data_row(cls, fmt, data, extra, colnames):
483 if fmt != 'row':
484 raise NotImplementedError
485 datarow = cls._data_as_row(data, extra, colnames)
486 unresolved = [c for c, v in datarow.items() if v is None]
487 if unresolved:
488 raise NotImplementedError(unresolved)
489 for colname, value in datarow.items():
490 if type(value) != str:
491 if colname == 'kind':
492 datarow[colname] = value.value
493 else:
494 datarow[colname] = str(value)
495 return datarow
496
497 @classmethod
498 def _render_data(cls, fmt, data, extra):
499 row = cls._render_data_row(fmt, data, extra, ['data'])
500 yield ' '.join(row.values())
501
502 @classmethod
503 def _resolve_parent(cls, parsed, *, _kind=None):
504 fileinfo = FileInfo(parsed.file.filename, -1)
505 if isinstance(parsed.parent, str):
506 if parsed.parent.isidentifier():
507 name = parsed.parent
508 else:
509 # XXX It could be something like "<kind> <name>".
510 raise NotImplementedError(repr(parsed.parent))
511 parent = ParsedItem(fileinfo, _kind, None, name, None)
512 elif type(parsed.parent) is tuple:
513 # XXX It could be something like (kind, name).
514 raise NotImplementedError(repr(parsed.parent))
515 else:
516 return parsed.parent
517 Parent = KIND_CLASSES.get(_kind, Declaration)
518 return Parent.from_parsed(parent)
519
520 @classmethod
521 def _parse_columns(cls, columns):
522 colnames = {} # {requested -> actual}
523 columns = list(columns or cls.FIELDS)
524 datacolumns = []
525 for i, colname in enumerate(columns):
526 if colname == 'file':
527 columns[i] = 'filename'
528 colnames['file'] = 'filename'
529 elif colname == 'lno':
530 columns[i] = 'line'
531 colnames['lno'] = 'line'
532 elif colname in ('filename', 'line'):
533 colnames[colname] = colname
534 elif colname == 'data':
535 datacolumns.append(colname)
536 colnames[colname] = None
537 elif colname in cls.FIELDS or colname == 'kind':
538 colnames[colname] = colname
539 else:
540 datacolumns.append(colname)
541 colnames[colname] = None
542 return columns, datacolumns, colnames
543
544 def __init__(self, file, name, data, parent=None, *,
545 _extra=None,
546 _shortkey=None,
547 _key=None,
548 ):
549 self.file = file
550 self.parent = parent or None
551 self.name = name
552 self.data = data
553 self._extra = _extra or {}
554 self._shortkey = _shortkey
555 self._key = _key
556
557 def __repr__(self):
558 args = [f'{n}={getattr(self, n)!r}'
559 for n in ['file', 'name', 'data', 'parent', *(self._extra or ())]]
560 return f'{type(self).__name__}({", ".join(args)})'
561
562 def __str__(self):
563 try:
564 return self._str
565 except AttributeError:
566 self._str = next(self.render())
567 return self._str
568
569 def __getattr__(self, name):
570 try:
571 return self._extra[name]
572 except KeyError:
573 raise AttributeError(name)
574
575 def __hash__(self):
576 return hash(self._key)
577
578 def __eq__(self, other):
579 if isinstance(other, HighlevelParsedItem):
580 return self._key == other._key
581 elif type(other) is tuple:
582 return self._key == other
583 else:
584 return NotImplemented
585
586 def __gt__(self, other):
587 if isinstance(other, HighlevelParsedItem):
588 return self._key > other._key
589 elif type(other) is tuple:
590 return self._key > other
591 else:
592 return NotImplemented
593
594 @property
595 def id(self):
596 return self.parsed.id
597
598 @property
599 def shortkey(self):
600 return self._shortkey
601
602 @property
603 def key(self):
604 return self._key
605
606 @property
607 def filename(self):
608 if not self.file:
609 return None
610 return self.file.filename
611
612 @property
613 def parsed(self):
614 try:
615 return self._parsed
616 except AttributeError:
617 parent = self.parent
618 if parent is not None and not isinstance(parent, str):
619 parent = parent.name
620 self._parsed = ParsedItem(
621 self.file,
622 self.kind,
623 parent,
624 self.name,
625 self._raw_data(),
626 )
627 return self._parsed
628
629 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
630 if self.file:
631 self.file = self.file.fix_filename(relroot, **kwargs)
632 return self
633
634 def as_rowdata(self, columns=None):
635 columns, datacolumns, colnames = self._parse_columns(columns)
636 return self._as_row(colnames, datacolumns, self._data_as_row)
637
638 def render_rowdata(self, columns=None):
639 columns, datacolumns, colnames = self._parse_columns(columns)
640 def data_as_row(data, ext, cols):
641 return self._render_data_row('row', data, ext, cols)
642 rowdata = self._as_row(colnames, datacolumns, data_as_row)
643 for column, value in rowdata.items():
644 colname = colnames.get(column)
645 if not colname:
646 continue
647 if column == 'kind':
648 value = value.value
649 else:
650 if column == 'parent':
651 if self.parent:
652 value = f'({self.parent.kind.value} {self.parent.name})'
653 if not value:
654 value = '-'
655 elif type(value) is VarType:
656 value = repr(str(value))
657 else:
658 value = str(value)
659 rowdata[column] = value
660 return rowdata
661
662 def _as_row(self, colnames, datacolumns, data_as_row):
663 try:
664 data = data_as_row(self.data, self._extra, datacolumns)
665 except NotImplementedError:
666 data = None
667 row = data or {}
668 for column, colname in colnames.items():
669 if colname == 'filename':
670 value = self.file.filename if self.file else None
671 elif colname == 'line':
672 value = self.file.lno if self.file else None
673 elif colname is None:
674 value = getattr(self, column, None)
675 else:
676 value = getattr(self, colname, None)
677 row.setdefault(column, value)
678 return row
679
680 def render(self, fmt='line'):
681 fmt = fmt or 'line'
682 try:
683 render = _FORMATS[fmt]
684 except KeyError:
685 raise TypeError(f'unsupported fmt {fmt!r}')
686 try:
687 data = self._render_data(fmt, self.data, self._extra)
688 except NotImplementedError:
689 data = '-'
690 yield from render(self, data)
691
692
693 ### formats ###
694
695 def _fmt_line(parsed, data=None):
696 parts = [
697 f'<{parsed.kind.value}>',
698 ]
699 parent = ''
700 if parsed.parent:
701 parent = parsed.parent
702 if not isinstance(parent, str):
703 if parent.kind is KIND.FUNCTION:
704 parent = f'{parent.name}()'
705 else:
706 parent = parent.name
707 name = f'<{parent}>.{parsed.name}'
708 else:
709 name = parsed.name
710 if data is None:
711 data = parsed.data
712 elif data is iter(data):
713 data, = data
714 parts.extend([
715 name,
716 f'<{data}>' if data else '-',
717 f'({str(parsed.file or "<unknown file>")})',
718 ])
719 yield '\t'.join(parts)
720
721
722 def _fmt_full(parsed, data=None):
723 if parsed.kind is KIND.VARIABLE and parsed.parent:
724 prefix = 'local '
725 suffix = f' ({parsed.parent.name})'
726 else:
727 # XXX Show other prefixes (e.g. global, public)
728 prefix = suffix = ''
729 yield f'{prefix}{parsed.kind.value} {parsed.name!r}{suffix}'
730 for column, info in parsed.render_rowdata().items():
731 if column == 'kind':
732 continue
733 if column == 'name':
734 continue
735 if column == 'parent' and parsed.kind is not KIND.VARIABLE:
736 continue
737 if column == 'data':
738 if parsed.kind in (KIND.STRUCT, KIND.UNION):
739 column = 'members'
740 elif parsed.kind is KIND.ENUM:
741 column = 'enumerators'
742 elif parsed.kind is KIND.STATEMENT:
743 column = 'text'
744 data, = data
745 else:
746 column = 'signature'
747 data, = data
748 if not data:
749 # yield f'\t{column}:\t-'
750 continue
751 elif isinstance(data, str):
752 yield f'\t{column}:\t{data!r}'
753 else:
754 yield f'\t{column}:'
755 for line in data:
756 yield f'\t\t- {line}'
757 else:
758 yield f'\t{column}:\t{info}'
759
760
761 _FORMATS = {
762 'raw': (lambda v, _d: [repr(v)]),
763 'brief': _fmt_line,
764 'line': _fmt_line,
765 'full': _fmt_full,
766 }
767
768
769 ### declarations ##
770
771 class ESC[4;38;5;81mDeclaration(ESC[4;38;5;149mHighlevelParsedItem):
772
773 @classmethod
774 def from_row(cls, row, **markers):
775 fixed = tuple(_tables.fix_row(row, **markers))
776 if cls is Declaration:
777 _, _, _, kind, _ = fixed
778 sub = KIND_CLASSES.get(KIND(kind))
779 if not sub or not issubclass(sub, Declaration):
780 raise TypeError(f'unsupported kind, got {row!r}')
781 else:
782 sub = cls
783 return sub._from_row(fixed)
784
785 @classmethod
786 def _from_row(cls, row):
787 filename, funcname, name, kind, data = row
788 kind = KIND._from_raw(kind)
789 if kind is not cls.kind:
790 raise TypeError(f'expected kind {cls.kind.value!r}, got {row!r}')
791 fileinfo = FileInfo.from_raw(filename)
792 if isinstance(data, str):
793 data, extra = cls._parse_data(data, fmt='row')
794 if extra:
795 return cls(fileinfo, name, data, funcname, _extra=extra)
796 else:
797 return cls(fileinfo, name, data, funcname)
798
799 @classmethod
800 def _resolve_parent(cls, parsed, *, _kind=None):
801 if _kind is None:
802 raise TypeError(f'{cls.kind.value} declarations do not have parents ({parsed})')
803 return super()._resolve_parent(parsed, _kind=_kind)
804
805 @classmethod
806 def _render_data(cls, fmt, data, extra):
807 if not data:
808 # XXX There should be some! Forward?
809 yield '???'
810 else:
811 yield from cls._format_data(fmt, data, extra)
812
813 @classmethod
814 def _render_data_row_item(cls, colname, data, extra):
815 if colname == 'data':
816 return cls._format_data('row', data, extra)
817 else:
818 return None
819
820 @classmethod
821 def _format_data(cls, fmt, data, extra):
822 raise NotImplementedError(fmt)
823
824 @classmethod
825 def _parse_data(cls, datastr, fmt=None):
826 """This is the reverse of _render_data."""
827 if not datastr or datastr is _tables.UNKNOWN or datastr == '???':
828 return None, None
829 elif datastr is _tables.EMPTY or datastr == '-':
830 # All the kinds have *something* even it is unknown.
831 raise TypeError('all declarations have data of some sort, got none')
832 else:
833 return cls._unformat_data(datastr, fmt)
834
835 @classmethod
836 def _unformat_data(cls, datastr, fmt=None):
837 raise NotImplementedError(fmt)
838
839
840 class ESC[4;38;5;81mVarType(ESC[4;38;5;149mnamedtuple('VarType', 'typequal typespec abstract')):
841
842 @classmethod
843 def from_str(cls, text):
844 orig = text
845 storage, sep, text = text.strip().partition(' ')
846 if not sep:
847 text = storage
848 storage = None
849 elif storage not in ('auto', 'register', 'static', 'extern'):
850 text = orig
851 storage = None
852 return cls._from_str(text), storage
853
854 @classmethod
855 def _from_str(cls, text):
856 orig = text
857 if text.startswith(('const ', 'volatile ')):
858 typequal, _, text = text.partition(' ')
859 else:
860 typequal = None
861
862 # Extract a series of identifiers/keywords.
863 m = re.match(r"^ *'?([a-zA-Z_]\w*(?:\s+[a-zA-Z_]\w*)*)\s*(.*?)'?\s*$", text)
864 if not m:
865 raise ValueError(f'invalid vartype text {orig!r}')
866 typespec, abstract = m.groups()
867
868 return cls(typequal, typespec, abstract or None)
869
870 def __str__(self):
871 parts = []
872 if self.qualifier:
873 parts.append(self.qualifier)
874 parts.append(self.spec + (self.abstract or ''))
875 return ' '.join(parts)
876
877 @property
878 def qualifier(self):
879 return self.typequal
880
881 @property
882 def spec(self):
883 return self.typespec
884
885
886 class ESC[4;38;5;81mVariable(ESC[4;38;5;149mDeclaration):
887 kind = KIND.VARIABLE
888
889 @classmethod
890 def _resolve_parent(cls, parsed):
891 return super()._resolve_parent(parsed, _kind=KIND.FUNCTION)
892
893 @classmethod
894 def _resolve_data(cls, data):
895 if not data:
896 return None, None
897 storage, vartype = _get_vartype(data)
898 return VarType(**vartype), {'storage': storage}
899
900 @classmethod
901 def _raw_data(self, data, extra):
902 vartype = data._asdict()
903 return {
904 'storage': extra['storage'],
905 'vartype': vartype,
906 }
907
908 @classmethod
909 def _format_data(cls, fmt, data, extra):
910 storage = extra.get('storage')
911 text = f'{storage} {data}' if storage else str(data)
912 if fmt in ('line', 'brief'):
913 yield text
914 #elif fmt == 'full':
915 elif fmt == 'row':
916 yield text
917 else:
918 raise NotImplementedError(fmt)
919
920 @classmethod
921 def _unformat_data(cls, datastr, fmt=None):
922 if fmt in ('line', 'brief'):
923 vartype, storage = VarType.from_str(datastr)
924 return vartype, {'storage': storage}
925 #elif fmt == 'full':
926 elif fmt == 'row':
927 vartype, storage = VarType.from_str(datastr)
928 return vartype, {'storage': storage}
929 else:
930 raise NotImplementedError(fmt)
931
932 def __init__(self, file, name, data, parent=None, storage=None):
933 super().__init__(file, name, data, parent,
934 _extra={'storage': storage or None},
935 _shortkey=f'({parent.name}).{name}' if parent else name,
936 _key=(str(file),
937 # Tilde comes after all other ascii characters.
938 f'~{parent or ""}~',
939 name,
940 ),
941 )
942 if storage:
943 if storage not in STORAGE:
944 # The parser must need an update.
945 raise NotImplementedError(storage)
946 # Otherwise we trust the compiler to have validated it.
947
948 @property
949 def vartype(self):
950 return self.data
951
952
953 class ESC[4;38;5;81mSignature(ESC[4;38;5;149mnamedtuple('Signature', 'params returntype inline isforward')):
954
955 @classmethod
956 def from_str(cls, text):
957 orig = text
958 storage, sep, text = text.strip().partition(' ')
959 if not sep:
960 text = storage
961 storage = None
962 elif storage not in ('auto', 'register', 'static', 'extern'):
963 text = orig
964 storage = None
965 return cls._from_str(text), storage
966
967 @classmethod
968 def _from_str(cls, text):
969 orig = text
970 inline, sep, text = text.partition('|')
971 if not sep:
972 text = inline
973 inline = None
974
975 isforward = False
976 if text.endswith(';'):
977 text = text[:-1]
978 isforward = True
979 elif text.endswith('{}'):
980 text = text[:-2]
981
982 index = text.rindex('(')
983 if index < 0:
984 raise ValueError(f'bad signature text {orig!r}')
985 params = text[index:]
986 while params.count('(') <= params.count(')'):
987 index = text.rindex('(', 0, index)
988 if index < 0:
989 raise ValueError(f'bad signature text {orig!r}')
990 params = text[index:]
991 text = text[:index]
992
993 returntype = VarType._from_str(text.rstrip())
994
995 return cls(params, returntype, inline, isforward)
996
997 def __str__(self):
998 parts = []
999 if self.inline:
1000 parts.extend([
1001 self.inline,
1002 '|',
1003 ])
1004 parts.extend([
1005 str(self.returntype),
1006 self.params,
1007 ';' if self.isforward else '{}',
1008 ])
1009 return ' '.join(parts)
1010
1011 @property
1012 def returns(self):
1013 return self.returntype
1014
1015
1016 class ESC[4;38;5;81mFunction(ESC[4;38;5;149mDeclaration):
1017 kind = KIND.FUNCTION
1018
1019 @classmethod
1020 def _resolve_data(cls, data):
1021 if not data:
1022 return None, None
1023 kwargs = dict(data)
1024 returntype = dict(data['returntype'])
1025 del returntype['storage']
1026 kwargs['returntype'] = VarType(**returntype)
1027 storage = kwargs.pop('storage')
1028 return Signature(**kwargs), {'storage': storage}
1029
1030 @classmethod
1031 def _raw_data(self, data):
1032 # XXX finish!
1033 return data
1034
1035 @classmethod
1036 def _format_data(cls, fmt, data, extra):
1037 storage = extra.get('storage')
1038 text = f'{storage} {data}' if storage else str(data)
1039 if fmt in ('line', 'brief'):
1040 yield text
1041 #elif fmt == 'full':
1042 elif fmt == 'row':
1043 yield text
1044 else:
1045 raise NotImplementedError(fmt)
1046
1047 @classmethod
1048 def _unformat_data(cls, datastr, fmt=None):
1049 if fmt in ('line', 'brief'):
1050 sig, storage = Signature.from_str(sig)
1051 return sig, {'storage': storage}
1052 #elif fmt == 'full':
1053 elif fmt == 'row':
1054 sig, storage = Signature.from_str(sig)
1055 return sig, {'storage': storage}
1056 else:
1057 raise NotImplementedError(fmt)
1058
1059 def __init__(self, file, name, data, parent=None, storage=None):
1060 super().__init__(file, name, data, parent, _extra={'storage': storage})
1061 self._shortkey = f'~{name}~ {self.data}'
1062 self._key = (
1063 str(file),
1064 self._shortkey,
1065 )
1066
1067 @property
1068 def signature(self):
1069 return self.data
1070
1071
1072 class ESC[4;38;5;81mTypeDeclaration(ESC[4;38;5;149mDeclaration):
1073
1074 def __init__(self, file, name, data, parent=None, *, _shortkey=None):
1075 if not _shortkey:
1076 _shortkey = f'{self.kind.value} {name}'
1077 super().__init__(file, name, data, parent,
1078 _shortkey=_shortkey,
1079 _key=(
1080 str(file),
1081 _shortkey,
1082 ),
1083 )
1084
1085
1086 class ESC[4;38;5;81mPOTSType(ESC[4;38;5;149mTypeDeclaration):
1087
1088 def __init__(self, name):
1089 _file = _data = _parent = None
1090 super().__init__(_file, name, _data, _parent, _shortkey=name)
1091
1092
1093 class ESC[4;38;5;81mFuncPtr(ESC[4;38;5;149mTypeDeclaration):
1094
1095 def __init__(self, vartype):
1096 _file = _name = _parent = None
1097 data = vartype
1098 self.vartype = vartype
1099 super().__init__(_file, _name, data, _parent, _shortkey=f'<{vartype}>')
1100
1101
1102 class ESC[4;38;5;81mTypeDef(ESC[4;38;5;149mTypeDeclaration):
1103 kind = KIND.TYPEDEF
1104
1105 @classmethod
1106 def _resolve_data(cls, data):
1107 if not data:
1108 raise NotImplementedError(data)
1109 vartype = dict(data)
1110 del vartype['storage']
1111 return VarType(**vartype), None
1112
1113 @classmethod
1114 def _raw_data(self, data):
1115 # XXX finish!
1116 return data
1117
1118 @classmethod
1119 def _format_data(cls, fmt, data, extra):
1120 text = str(data)
1121 if fmt in ('line', 'brief'):
1122 yield text
1123 elif fmt == 'full':
1124 yield text
1125 elif fmt == 'row':
1126 yield text
1127 else:
1128 raise NotImplementedError(fmt)
1129
1130 @classmethod
1131 def _unformat_data(cls, datastr, fmt=None):
1132 if fmt in ('line', 'brief'):
1133 vartype, _ = VarType.from_str(datastr)
1134 return vartype, None
1135 #elif fmt == 'full':
1136 elif fmt == 'row':
1137 vartype, _ = VarType.from_str(datastr)
1138 return vartype, None
1139 else:
1140 raise NotImplementedError(fmt)
1141
1142 def __init__(self, file, name, data, parent=None):
1143 super().__init__(file, name, data, parent, _shortkey=name)
1144
1145 @property
1146 def vartype(self):
1147 return self.data
1148
1149
1150 class ESC[4;38;5;81mMember(ESC[4;38;5;149mnamedtuple('Member', 'name vartype size')):
1151
1152 @classmethod
1153 def from_data(cls, raw, index):
1154 name = raw.name if raw.name else index
1155 vartype = size = None
1156 if type(raw.data) is int:
1157 size = raw.data
1158 elif isinstance(raw.data, str):
1159 size = int(raw.data)
1160 elif raw.data:
1161 vartype = dict(raw.data)
1162 del vartype['storage']
1163 if 'size' in vartype:
1164 size = vartype.pop('size')
1165 if isinstance(size, str) and size.isdigit():
1166 size = int(size)
1167 vartype = VarType(**vartype)
1168 return cls(name, vartype, size)
1169
1170 @classmethod
1171 def from_str(cls, text):
1172 name, _, vartype = text.partition(': ')
1173 if name.startswith('#'):
1174 name = int(name[1:])
1175 if vartype.isdigit():
1176 size = int(vartype)
1177 vartype = None
1178 else:
1179 vartype, _ = VarType.from_str(vartype)
1180 size = None
1181 return cls(name, vartype, size)
1182
1183 def __str__(self):
1184 name = self.name if isinstance(self.name, str) else f'#{self.name}'
1185 return f'{name}: {self.vartype or self.size}'
1186
1187
1188 class ESC[4;38;5;81m_StructUnion(ESC[4;38;5;149mTypeDeclaration):
1189
1190 @classmethod
1191 def _resolve_data(cls, data):
1192 if not data:
1193 # XXX There should be some! Forward?
1194 return None, None
1195 return [Member.from_data(v, i) for i, v in enumerate(data)], None
1196
1197 @classmethod
1198 def _raw_data(self, data):
1199 # XXX finish!
1200 return data
1201
1202 @classmethod
1203 def _format_data(cls, fmt, data, extra):
1204 if fmt in ('line', 'brief'):
1205 members = ', '.join(f'<{m}>' for m in data)
1206 yield f'[{members}]'
1207 elif fmt == 'full':
1208 for member in data:
1209 yield f'{member}'
1210 elif fmt == 'row':
1211 members = ', '.join(f'<{m}>' for m in data)
1212 yield f'[{members}]'
1213 else:
1214 raise NotImplementedError(fmt)
1215
1216 @classmethod
1217 def _unformat_data(cls, datastr, fmt=None):
1218 if fmt in ('line', 'brief'):
1219 members = [Member.from_str(m[1:-1])
1220 for m in datastr[1:-1].split(', ')]
1221 return members, None
1222 #elif fmt == 'full':
1223 elif fmt == 'row':
1224 members = [Member.from_str(m.rstrip('>').lstrip('<'))
1225 for m in datastr[1:-1].split('>, <')]
1226 return members, None
1227 else:
1228 raise NotImplementedError(fmt)
1229
1230 def __init__(self, file, name, data, parent=None):
1231 super().__init__(file, name, data, parent)
1232
1233 @property
1234 def members(self):
1235 return self.data
1236
1237
1238 class ESC[4;38;5;81mStruct(ESC[4;38;5;149m_StructUnion):
1239 kind = KIND.STRUCT
1240
1241
1242 class ESC[4;38;5;81mUnion(ESC[4;38;5;149m_StructUnion):
1243 kind = KIND.UNION
1244
1245
1246 class ESC[4;38;5;81mEnum(ESC[4;38;5;149mTypeDeclaration):
1247 kind = KIND.ENUM
1248
1249 @classmethod
1250 def _resolve_data(cls, data):
1251 if not data:
1252 # XXX There should be some! Forward?
1253 return None, None
1254 enumerators = [e if isinstance(e, str) else e.name
1255 for e in data]
1256 return enumerators, None
1257
1258 @classmethod
1259 def _raw_data(self, data):
1260 # XXX finish!
1261 return data
1262
1263 @classmethod
1264 def _format_data(cls, fmt, data, extra):
1265 if fmt in ('line', 'brief'):
1266 yield repr(data)
1267 elif fmt == 'full':
1268 for enumerator in data:
1269 yield f'{enumerator}'
1270 elif fmt == 'row':
1271 # XXX This won't work with CSV...
1272 yield ','.join(data)
1273 else:
1274 raise NotImplementedError(fmt)
1275
1276 @classmethod
1277 def _unformat_data(cls, datastr, fmt=None):
1278 if fmt in ('line', 'brief'):
1279 return _strutil.unrepr(datastr), None
1280 #elif fmt == 'full':
1281 elif fmt == 'row':
1282 return datastr.split(','), None
1283 else:
1284 raise NotImplementedError(fmt)
1285
1286 def __init__(self, file, name, data, parent=None):
1287 super().__init__(file, name, data, parent)
1288
1289 @property
1290 def enumerators(self):
1291 return self.data
1292
1293
1294 ### statements ###
1295
1296 class ESC[4;38;5;81mStatement(ESC[4;38;5;149mHighlevelParsedItem):
1297 kind = KIND.STATEMENT
1298
1299 @classmethod
1300 def _resolve_data(cls, data):
1301 # XXX finish!
1302 return data, None
1303
1304 @classmethod
1305 def _raw_data(self, data):
1306 # XXX finish!
1307 return data
1308
1309 @classmethod
1310 def _render_data(cls, fmt, data, extra):
1311 # XXX Handle other formats?
1312 return repr(data)
1313
1314 @classmethod
1315 def _parse_data(self, datastr, fmt=None):
1316 # XXX Handle other formats?
1317 return _strutil.unrepr(datastr), None
1318
1319 def __init__(self, file, name, data, parent=None):
1320 super().__init__(file, name, data, parent,
1321 _shortkey=data or '',
1322 _key=(
1323 str(file),
1324 file.lno,
1325 # XXX Only one stmt per line?
1326 ),
1327 )
1328
1329 @property
1330 def text(self):
1331 return self.data
1332
1333
1334 ###
1335
1336 KIND_CLASSES = {cls.kind: cls for cls in [
1337 Variable,
1338 Function,
1339 TypeDef,
1340 Struct,
1341 Union,
1342 Enum,
1343 Statement,
1344 ]}
1345
1346
1347 def resolve_parsed(parsed):
1348 if isinstance(parsed, HighlevelParsedItem):
1349 return parsed
1350 try:
1351 cls = KIND_CLASSES[parsed.kind]
1352 except KeyError:
1353 raise ValueError(f'unsupported kind in {parsed!r}')
1354 return cls.from_parsed(parsed)
1355
1356
1357 def set_flag(item, name, value):
1358 try:
1359 setattr(item, name, value)
1360 except AttributeError:
1361 object.__setattr__(item, name, value)
1362
1363
1364 #############################
1365 # composite
1366
1367 class ESC[4;38;5;81mDeclarations:
1368
1369 @classmethod
1370 def from_decls(cls, decls):
1371 return cls(decls)
1372
1373 @classmethod
1374 def from_parsed(cls, items):
1375 decls = (resolve_parsed(item)
1376 for item in items
1377 if item.kind is not KIND.STATEMENT)
1378 return cls.from_decls(decls)
1379
1380 @classmethod
1381 def _resolve_key(cls, raw):
1382 if isinstance(raw, str):
1383 raw = [raw]
1384 elif isinstance(raw, Declaration):
1385 raw = (
1386 raw.filename if cls._is_public(raw) else None,
1387 # `raw.parent` is always None for types and functions.
1388 raw.parent if raw.kind is KIND.VARIABLE else None,
1389 raw.name,
1390 )
1391
1392 extra = None
1393 if len(raw) == 1:
1394 name, = raw
1395 if name:
1396 name = str(name)
1397 if name.endswith(('.c', '.h')):
1398 # This is only legit as a query.
1399 key = (name, None, None)
1400 else:
1401 key = (None, None, name)
1402 else:
1403 key = (None, None, None)
1404 elif len(raw) == 2:
1405 parent, name = raw
1406 name = str(name)
1407 if isinstance(parent, Declaration):
1408 key = (None, parent.name, name)
1409 elif not parent:
1410 key = (None, None, name)
1411 else:
1412 parent = str(parent)
1413 if parent.endswith(('.c', '.h')):
1414 key = (parent, None, name)
1415 else:
1416 key = (None, parent, name)
1417 else:
1418 key, extra = raw[:3], raw[3:]
1419 filename, funcname, name = key
1420 filename = str(filename) if filename else None
1421 if isinstance(funcname, Declaration):
1422 funcname = funcname.name
1423 else:
1424 funcname = str(funcname) if funcname else None
1425 name = str(name) if name else None
1426 key = (filename, funcname, name)
1427 return key, extra
1428
1429 @classmethod
1430 def _is_public(cls, decl):
1431 # For .c files don't we need info from .h files to make this decision?
1432 # XXX Check for "extern".
1433 # For now we treat all decls a "private" (have filename set).
1434 return False
1435
1436 def __init__(self, decls):
1437 # (file, func, name) -> decl
1438 # "public":
1439 # * (None, None, name)
1440 # "private", "global":
1441 # * (file, None, name)
1442 # "private", "local":
1443 # * (file, func, name)
1444 if hasattr(decls, 'items'):
1445 self._decls = decls
1446 else:
1447 self._decls = {}
1448 self._extend(decls)
1449
1450 # XXX always validate?
1451
1452 def validate(self):
1453 for key, decl in self._decls.items():
1454 if type(key) is not tuple or len(key) != 3:
1455 raise ValueError(f'expected 3-tuple key, got {key!r} (for decl {decl!r})')
1456 filename, funcname, name = key
1457 if not name:
1458 raise ValueError(f'expected name in key, got {key!r} (for decl {decl!r})')
1459 elif type(name) is not str:
1460 raise ValueError(f'expected name in key to be str, got {key!r} (for decl {decl!r})')
1461 # XXX Check filename type?
1462 # XXX Check funcname type?
1463
1464 if decl.kind is KIND.STATEMENT:
1465 raise ValueError(f'expected a declaration, got {decl!r}')
1466
1467 def __repr__(self):
1468 return f'{type(self).__name__}({list(self)})'
1469
1470 def __len__(self):
1471 return len(self._decls)
1472
1473 def __iter__(self):
1474 yield from self._decls
1475
1476 def __getitem__(self, key):
1477 # XXX Be more exact for the 3-tuple case?
1478 if type(key) not in (str, tuple):
1479 raise KeyError(f'unsupported key {key!r}')
1480 resolved, extra = self._resolve_key(key)
1481 if extra:
1482 raise KeyError(f'key must have at most 3 parts, got {key!r}')
1483 if not resolved[2]:
1484 raise ValueError(f'expected name in key, got {key!r}')
1485 try:
1486 return self._decls[resolved]
1487 except KeyError:
1488 if type(key) is tuple and len(key) == 3:
1489 filename, funcname, name = key
1490 else:
1491 filename, funcname, name = resolved
1492 if filename and not filename.endswith(('.c', '.h')):
1493 raise KeyError(f'invalid filename in key {key!r}')
1494 elif funcname and funcname.endswith(('.c', '.h')):
1495 raise KeyError(f'invalid funcname in key {key!r}')
1496 elif name and name.endswith(('.c', '.h')):
1497 raise KeyError(f'invalid name in key {key!r}')
1498 else:
1499 raise # re-raise
1500
1501 @property
1502 def types(self):
1503 return self._find(kind=KIND.TYPES)
1504
1505 @property
1506 def functions(self):
1507 return self._find(None, None, None, KIND.FUNCTION)
1508
1509 @property
1510 def variables(self):
1511 return self._find(None, None, None, KIND.VARIABLE)
1512
1513 def iter_all(self):
1514 yield from self._decls.values()
1515
1516 def get(self, key, default=None):
1517 try:
1518 return self[key]
1519 except KeyError:
1520 return default
1521
1522 #def add_decl(self, decl, key=None):
1523 # decl = _resolve_parsed(decl)
1524 # self._add_decl(decl, key)
1525
1526 def find(self, *key, **explicit):
1527 if not key:
1528 if not explicit:
1529 return iter(self)
1530 return self._find(**explicit)
1531
1532 resolved, extra = self._resolve_key(key)
1533 filename, funcname, name = resolved
1534 if not extra:
1535 kind = None
1536 elif len(extra) == 1:
1537 kind, = extra
1538 else:
1539 raise KeyError(f'key must have at most 4 parts, got {key!r}')
1540
1541 implicit= {}
1542 if filename:
1543 implicit['filename'] = filename
1544 if funcname:
1545 implicit['funcname'] = funcname
1546 if name:
1547 implicit['name'] = name
1548 if kind:
1549 implicit['kind'] = kind
1550 return self._find(**implicit, **explicit)
1551
1552 def _find(self, filename=None, funcname=None, name=None, kind=None):
1553 for decl in self._decls.values():
1554 if filename and decl.filename != filename:
1555 continue
1556 if funcname:
1557 if decl.kind is not KIND.VARIABLE:
1558 continue
1559 if decl.parent.name != funcname:
1560 continue
1561 if name and decl.name != name:
1562 continue
1563 if kind:
1564 kinds = KIND.resolve_group(kind)
1565 if decl.kind not in kinds:
1566 continue
1567 yield decl
1568
1569 def _add_decl(self, decl, key=None):
1570 if key:
1571 if type(key) not in (str, tuple):
1572 raise NotImplementedError((key, decl))
1573 # Any partial key will be turned into a full key, but that
1574 # same partial key will still match a key lookup.
1575 resolved, _ = self._resolve_key(key)
1576 if not resolved[2]:
1577 raise ValueError(f'expected name in key, got {key!r}')
1578 key = resolved
1579 # XXX Also add with the decl-derived key if not the same?
1580 else:
1581 key, _ = self._resolve_key(decl)
1582 self._decls[key] = decl
1583
1584 def _extend(self, decls):
1585 decls = iter(decls)
1586 # Check only the first item.
1587 for decl in decls:
1588 if isinstance(decl, Declaration):
1589 self._add_decl(decl)
1590 # Add the rest without checking.
1591 for decl in decls:
1592 self._add_decl(decl)
1593 elif isinstance(decl, HighlevelParsedItem):
1594 raise NotImplementedError(decl)
1595 else:
1596 try:
1597 key, decl = decl
1598 except ValueError:
1599 raise NotImplementedError(decl)
1600 if not isinstance(decl, Declaration):
1601 raise NotImplementedError(decl)
1602 self._add_decl(decl, key)
1603 # Add the rest without checking.
1604 for key, decl in decls:
1605 self._add_decl(decl, key)
1606 # The iterator will be exhausted at this point.