1 from collections import namedtuple
2 import enum
3 import re
4
5 from c_common import fsutil
6 from c_common.clsutil import classonly
7 import c_common.misc as _misc
8 import c_common.strutil as _strutil
9 import c_common.tables as _tables
10 from .parser._regexes import _STORAGE
11
12
13 FIXED_TYPE = _misc.Labeled('FIXED_TYPE')
14
15 STORAGE = frozenset(_STORAGE)
16
17
18 #############################
19 # kinds
20
21 @enum.unique
22 class ESC[4;38;5;81mKIND(ESC[4;38;5;149menumESC[4;38;5;149m.ESC[4;38;5;149mEnum):
23
24 # XXX Use these in the raw parser code.
25 TYPEDEF = 'typedef'
26 STRUCT = 'struct'
27 UNION = 'union'
28 ENUM = 'enum'
29 FUNCTION = 'function'
30 VARIABLE = 'variable'
31 STATEMENT = 'statement'
32
33 @classonly
34 def _from_raw(cls, raw):
35 if raw is None:
36 return None
37 elif isinstance(raw, cls):
38 return raw
39 elif type(raw) is str:
40 # We could use cls[raw] for the upper-case form,
41 # but there's no need to go to the trouble.
42 return cls(raw.lower())
43 else:
44 raise NotImplementedError(raw)
45
46 @classonly
47 def by_priority(cls, group=None):
48 if group is None:
49 return cls._ALL_BY_PRIORITY.copy()
50 elif group == 'type':
51 return cls._TYPE_DECLS_BY_PRIORITY.copy()
52 elif group == 'decl':
53 return cls._ALL_DECLS_BY_PRIORITY.copy()
54 elif isinstance(group, str):
55 raise NotImplementedError(group)
56 else:
57 # XXX Treat group as a set of kinds & return in priority order?
58 raise NotImplementedError(group)
59
60 @classonly
61 def is_type_decl(cls, kind):
62 if kind in cls.TYPES:
63 return True
64 if not isinstance(kind, cls):
65 raise TypeError(f'expected KIND, got {kind!r}')
66 return False
67
68 @classonly
69 def is_decl(cls, kind):
70 if kind in cls.DECLS:
71 return True
72 if not isinstance(kind, cls):
73 raise TypeError(f'expected KIND, got {kind!r}')
74 return False
75
76 @classonly
77 def get_group(cls, kind, *, groups=None):
78 if not isinstance(kind, cls):
79 raise TypeError(f'expected KIND, got {kind!r}')
80 if groups is None:
81 groups = ['type']
82 elif not groups:
83 groups = ()
84 elif isinstance(groups, str):
85 group = groups
86 if group not in cls._GROUPS:
87 raise ValueError(f'unsupported group {group!r}')
88 groups = [group]
89 else:
90 unsupported = [g for g in groups if g not in cls._GROUPS]
91 if unsupported:
92 raise ValueError(f'unsupported groups {", ".join(repr(unsupported))}')
93 for group in groups:
94 if kind in cls._GROUPS[group]:
95 return group
96 else:
97 return kind.value
98
99 @classonly
100 def resolve_group(cls, group):
101 if isinstance(group, cls):
102 return {group}
103 elif isinstance(group, str):
104 try:
105 return cls._GROUPS[group].copy()
106 except KeyError:
107 raise ValueError(f'unsupported group {group!r}')
108 else:
109 resolved = set()
110 for gr in group:
111 resolve.update(cls.resolve_group(gr))
112 return resolved
113 #return {*cls.resolve_group(g) for g in group}
114
115
116 KIND._TYPE_DECLS_BY_PRIORITY = [
117 # These are in preferred order.
118 KIND.TYPEDEF,
119 KIND.STRUCT,
120 KIND.UNION,
121 KIND.ENUM,
122 ]
123 KIND._ALL_DECLS_BY_PRIORITY = [
124 # These are in preferred order.
125 *KIND._TYPE_DECLS_BY_PRIORITY,
126 KIND.FUNCTION,
127 KIND.VARIABLE,
128 ]
129 KIND._ALL_BY_PRIORITY = [
130 # These are in preferred order.
131 *KIND._ALL_DECLS_BY_PRIORITY,
132 KIND.STATEMENT,
133 ]
134
135 KIND.TYPES = frozenset(KIND._TYPE_DECLS_BY_PRIORITY)
136 KIND.DECLS = frozenset(KIND._ALL_DECLS_BY_PRIORITY)
137 KIND._GROUPS = {
138 'type': KIND.TYPES,
139 'decl': KIND.DECLS,
140 }
141 KIND._GROUPS.update((k.value, {k}) for k in KIND)
142
143
144 def get_kind_group(item):
145 return KIND.get_group(item.kind)
146
147
148 #############################
149 # low-level
150
151 def _fix_filename(filename, relroot, *,
152 formatted=True,
153 **kwargs):
154 if formatted:
155 fix = fsutil.format_filename
156 else:
157 fix = fsutil.fix_filename
158 return fix(filename, relroot=relroot, **kwargs)
159
160
161 class ESC[4;38;5;81mFileInfo(ESC[4;38;5;149mnamedtuple('FileInfo', 'filename lno')):
162 @classmethod
163 def from_raw(cls, raw):
164 if isinstance(raw, cls):
165 return raw
166 elif isinstance(raw, tuple):
167 return cls(*raw)
168 elif not raw:
169 return None
170 elif isinstance(raw, str):
171 return cls(raw, -1)
172 else:
173 raise TypeError(f'unsupported "raw": {raw:!r}')
174
175 def __str__(self):
176 return self.filename
177
178 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
179 filename = _fix_filename(self.filename, relroot, **kwargs)
180 if filename == self.filename:
181 return self
182 return self._replace(filename=filename)
183
184
185 class ESC[4;38;5;81mSourceLine(ESC[4;38;5;149mnamedtuple('Line', 'file kind data conditions')):
186 KINDS = (
187 #'directive', # data is ...
188 'source', # "data" is the line
189 #'comment', # "data" is the text, including comment markers
190 )
191
192 @property
193 def filename(self):
194 return self.file.filename
195
196 @property
197 def lno(self):
198 return self.file.lno
199
200
201 class ESC[4;38;5;81mDeclID(ESC[4;38;5;149mnamedtuple('DeclID', 'filename funcname name')):
202 """The globally-unique identifier for a declaration."""
203
204 @classmethod
205 def from_row(cls, row, **markers):
206 row = _tables.fix_row(row, **markers)
207 return cls(*row)
208
209 # We have to provde _make() becaose we implemented __new__().
210
211 @classmethod
212 def _make(cls, iterable):
213 try:
214 return cls(*iterable)
215 except Exception:
216 super()._make(iterable)
217 raise # re-raise
218
219 def __new__(cls, filename, funcname, name):
220 self = super().__new__(
221 cls,
222 filename=str(filename) if filename else None,
223 funcname=str(funcname) if funcname else None,
224 name=str(name) if name else None,
225 )
226 self._compare = tuple(v or '' for v in self)
227 return self
228
229 def __hash__(self):
230 return super().__hash__()
231
232 def __eq__(self, other):
233 try:
234 other = tuple(v or '' for v in other)
235 except TypeError:
236 return NotImplemented
237 return self._compare == other
238
239 def __gt__(self, other):
240 try:
241 other = tuple(v or '' for v in other)
242 except TypeError:
243 return NotImplemented
244 return self._compare > other
245
246 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
247 filename = _fix_filename(self.filename, relroot, **kwargs)
248 if filename == self.filename:
249 return self
250 return self._replace(filename=filename)
251
252
253 class ESC[4;38;5;81mParsedItem(ESC[4;38;5;149mnamedtuple('ParsedItem', 'file kind parent name data')):
254
255 @classmethod
256 def from_raw(cls, raw):
257 if isinstance(raw, cls):
258 return raw
259 elif isinstance(raw, tuple):
260 return cls(*raw)
261 else:
262 raise TypeError(f'unsupported "raw": {raw:!r}')
263
264 @classmethod
265 def from_row(cls, row, columns=None):
266 if not columns:
267 colnames = 'filename funcname name kind data'.split()
268 else:
269 colnames = list(columns)
270 for i, column in enumerate(colnames):
271 if column == 'file':
272 colnames[i] = 'filename'
273 elif column == 'funcname':
274 colnames[i] = 'parent'
275 if len(row) != len(set(colnames)):
276 raise NotImplementedError(columns, row)
277 kwargs = {}
278 for column, value in zip(colnames, row):
279 if column == 'filename':
280 kwargs['file'] = FileInfo.from_raw(value)
281 elif column == 'kind':
282 kwargs['kind'] = KIND(value)
283 elif column in cls._fields:
284 kwargs[column] = value
285 else:
286 raise NotImplementedError(column)
287 return cls(**kwargs)
288
289 @property
290 def id(self):
291 try:
292 return self._id
293 except AttributeError:
294 if self.kind is KIND.STATEMENT:
295 self._id = None
296 else:
297 self._id = DeclID(str(self.file), self.funcname, self.name)
298 return self._id
299
300 @property
301 def filename(self):
302 if not self.file:
303 return None
304 return self.file.filename
305
306 @property
307 def lno(self):
308 if not self.file:
309 return -1
310 return self.file.lno
311
312 @property
313 def funcname(self):
314 if not self.parent:
315 return None
316 if type(self.parent) is str:
317 return self.parent
318 else:
319 return self.parent.name
320
321 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
322 fixed = self.file.fix_filename(relroot, **kwargs)
323 if fixed == self.file:
324 return self
325 return self._replace(file=fixed)
326
327 def as_row(self, columns=None):
328 if not columns:
329 columns = self._fields
330 row = []
331 for column in columns:
332 if column == 'file':
333 value = self.filename
334 elif column == 'kind':
335 value = self.kind.value
336 elif column == 'data':
337 value = self._render_data()
338 else:
339 value = getattr(self, column)
340 row.append(value)
341 return row
342
343 def _render_data(self):
344 if not self.data:
345 return None
346 elif isinstance(self.data, str):
347 return self.data
348 else:
349 # XXX
350 raise NotImplementedError
351
352
353 def _get_vartype(data):
354 try:
355 vartype = dict(data['vartype'])
356 except KeyError:
357 vartype = dict(data)
358 storage = data.get('storage')
359 else:
360 storage = data.get('storage') or vartype.get('storage')
361 del vartype['storage']
362 return storage, vartype
363
364
365 def get_parsed_vartype(decl):
366 kind = getattr(decl, 'kind', None)
367 if isinstance(decl, ParsedItem):
368 storage, vartype = _get_vartype(decl.data)
369 typequal = vartype['typequal']
370 typespec = vartype['typespec']
371 abstract = vartype['abstract']
372 elif isinstance(decl, dict):
373 kind = decl.get('kind')
374 storage, vartype = _get_vartype(decl)
375 typequal = vartype['typequal']
376 typespec = vartype['typespec']
377 abstract = vartype['abstract']
378 elif isinstance(decl, VarType):
379 storage = None
380 typequal, typespec, abstract = decl
381 elif isinstance(decl, TypeDef):
382 storage = None
383 typequal, typespec, abstract = decl.vartype
384 elif isinstance(decl, Variable):
385 storage = decl.storage
386 typequal, typespec, abstract = decl.vartype
387 elif isinstance(decl, Signature):
388 storage = None
389 typequal, typespec, abstract = decl.returntype
390 elif isinstance(decl, Function):
391 storage = decl.storage
392 typequal, typespec, abstract = decl.signature.returntype
393 elif isinstance(decl, str):
394 vartype, storage = VarType.from_str(decl)
395 typequal, typespec, abstract = vartype
396 else:
397 raise NotImplementedError(decl)
398 return kind, storage, typequal, typespec, abstract
399
400
401 def get_default_storage(decl):
402 if decl.kind not in (KIND.VARIABLE, KIND.FUNCTION):
403 return None
404 return 'extern' if decl.parent is None else 'auto'
405
406
407 def get_effective_storage(decl, *, default=None):
408 # Note that "static" limits access to just that C module
409 # and "extern" (the default for module-level) allows access
410 # outside the C module.
411 if default is None:
412 default = get_default_storage(decl)
413 if default is None:
414 return None
415 try:
416 storage = decl.storage
417 except AttributeError:
418 storage, _ = _get_vartype(decl.data)
419 return storage or default
420
421
422 #############################
423 # high-level
424
425 class ESC[4;38;5;81mHighlevelParsedItem:
426
427 kind = None
428
429 FIELDS = ('file', 'parent', 'name', 'data')
430
431 @classmethod
432 def from_parsed(cls, parsed):
433 if parsed.kind is not cls.kind:
434 raise TypeError(f'kind mismatch ({parsed.kind.value} != {cls.kind.value})')
435 data, extra = cls._resolve_data(parsed.data)
436 self = cls(
437 cls._resolve_file(parsed),
438 parsed.name,
439 data,
440 cls._resolve_parent(parsed) if parsed.parent else None,
441 **extra or {}
442 )
443 self._parsed = parsed
444 return self
445
446 @classmethod
447 def _resolve_file(cls, parsed):
448 fileinfo = FileInfo.from_raw(parsed.file)
449 if not fileinfo:
450 raise NotImplementedError(parsed)
451 return fileinfo
452
453 @classmethod
454 def _resolve_data(cls, data):
455 return data, None
456
457 @classmethod
458 def _raw_data(cls, data, extra):
459 if isinstance(data, str):
460 return data
461 else:
462 raise NotImplementedError(data)
463
464 @classmethod
465 def _data_as_row(cls, data, extra, colnames):
466 row = {}
467 for colname in colnames:
468 if colname in row:
469 continue
470 rendered = cls._render_data_row_item(colname, data, extra)
471 if rendered is iter(rendered):
472 rendered, = rendered
473 row[colname] = rendered
474 return row
475
476 @classmethod
477 def _render_data_row_item(cls, colname, data, extra):
478 if colname == 'data':
479 return str(data)
480 else:
481 return None
482
483 @classmethod
484 def _render_data_row(cls, fmt, data, extra, colnames):
485 if fmt != 'row':
486 raise NotImplementedError
487 datarow = cls._data_as_row(data, extra, colnames)
488 unresolved = [c for c, v in datarow.items() if v is None]
489 if unresolved:
490 raise NotImplementedError(unresolved)
491 for colname, value in datarow.items():
492 if type(value) != str:
493 if colname == 'kind':
494 datarow[colname] = value.value
495 else:
496 datarow[colname] = str(value)
497 return datarow
498
499 @classmethod
500 def _render_data(cls, fmt, data, extra):
501 row = cls._render_data_row(fmt, data, extra, ['data'])
502 yield ' '.join(row.values())
503
504 @classmethod
505 def _resolve_parent(cls, parsed, *, _kind=None):
506 fileinfo = FileInfo(parsed.file.filename, -1)
507 if isinstance(parsed.parent, str):
508 if parsed.parent.isidentifier():
509 name = parsed.parent
510 else:
511 # XXX It could be something like "<kind> <name>".
512 raise NotImplementedError(repr(parsed.parent))
513 parent = ParsedItem(fileinfo, _kind, None, name, None)
514 elif type(parsed.parent) is tuple:
515 # XXX It could be something like (kind, name).
516 raise NotImplementedError(repr(parsed.parent))
517 else:
518 return parsed.parent
519 Parent = KIND_CLASSES.get(_kind, Declaration)
520 return Parent.from_parsed(parent)
521
522 @classmethod
523 def _parse_columns(cls, columns):
524 colnames = {} # {requested -> actual}
525 columns = list(columns or cls.FIELDS)
526 datacolumns = []
527 for i, colname in enumerate(columns):
528 if colname == 'file':
529 columns[i] = 'filename'
530 colnames['file'] = 'filename'
531 elif colname == 'lno':
532 columns[i] = 'line'
533 colnames['lno'] = 'line'
534 elif colname in ('filename', 'line'):
535 colnames[colname] = colname
536 elif colname == 'data':
537 datacolumns.append(colname)
538 colnames[colname] = None
539 elif colname in cls.FIELDS or colname == 'kind':
540 colnames[colname] = colname
541 else:
542 datacolumns.append(colname)
543 colnames[colname] = None
544 return columns, datacolumns, colnames
545
546 def __init__(self, file, name, data, parent=None, *,
547 _extra=None,
548 _shortkey=None,
549 _key=None,
550 ):
551 self.file = file
552 self.parent = parent or None
553 self.name = name
554 self.data = data
555 self._extra = _extra or {}
556 self._shortkey = _shortkey
557 self._key = _key
558
559 def __repr__(self):
560 args = [f'{n}={getattr(self, n)!r}'
561 for n in ['file', 'name', 'data', 'parent', *(self._extra or ())]]
562 return f'{type(self).__name__}({", ".join(args)})'
563
564 def __str__(self):
565 try:
566 return self._str
567 except AttributeError:
568 self._str = next(self.render())
569 return self._str
570
571 def __getattr__(self, name):
572 try:
573 return self._extra[name]
574 except KeyError:
575 raise AttributeError(name)
576
577 def __hash__(self):
578 return hash(self._key)
579
580 def __eq__(self, other):
581 if isinstance(other, HighlevelParsedItem):
582 return self._key == other._key
583 elif type(other) is tuple:
584 return self._key == other
585 else:
586 return NotImplemented
587
588 def __gt__(self, other):
589 if isinstance(other, HighlevelParsedItem):
590 return self._key > other._key
591 elif type(other) is tuple:
592 return self._key > other
593 else:
594 return NotImplemented
595
596 @property
597 def id(self):
598 return self.parsed.id
599
600 @property
601 def shortkey(self):
602 return self._shortkey
603
604 @property
605 def key(self):
606 return self._key
607
608 @property
609 def filename(self):
610 if not self.file:
611 return None
612 return self.file.filename
613
614 @property
615 def parsed(self):
616 try:
617 return self._parsed
618 except AttributeError:
619 parent = self.parent
620 if parent is not None and not isinstance(parent, str):
621 parent = parent.name
622 self._parsed = ParsedItem(
623 self.file,
624 self.kind,
625 parent,
626 self.name,
627 self._raw_data(),
628 )
629 return self._parsed
630
631 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs):
632 if self.file:
633 self.file = self.file.fix_filename(relroot, **kwargs)
634 return self
635
636 def as_rowdata(self, columns=None):
637 columns, datacolumns, colnames = self._parse_columns(columns)
638 return self._as_row(colnames, datacolumns, self._data_as_row)
639
640 def render_rowdata(self, columns=None):
641 columns, datacolumns, colnames = self._parse_columns(columns)
642 def data_as_row(data, ext, cols):
643 return self._render_data_row('row', data, ext, cols)
644 rowdata = self._as_row(colnames, datacolumns, data_as_row)
645 for column, value in rowdata.items():
646 colname = colnames.get(column)
647 if not colname:
648 continue
649 if column == 'kind':
650 value = value.value
651 else:
652 if column == 'parent':
653 if self.parent:
654 value = f'({self.parent.kind.value} {self.parent.name})'
655 if not value:
656 value = '-'
657 elif type(value) is VarType:
658 value = repr(str(value))
659 else:
660 value = str(value)
661 rowdata[column] = value
662 return rowdata
663
664 def _as_row(self, colnames, datacolumns, data_as_row):
665 try:
666 data = data_as_row(self.data, self._extra, datacolumns)
667 except NotImplementedError:
668 data = None
669 row = data or {}
670 for column, colname in colnames.items():
671 if colname == 'filename':
672 value = self.file.filename if self.file else None
673 elif colname == 'line':
674 value = self.file.lno if self.file else None
675 elif colname is None:
676 value = getattr(self, column, None)
677 else:
678 value = getattr(self, colname, None)
679 row.setdefault(column, value)
680 return row
681
682 def render(self, fmt='line'):
683 fmt = fmt or 'line'
684 try:
685 render = _FORMATS[fmt]
686 except KeyError:
687 raise TypeError(f'unsupported fmt {fmt!r}')
688 try:
689 data = self._render_data(fmt, self.data, self._extra)
690 except NotImplementedError:
691 data = '-'
692 yield from render(self, data)
693
694
695 ### formats ###
696
697 def _fmt_line(parsed, data=None):
698 parts = [
699 f'<{parsed.kind.value}>',
700 ]
701 parent = ''
702 if parsed.parent:
703 parent = parsed.parent
704 if not isinstance(parent, str):
705 if parent.kind is KIND.FUNCTION:
706 parent = f'{parent.name}()'
707 else:
708 parent = parent.name
709 name = f'<{parent}>.{parsed.name}'
710 else:
711 name = parsed.name
712 if data is None:
713 data = parsed.data
714 elif data is iter(data):
715 data, = data
716 parts.extend([
717 name,
718 f'<{data}>' if data else '-',
719 f'({str(parsed.file or "<unknown file>")})',
720 ])
721 yield '\t'.join(parts)
722
723
724 def _fmt_full(parsed, data=None):
725 if parsed.kind is KIND.VARIABLE and parsed.parent:
726 prefix = 'local '
727 suffix = f' ({parsed.parent.name})'
728 else:
729 # XXX Show other prefixes (e.g. global, public)
730 prefix = suffix = ''
731 yield f'{prefix}{parsed.kind.value} {parsed.name!r}{suffix}'
732 for column, info in parsed.render_rowdata().items():
733 if column == 'kind':
734 continue
735 if column == 'name':
736 continue
737 if column == 'parent' and parsed.kind is not KIND.VARIABLE:
738 continue
739 if column == 'data':
740 if parsed.kind in (KIND.STRUCT, KIND.UNION):
741 column = 'members'
742 elif parsed.kind is KIND.ENUM:
743 column = 'enumerators'
744 elif parsed.kind is KIND.STATEMENT:
745 column = 'text'
746 data, = data
747 else:
748 column = 'signature'
749 data, = data
750 if not data:
751 # yield f'\t{column}:\t-'
752 continue
753 elif isinstance(data, str):
754 yield f'\t{column}:\t{data!r}'
755 else:
756 yield f'\t{column}:'
757 for line in data:
758 yield f'\t\t- {line}'
759 else:
760 yield f'\t{column}:\t{info}'
761
762
763 _FORMATS = {
764 'raw': (lambda v, _d: [repr(v)]),
765 'brief': _fmt_line,
766 'line': _fmt_line,
767 'full': _fmt_full,
768 }
769
770
771 ### declarations ##
772
773 class ESC[4;38;5;81mDeclaration(ESC[4;38;5;149mHighlevelParsedItem):
774
775 @classmethod
776 def from_row(cls, row, **markers):
777 fixed = tuple(_tables.fix_row(row, **markers))
778 if cls is Declaration:
779 _, _, _, kind, _ = fixed
780 sub = KIND_CLASSES.get(KIND(kind))
781 if not sub or not issubclass(sub, Declaration):
782 raise TypeError(f'unsupported kind, got {row!r}')
783 else:
784 sub = cls
785 return sub._from_row(fixed)
786
787 @classmethod
788 def _from_row(cls, row):
789 filename, funcname, name, kind, data = row
790 kind = KIND._from_raw(kind)
791 if kind is not cls.kind:
792 raise TypeError(f'expected kind {cls.kind.value!r}, got {row!r}')
793 fileinfo = FileInfo.from_raw(filename)
794 extra = None
795 if isinstance(data, str):
796 data, extra = cls._parse_data(data, fmt='row')
797 if extra:
798 return cls(fileinfo, name, data, funcname, _extra=extra)
799 else:
800 return cls(fileinfo, name, data, funcname)
801
802 @classmethod
803 def _resolve_parent(cls, parsed, *, _kind=None):
804 if _kind is None:
805 raise TypeError(f'{cls.kind.value} declarations do not have parents ({parsed})')
806 return super()._resolve_parent(parsed, _kind=_kind)
807
808 @classmethod
809 def _render_data(cls, fmt, data, extra):
810 if not data:
811 # XXX There should be some! Forward?
812 yield '???'
813 else:
814 yield from cls._format_data(fmt, data, extra)
815
816 @classmethod
817 def _render_data_row_item(cls, colname, data, extra):
818 if colname == 'data':
819 return cls._format_data('row', data, extra)
820 else:
821 return None
822
823 @classmethod
824 def _format_data(cls, fmt, data, extra):
825 raise NotImplementedError(fmt)
826
827 @classmethod
828 def _parse_data(cls, datastr, fmt=None):
829 """This is the reverse of _render_data."""
830 if not datastr or datastr is _tables.UNKNOWN or datastr == '???':
831 return None, None
832 elif datastr is _tables.EMPTY or datastr == '-':
833 # All the kinds have *something* even it is unknown.
834 raise TypeError('all declarations have data of some sort, got none')
835 else:
836 return cls._unformat_data(datastr, fmt)
837
838 @classmethod
839 def _unformat_data(cls, datastr, fmt=None):
840 raise NotImplementedError(fmt)
841
842
843 class ESC[4;38;5;81mVarType(ESC[4;38;5;149mnamedtuple('VarType', 'typequal typespec abstract')):
844
845 @classmethod
846 def from_str(cls, text):
847 orig = text
848 storage, sep, text = text.strip().partition(' ')
849 if not sep:
850 text = storage
851 storage = None
852 elif storage not in ('auto', 'register', 'static', 'extern'):
853 text = orig
854 storage = None
855 return cls._from_str(text), storage
856
857 @classmethod
858 def _from_str(cls, text):
859 orig = text
860 if text.startswith(('const ', 'volatile ')):
861 typequal, _, text = text.partition(' ')
862 else:
863 typequal = None
864
865 # Extract a series of identifiers/keywords.
866 m = re.match(r"^ *'?([a-zA-Z_]\w*(?:\s+[a-zA-Z_]\w*)*)\s*(.*?)'?\s*$", text)
867 if not m:
868 raise ValueError(f'invalid vartype text {orig!r}')
869 typespec, abstract = m.groups()
870
871 return cls(typequal, typespec, abstract or None)
872
873 def __str__(self):
874 parts = []
875 if self.qualifier:
876 parts.append(self.qualifier)
877 parts.append(self.spec + (self.abstract or ''))
878 return ' '.join(parts)
879
880 @property
881 def qualifier(self):
882 return self.typequal
883
884 @property
885 def spec(self):
886 return self.typespec
887
888
889 class ESC[4;38;5;81mVariable(ESC[4;38;5;149mDeclaration):
890 kind = KIND.VARIABLE
891
892 @classmethod
893 def _resolve_parent(cls, parsed):
894 return super()._resolve_parent(parsed, _kind=KIND.FUNCTION)
895
896 @classmethod
897 def _resolve_data(cls, data):
898 if not data:
899 return None, None
900 storage, vartype = _get_vartype(data)
901 return VarType(**vartype), {'storage': storage}
902
903 @classmethod
904 def _raw_data(self, data, extra):
905 vartype = data._asdict()
906 return {
907 'storage': extra['storage'],
908 'vartype': vartype,
909 }
910
911 @classmethod
912 def _format_data(cls, fmt, data, extra):
913 storage = extra.get('storage')
914 text = f'{storage} {data}' if storage else str(data)
915 if fmt in ('line', 'brief'):
916 yield text
917 #elif fmt == 'full':
918 elif fmt == 'row':
919 yield text
920 else:
921 raise NotImplementedError(fmt)
922
923 @classmethod
924 def _unformat_data(cls, datastr, fmt=None):
925 if fmt in ('line', 'brief'):
926 vartype, storage = VarType.from_str(datastr)
927 return vartype, {'storage': storage}
928 #elif fmt == 'full':
929 elif fmt == 'row':
930 vartype, storage = VarType.from_str(datastr)
931 return vartype, {'storage': storage}
932 else:
933 raise NotImplementedError(fmt)
934
935 def __init__(self, file, name, data, parent=None, storage=None):
936 super().__init__(file, name, data, parent,
937 _extra={'storage': storage or None},
938 _shortkey=f'({parent.name}).{name}' if parent else name,
939 _key=(str(file),
940 # Tilde comes after all other ascii characters.
941 f'~{parent or ""}~',
942 name,
943 ),
944 )
945 if storage:
946 if storage not in STORAGE:
947 # The parser must need an update.
948 raise NotImplementedError(storage)
949 # Otherwise we trust the compiler to have validated it.
950
951 @property
952 def vartype(self):
953 return self.data
954
955
956 class ESC[4;38;5;81mSignature(ESC[4;38;5;149mnamedtuple('Signature', 'params returntype inline isforward')):
957
958 @classmethod
959 def from_str(cls, text):
960 orig = text
961 storage, sep, text = text.strip().partition(' ')
962 if not sep:
963 text = storage
964 storage = None
965 elif storage not in ('auto', 'register', 'static', 'extern'):
966 text = orig
967 storage = None
968 return cls._from_str(text), storage
969
970 @classmethod
971 def _from_str(cls, text):
972 orig = text
973 inline, sep, text = text.partition('|')
974 if not sep:
975 text = inline
976 inline = None
977
978 isforward = False
979 if text.endswith(';'):
980 text = text[:-1]
981 isforward = True
982 elif text.endswith('{}'):
983 text = text[:-2]
984
985 index = text.rindex('(')
986 if index < 0:
987 raise ValueError(f'bad signature text {orig!r}')
988 params = text[index:]
989 while params.count('(') <= params.count(')'):
990 index = text.rindex('(', 0, index)
991 if index < 0:
992 raise ValueError(f'bad signature text {orig!r}')
993 params = text[index:]
994 text = text[:index]
995
996 returntype = VarType._from_str(text.rstrip())
997
998 return cls(params, returntype, inline, isforward)
999
1000 def __str__(self):
1001 parts = []
1002 if self.inline:
1003 parts.extend([
1004 self.inline,
1005 '|',
1006 ])
1007 parts.extend([
1008 str(self.returntype),
1009 self.params,
1010 ';' if self.isforward else '{}',
1011 ])
1012 return ' '.join(parts)
1013
1014 @property
1015 def returns(self):
1016 return self.returntype
1017
1018 @property
1019 def typequal(self):
1020 return self.returntype.typequal
1021
1022 @property
1023 def typespec(self):
1024 return self.returntype.typespec
1025
1026 @property
1027 def abstract(self):
1028 return self.returntype.abstract
1029
1030
1031 class ESC[4;38;5;81mFunction(ESC[4;38;5;149mDeclaration):
1032 kind = KIND.FUNCTION
1033
1034 @classmethod
1035 def _resolve_data(cls, data):
1036 if not data:
1037 return None, None
1038 kwargs = dict(data)
1039 returntype = dict(data['returntype'])
1040 del returntype['storage']
1041 kwargs['returntype'] = VarType(**returntype)
1042 storage = kwargs.pop('storage')
1043 return Signature(**kwargs), {'storage': storage}
1044
1045 @classmethod
1046 def _raw_data(self, data):
1047 # XXX finish!
1048 return data
1049
1050 @classmethod
1051 def _format_data(cls, fmt, data, extra):
1052 storage = extra.get('storage')
1053 text = f'{storage} {data}' if storage else str(data)
1054 if fmt in ('line', 'brief'):
1055 yield text
1056 #elif fmt == 'full':
1057 elif fmt == 'row':
1058 yield text
1059 else:
1060 raise NotImplementedError(fmt)
1061
1062 @classmethod
1063 def _unformat_data(cls, datastr, fmt=None):
1064 if fmt in ('line', 'brief'):
1065 sig, storage = Signature.from_str(sig)
1066 return sig, {'storage': storage}
1067 #elif fmt == 'full':
1068 elif fmt == 'row':
1069 sig, storage = Signature.from_str(sig)
1070 return sig, {'storage': storage}
1071 else:
1072 raise NotImplementedError(fmt)
1073
1074 def __init__(self, file, name, data, parent=None, storage=None):
1075 super().__init__(file, name, data, parent, _extra={'storage': storage})
1076 self._shortkey = f'~{name}~ {self.data}'
1077 self._key = (
1078 str(file),
1079 self._shortkey,
1080 )
1081
1082 @property
1083 def signature(self):
1084 return self.data
1085
1086
1087 class ESC[4;38;5;81mTypeDeclaration(ESC[4;38;5;149mDeclaration):
1088
1089 def __init__(self, file, name, data, parent=None, *, _shortkey=None):
1090 if not _shortkey:
1091 _shortkey = f'{self.kind.value} {name}'
1092 super().__init__(file, name, data, parent,
1093 _shortkey=_shortkey,
1094 _key=(
1095 str(file),
1096 _shortkey,
1097 ),
1098 )
1099
1100
1101 class ESC[4;38;5;81mPOTSType(ESC[4;38;5;149mTypeDeclaration):
1102
1103 def __init__(self, name):
1104 _file = _data = _parent = None
1105 super().__init__(_file, name, _data, _parent, _shortkey=name)
1106
1107
1108 class ESC[4;38;5;81mFuncPtr(ESC[4;38;5;149mTypeDeclaration):
1109
1110 def __init__(self, vartype):
1111 _file = _name = _parent = None
1112 data = vartype
1113 self.vartype = vartype
1114 super().__init__(_file, _name, data, _parent, _shortkey=f'<{vartype}>')
1115
1116
1117 class ESC[4;38;5;81mTypeDef(ESC[4;38;5;149mTypeDeclaration):
1118 kind = KIND.TYPEDEF
1119
1120 @classmethod
1121 def _resolve_data(cls, data):
1122 if not data:
1123 raise NotImplementedError(data)
1124 kwargs = dict(data)
1125 del kwargs['storage']
1126 if 'returntype' in kwargs:
1127 vartype = kwargs['returntype']
1128 del vartype['storage']
1129 kwargs['returntype'] = VarType(**vartype)
1130 datacls = Signature
1131 else:
1132 datacls = VarType
1133 return datacls(**kwargs), None
1134
1135 @classmethod
1136 def _raw_data(self, data):
1137 # XXX finish!
1138 return data
1139
1140 @classmethod
1141 def _format_data(cls, fmt, data, extra):
1142 text = str(data)
1143 if fmt in ('line', 'brief'):
1144 yield text
1145 elif fmt == 'full':
1146 yield text
1147 elif fmt == 'row':
1148 yield text
1149 else:
1150 raise NotImplementedError(fmt)
1151
1152 @classmethod
1153 def _unformat_data(cls, datastr, fmt=None):
1154 if fmt in ('line', 'brief'):
1155 vartype, _ = VarType.from_str(datastr)
1156 return vartype, None
1157 #elif fmt == 'full':
1158 elif fmt == 'row':
1159 vartype, _ = VarType.from_str(datastr)
1160 return vartype, None
1161 else:
1162 raise NotImplementedError(fmt)
1163
1164 def __init__(self, file, name, data, parent=None):
1165 super().__init__(file, name, data, parent, _shortkey=name)
1166
1167 @property
1168 def vartype(self):
1169 return self.data
1170
1171
1172 class ESC[4;38;5;81mMember(ESC[4;38;5;149mnamedtuple('Member', 'name vartype size')):
1173
1174 @classmethod
1175 def from_data(cls, raw, index):
1176 name = raw.name if raw.name else index
1177 vartype = size = None
1178 if type(raw.data) is int:
1179 size = raw.data
1180 elif isinstance(raw.data, str):
1181 size = int(raw.data)
1182 elif raw.data:
1183 vartype = dict(raw.data)
1184 del vartype['storage']
1185 if 'size' in vartype:
1186 size = vartype.pop('size')
1187 if isinstance(size, str) and size.isdigit():
1188 size = int(size)
1189 vartype = VarType(**vartype)
1190 return cls(name, vartype, size)
1191
1192 @classmethod
1193 def from_str(cls, text):
1194 name, _, vartype = text.partition(': ')
1195 if name.startswith('#'):
1196 name = int(name[1:])
1197 if vartype.isdigit():
1198 size = int(vartype)
1199 vartype = None
1200 else:
1201 vartype, _ = VarType.from_str(vartype)
1202 size = None
1203 return cls(name, vartype, size)
1204
1205 def __str__(self):
1206 name = self.name if isinstance(self.name, str) else f'#{self.name}'
1207 return f'{name}: {self.vartype or self.size}'
1208
1209
1210 class ESC[4;38;5;81m_StructUnion(ESC[4;38;5;149mTypeDeclaration):
1211
1212 @classmethod
1213 def _resolve_data(cls, data):
1214 if not data:
1215 # XXX There should be some! Forward?
1216 return None, None
1217 return [Member.from_data(v, i) for i, v in enumerate(data)], None
1218
1219 @classmethod
1220 def _raw_data(self, data):
1221 # XXX finish!
1222 return data
1223
1224 @classmethod
1225 def _format_data(cls, fmt, data, extra):
1226 if fmt in ('line', 'brief'):
1227 members = ', '.join(f'<{m}>' for m in data)
1228 yield f'[{members}]'
1229 elif fmt == 'full':
1230 for member in data:
1231 yield f'{member}'
1232 elif fmt == 'row':
1233 members = ', '.join(f'<{m}>' for m in data)
1234 yield f'[{members}]'
1235 else:
1236 raise NotImplementedError(fmt)
1237
1238 @classmethod
1239 def _unformat_data(cls, datastr, fmt=None):
1240 if fmt in ('line', 'brief'):
1241 members = [Member.from_str(m[1:-1])
1242 for m in datastr[1:-1].split(', ')]
1243 return members, None
1244 #elif fmt == 'full':
1245 elif fmt == 'row':
1246 members = [Member.from_str(m.rstrip('>').lstrip('<'))
1247 for m in datastr[1:-1].split('>, <')]
1248 return members, None
1249 else:
1250 raise NotImplementedError(fmt)
1251
1252 def __init__(self, file, name, data, parent=None):
1253 super().__init__(file, name, data, parent)
1254
1255 @property
1256 def members(self):
1257 return self.data
1258
1259
1260 class ESC[4;38;5;81mStruct(ESC[4;38;5;149m_StructUnion):
1261 kind = KIND.STRUCT
1262
1263
1264 class ESC[4;38;5;81mUnion(ESC[4;38;5;149m_StructUnion):
1265 kind = KIND.UNION
1266
1267
1268 class ESC[4;38;5;81mEnum(ESC[4;38;5;149mTypeDeclaration):
1269 kind = KIND.ENUM
1270
1271 @classmethod
1272 def _resolve_data(cls, data):
1273 if not data:
1274 # XXX There should be some! Forward?
1275 return None, None
1276 enumerators = [e if isinstance(e, str) else e.name
1277 for e in data]
1278 return enumerators, None
1279
1280 @classmethod
1281 def _raw_data(self, data):
1282 # XXX finish!
1283 return data
1284
1285 @classmethod
1286 def _format_data(cls, fmt, data, extra):
1287 if fmt in ('line', 'brief'):
1288 yield repr(data)
1289 elif fmt == 'full':
1290 for enumerator in data:
1291 yield f'{enumerator}'
1292 elif fmt == 'row':
1293 # XXX This won't work with CSV...
1294 yield ','.join(data)
1295 else:
1296 raise NotImplementedError(fmt)
1297
1298 @classmethod
1299 def _unformat_data(cls, datastr, fmt=None):
1300 if fmt in ('line', 'brief'):
1301 return _strutil.unrepr(datastr), None
1302 #elif fmt == 'full':
1303 elif fmt == 'row':
1304 return datastr.split(','), None
1305 else:
1306 raise NotImplementedError(fmt)
1307
1308 def __init__(self, file, name, data, parent=None):
1309 super().__init__(file, name, data, parent)
1310
1311 @property
1312 def enumerators(self):
1313 return self.data
1314
1315
1316 ### statements ###
1317
1318 class ESC[4;38;5;81mStatement(ESC[4;38;5;149mHighlevelParsedItem):
1319 kind = KIND.STATEMENT
1320
1321 @classmethod
1322 def _resolve_data(cls, data):
1323 # XXX finish!
1324 return data, None
1325
1326 @classmethod
1327 def _raw_data(self, data):
1328 # XXX finish!
1329 return data
1330
1331 @classmethod
1332 def _render_data(cls, fmt, data, extra):
1333 # XXX Handle other formats?
1334 return repr(data)
1335
1336 @classmethod
1337 def _parse_data(self, datastr, fmt=None):
1338 # XXX Handle other formats?
1339 return _strutil.unrepr(datastr), None
1340
1341 def __init__(self, file, name, data, parent=None):
1342 super().__init__(file, name, data, parent,
1343 _shortkey=data or '',
1344 _key=(
1345 str(file),
1346 file.lno,
1347 # XXX Only one stmt per line?
1348 ),
1349 )
1350
1351 @property
1352 def text(self):
1353 return self.data
1354
1355
1356 ###
1357
1358 KIND_CLASSES = {cls.kind: cls for cls in [
1359 Variable,
1360 Function,
1361 TypeDef,
1362 Struct,
1363 Union,
1364 Enum,
1365 Statement,
1366 ]}
1367
1368
1369 def resolve_parsed(parsed):
1370 if isinstance(parsed, HighlevelParsedItem):
1371 return parsed
1372 try:
1373 cls = KIND_CLASSES[parsed.kind]
1374 except KeyError:
1375 raise ValueError(f'unsupported kind in {parsed!r}')
1376 return cls.from_parsed(parsed)
1377
1378
1379 def set_flag(item, name, value):
1380 try:
1381 setattr(item, name, value)
1382 except AttributeError:
1383 object.__setattr__(item, name, value)
1384
1385
1386 #############################
1387 # composite
1388
1389 class ESC[4;38;5;81mDeclarations:
1390
1391 @classmethod
1392 def from_decls(cls, decls):
1393 return cls(decls)
1394
1395 @classmethod
1396 def from_parsed(cls, items):
1397 decls = (resolve_parsed(item)
1398 for item in items
1399 if item.kind is not KIND.STATEMENT)
1400 return cls.from_decls(decls)
1401
1402 @classmethod
1403 def _resolve_key(cls, raw):
1404 if isinstance(raw, str):
1405 raw = [raw]
1406 elif isinstance(raw, Declaration):
1407 raw = (
1408 raw.filename if cls._is_public(raw) else None,
1409 # `raw.parent` is always None for types and functions.
1410 raw.parent if raw.kind is KIND.VARIABLE else None,
1411 raw.name,
1412 )
1413
1414 extra = None
1415 if len(raw) == 1:
1416 name, = raw
1417 if name:
1418 name = str(name)
1419 if name.endswith(('.c', '.h')):
1420 # This is only legit as a query.
1421 key = (name, None, None)
1422 else:
1423 key = (None, None, name)
1424 else:
1425 key = (None, None, None)
1426 elif len(raw) == 2:
1427 parent, name = raw
1428 name = str(name)
1429 if isinstance(parent, Declaration):
1430 key = (None, parent.name, name)
1431 elif not parent:
1432 key = (None, None, name)
1433 else:
1434 parent = str(parent)
1435 if parent.endswith(('.c', '.h')):
1436 key = (parent, None, name)
1437 else:
1438 key = (None, parent, name)
1439 else:
1440 key, extra = raw[:3], raw[3:]
1441 filename, funcname, name = key
1442 filename = str(filename) if filename else None
1443 if isinstance(funcname, Declaration):
1444 funcname = funcname.name
1445 else:
1446 funcname = str(funcname) if funcname else None
1447 name = str(name) if name else None
1448 key = (filename, funcname, name)
1449 return key, extra
1450
1451 @classmethod
1452 def _is_public(cls, decl):
1453 # For .c files don't we need info from .h files to make this decision?
1454 # XXX Check for "extern".
1455 # For now we treat all decls a "private" (have filename set).
1456 return False
1457
1458 def __init__(self, decls):
1459 # (file, func, name) -> decl
1460 # "public":
1461 # * (None, None, name)
1462 # "private", "global":
1463 # * (file, None, name)
1464 # "private", "local":
1465 # * (file, func, name)
1466 if hasattr(decls, 'items'):
1467 self._decls = decls
1468 else:
1469 self._decls = {}
1470 self._extend(decls)
1471
1472 # XXX always validate?
1473
1474 def validate(self):
1475 for key, decl in self._decls.items():
1476 if type(key) is not tuple or len(key) != 3:
1477 raise ValueError(f'expected 3-tuple key, got {key!r} (for decl {decl!r})')
1478 filename, funcname, name = key
1479 if not name:
1480 raise ValueError(f'expected name in key, got {key!r} (for decl {decl!r})')
1481 elif type(name) is not str:
1482 raise ValueError(f'expected name in key to be str, got {key!r} (for decl {decl!r})')
1483 # XXX Check filename type?
1484 # XXX Check funcname type?
1485
1486 if decl.kind is KIND.STATEMENT:
1487 raise ValueError(f'expected a declaration, got {decl!r}')
1488
1489 def __repr__(self):
1490 return f'{type(self).__name__}({list(self)})'
1491
1492 def __len__(self):
1493 return len(self._decls)
1494
1495 def __iter__(self):
1496 yield from self._decls
1497
1498 def __getitem__(self, key):
1499 # XXX Be more exact for the 3-tuple case?
1500 if type(key) not in (str, tuple):
1501 raise KeyError(f'unsupported key {key!r}')
1502 resolved, extra = self._resolve_key(key)
1503 if extra:
1504 raise KeyError(f'key must have at most 3 parts, got {key!r}')
1505 if not resolved[2]:
1506 raise ValueError(f'expected name in key, got {key!r}')
1507 try:
1508 return self._decls[resolved]
1509 except KeyError:
1510 if type(key) is tuple and len(key) == 3:
1511 filename, funcname, name = key
1512 else:
1513 filename, funcname, name = resolved
1514 if filename and not filename.endswith(('.c', '.h')):
1515 raise KeyError(f'invalid filename in key {key!r}')
1516 elif funcname and funcname.endswith(('.c', '.h')):
1517 raise KeyError(f'invalid funcname in key {key!r}')
1518 elif name and name.endswith(('.c', '.h')):
1519 raise KeyError(f'invalid name in key {key!r}')
1520 else:
1521 raise # re-raise
1522
1523 @property
1524 def types(self):
1525 return self._find(kind=KIND.TYPES)
1526
1527 @property
1528 def functions(self):
1529 return self._find(None, None, None, KIND.FUNCTION)
1530
1531 @property
1532 def variables(self):
1533 return self._find(None, None, None, KIND.VARIABLE)
1534
1535 def iter_all(self):
1536 yield from self._decls.values()
1537
1538 def get(self, key, default=None):
1539 try:
1540 return self[key]
1541 except KeyError:
1542 return default
1543
1544 #def add_decl(self, decl, key=None):
1545 # decl = _resolve_parsed(decl)
1546 # self._add_decl(decl, key)
1547
1548 def find(self, *key, **explicit):
1549 if not key:
1550 if not explicit:
1551 return iter(self)
1552 return self._find(**explicit)
1553
1554 resolved, extra = self._resolve_key(key)
1555 filename, funcname, name = resolved
1556 if not extra:
1557 kind = None
1558 elif len(extra) == 1:
1559 kind, = extra
1560 else:
1561 raise KeyError(f'key must have at most 4 parts, got {key!r}')
1562
1563 implicit= {}
1564 if filename:
1565 implicit['filename'] = filename
1566 if funcname:
1567 implicit['funcname'] = funcname
1568 if name:
1569 implicit['name'] = name
1570 if kind:
1571 implicit['kind'] = kind
1572 return self._find(**implicit, **explicit)
1573
1574 def _find(self, filename=None, funcname=None, name=None, kind=None):
1575 for decl in self._decls.values():
1576 if filename and decl.filename != filename:
1577 continue
1578 if funcname:
1579 if decl.kind is not KIND.VARIABLE:
1580 continue
1581 if decl.parent.name != funcname:
1582 continue
1583 if name and decl.name != name:
1584 continue
1585 if kind:
1586 kinds = KIND.resolve_group(kind)
1587 if decl.kind not in kinds:
1588 continue
1589 yield decl
1590
1591 def _add_decl(self, decl, key=None):
1592 if key:
1593 if type(key) not in (str, tuple):
1594 raise NotImplementedError((key, decl))
1595 # Any partial key will be turned into a full key, but that
1596 # same partial key will still match a key lookup.
1597 resolved, _ = self._resolve_key(key)
1598 if not resolved[2]:
1599 raise ValueError(f'expected name in key, got {key!r}')
1600 key = resolved
1601 # XXX Also add with the decl-derived key if not the same?
1602 else:
1603 key, _ = self._resolve_key(decl)
1604 self._decls[key] = decl
1605
1606 def _extend(self, decls):
1607 decls = iter(decls)
1608 # Check only the first item.
1609 for decl in decls:
1610 if isinstance(decl, Declaration):
1611 self._add_decl(decl)
1612 # Add the rest without checking.
1613 for decl in decls:
1614 self._add_decl(decl)
1615 elif isinstance(decl, HighlevelParsedItem):
1616 raise NotImplementedError(decl)
1617 else:
1618 try:
1619 key, decl = decl
1620 except ValueError:
1621 raise NotImplementedError(decl)
1622 if not isinstance(decl, Declaration):
1623 raise NotImplementedError(decl)
1624 self._add_decl(decl, key)
1625 # Add the rest without checking.
1626 for key, decl in decls:
1627 self._add_decl(decl, key)
1628 # The iterator will be exhausted at this point.