1from collections import namedtuple 2import enum 3import re 4 5from c_common import fsutil 6from c_common.clsutil import classonly 7import c_common.misc as _misc 8import c_common.strutil as _strutil 9import c_common.tables as _tables 10from .parser._regexes import _STORAGE 11 12 13FIXED_TYPE = _misc.Labeled('FIXED_TYPE') 14 15STORAGE = frozenset(_STORAGE) 16 17 18############################# 19# kinds 20 21@enum.unique 22class KIND(enum.Enum): 23 24 # XXX Use these in the raw parser code. 25 TYPEDEF = 'typedef' 26 STRUCT = 'struct' 27 UNION = 'union' 28 ENUM = 'enum' 29 FUNCTION = 'function' 30 VARIABLE = 'variable' 31 STATEMENT = 'statement' 32 33 @classonly 34 def _from_raw(cls, raw): 35 if raw is None: 36 return None 37 elif isinstance(raw, cls): 38 return raw 39 elif type(raw) is str: 40 # We could use cls[raw] for the upper-case form, 41 # but there's no need to go to the trouble. 42 return cls(raw.lower()) 43 else: 44 raise NotImplementedError(raw) 45 46 @classonly 47 def by_priority(cls, group=None): 48 if group is None: 49 return cls._ALL_BY_PRIORITY.copy() 50 elif group == 'type': 51 return cls._TYPE_DECLS_BY_PRIORITY.copy() 52 elif group == 'decl': 53 return cls._ALL_DECLS_BY_PRIORITY.copy() 54 elif isinstance(group, str): 55 raise NotImplementedError(group) 56 else: 57 # XXX Treat group as a set of kinds & return in priority order? 58 raise NotImplementedError(group) 59 60 @classonly 61 def is_type_decl(cls, kind): 62 if kind in cls.TYPES: 63 return True 64 if not isinstance(kind, cls): 65 raise TypeError(f'expected KIND, got {kind!r}') 66 return False 67 68 @classonly 69 def is_decl(cls, kind): 70 if kind in cls.DECLS: 71 return True 72 if not isinstance(kind, cls): 73 raise TypeError(f'expected KIND, got {kind!r}') 74 return False 75 76 @classonly 77 def get_group(cls, kind, *, groups=None): 78 if not isinstance(kind, cls): 79 raise TypeError(f'expected KIND, got {kind!r}') 80 if groups is None: 81 groups = ['type'] 82 elif not groups: 83 groups = () 84 elif isinstance(groups, str): 85 group = groups 86 if group not in cls._GROUPS: 87 raise ValueError(f'unsupported group {group!r}') 88 groups = [group] 89 else: 90 unsupported = [g for g in groups if g not in cls._GROUPS] 91 if unsupported: 92 raise ValueError(f'unsupported groups {", ".join(repr(unsupported))}') 93 for group in groups: 94 if kind in cls._GROUPS[group]: 95 return group 96 else: 97 return kind.value 98 99 @classonly 100 def resolve_group(cls, group): 101 if isinstance(group, cls): 102 return {group} 103 elif isinstance(group, str): 104 try: 105 return cls._GROUPS[group].copy() 106 except KeyError: 107 raise ValueError(f'unsupported group {group!r}') 108 else: 109 resolved = set() 110 for gr in group: 111 resolve.update(cls.resolve_group(gr)) 112 return resolved 113 #return {*cls.resolve_group(g) for g in group} 114 115 116KIND._TYPE_DECLS_BY_PRIORITY = [ 117 # These are in preferred order. 118 KIND.TYPEDEF, 119 KIND.STRUCT, 120 KIND.UNION, 121 KIND.ENUM, 122] 123KIND._ALL_DECLS_BY_PRIORITY = [ 124 # These are in preferred order. 125 *KIND._TYPE_DECLS_BY_PRIORITY, 126 KIND.FUNCTION, 127 KIND.VARIABLE, 128] 129KIND._ALL_BY_PRIORITY = [ 130 # These are in preferred order. 131 *KIND._ALL_DECLS_BY_PRIORITY, 132 KIND.STATEMENT, 133] 134 135KIND.TYPES = frozenset(KIND._TYPE_DECLS_BY_PRIORITY) 136KIND.DECLS = frozenset(KIND._ALL_DECLS_BY_PRIORITY) 137KIND._GROUPS = { 138 'type': KIND.TYPES, 139 'decl': KIND.DECLS, 140} 141KIND._GROUPS.update((k.value, {k}) for k in KIND) 142 143 144def get_kind_group(item): 145 return KIND.get_group(item.kind) 146 147 148############################# 149# low-level 150 151def _fix_filename(filename, relroot, *, 152 formatted=True, 153 **kwargs): 154 if formatted: 155 fix = fsutil.format_filename 156 else: 157 fix = fsutil.fix_filename 158 return fix(filename, relroot=relroot, **kwargs) 159 160 161class FileInfo(namedtuple('FileInfo', 'filename lno')): 162 @classmethod 163 def from_raw(cls, raw): 164 if isinstance(raw, cls): 165 return raw 166 elif isinstance(raw, tuple): 167 return cls(*raw) 168 elif not raw: 169 return None 170 elif isinstance(raw, str): 171 return cls(raw, -1) 172 else: 173 raise TypeError(f'unsupported "raw": {raw:!r}') 174 175 def __str__(self): 176 return self.filename 177 178 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): 179 filename = _fix_filename(self.filename, relroot, **kwargs) 180 if filename == self.filename: 181 return self 182 return self._replace(filename=filename) 183 184 185class SourceLine(namedtuple('Line', 'file kind data conditions')): 186 KINDS = ( 187 #'directive', # data is ... 188 'source', # "data" is the line 189 #'comment', # "data" is the text, including comment markers 190 ) 191 192 @property 193 def filename(self): 194 return self.file.filename 195 196 @property 197 def lno(self): 198 return self.file.lno 199 200 201class DeclID(namedtuple('DeclID', 'filename funcname name')): 202 """The globally-unique identifier for a declaration.""" 203 204 @classmethod 205 def from_row(cls, row, **markers): 206 row = _tables.fix_row(row, **markers) 207 return cls(*row) 208 209 # We have to provide _make() because we implemented __new__(). 210 211 @classmethod 212 def _make(cls, iterable): 213 try: 214 return cls(*iterable) 215 except Exception: 216 super()._make(iterable) 217 raise # re-raise 218 219 def __new__(cls, filename, funcname, name): 220 self = super().__new__( 221 cls, 222 filename=str(filename) if filename else None, 223 funcname=str(funcname) if funcname else None, 224 name=str(name) if name else None, 225 ) 226 self._compare = tuple(v or '' for v in self) 227 return self 228 229 def __hash__(self): 230 return super().__hash__() 231 232 def __eq__(self, other): 233 try: 234 other = tuple(v or '' for v in other) 235 except TypeError: 236 return NotImplemented 237 return self._compare == other 238 239 def __gt__(self, other): 240 try: 241 other = tuple(v or '' for v in other) 242 except TypeError: 243 return NotImplemented 244 return self._compare > other 245 246 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): 247 filename = _fix_filename(self.filename, relroot, **kwargs) 248 if filename == self.filename: 249 return self 250 return self._replace(filename=filename) 251 252 253class ParsedItem(namedtuple('ParsedItem', 'file kind parent name data')): 254 255 @classmethod 256 def from_raw(cls, raw): 257 if isinstance(raw, cls): 258 return raw 259 elif isinstance(raw, tuple): 260 return cls(*raw) 261 else: 262 raise TypeError(f'unsupported "raw": {raw:!r}') 263 264 @classmethod 265 def from_row(cls, row, columns=None): 266 if not columns: 267 colnames = 'filename funcname name kind data'.split() 268 else: 269 colnames = list(columns) 270 for i, column in enumerate(colnames): 271 if column == 'file': 272 colnames[i] = 'filename' 273 elif column == 'funcname': 274 colnames[i] = 'parent' 275 if len(row) != len(set(colnames)): 276 raise NotImplementedError(columns, row) 277 kwargs = {} 278 for column, value in zip(colnames, row): 279 if column == 'filename': 280 kwargs['file'] = FileInfo.from_raw(value) 281 elif column == 'kind': 282 kwargs['kind'] = KIND(value) 283 elif column in cls._fields: 284 kwargs[column] = value 285 else: 286 raise NotImplementedError(column) 287 return cls(**kwargs) 288 289 @property 290 def id(self): 291 try: 292 return self._id 293 except AttributeError: 294 if self.kind is KIND.STATEMENT: 295 self._id = None 296 else: 297 self._id = DeclID(str(self.file), self.funcname, self.name) 298 return self._id 299 300 @property 301 def filename(self): 302 if not self.file: 303 return None 304 return self.file.filename 305 306 @property 307 def lno(self): 308 if not self.file: 309 return -1 310 return self.file.lno 311 312 @property 313 def funcname(self): 314 if not self.parent: 315 return None 316 if type(self.parent) is str: 317 return self.parent 318 else: 319 return self.parent.name 320 321 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): 322 fixed = self.file.fix_filename(relroot, **kwargs) 323 if fixed == self.file: 324 return self 325 return self._replace(file=fixed) 326 327 def as_row(self, columns=None): 328 if not columns: 329 columns = self._fields 330 row = [] 331 for column in columns: 332 if column == 'file': 333 value = self.filename 334 elif column == 'kind': 335 value = self.kind.value 336 elif column == 'data': 337 value = self._render_data() 338 else: 339 value = getattr(self, column) 340 row.append(value) 341 return row 342 343 def _render_data(self): 344 if not self.data: 345 return None 346 elif isinstance(self.data, str): 347 return self.data 348 else: 349 # XXX 350 raise NotImplementedError 351 352 353def _get_vartype(data): 354 try: 355 vartype = dict(data['vartype']) 356 except KeyError: 357 vartype = dict(data) 358 storage = data.get('storage') 359 else: 360 storage = data.get('storage') or vartype.get('storage') 361 del vartype['storage'] 362 return storage, vartype 363 364 365def get_parsed_vartype(decl): 366 kind = getattr(decl, 'kind', None) 367 if isinstance(decl, ParsedItem): 368 storage, vartype = _get_vartype(decl.data) 369 typequal = vartype['typequal'] 370 typespec = vartype['typespec'] 371 abstract = vartype['abstract'] 372 elif isinstance(decl, dict): 373 kind = decl.get('kind') 374 storage, vartype = _get_vartype(decl) 375 typequal = vartype['typequal'] 376 typespec = vartype['typespec'] 377 abstract = vartype['abstract'] 378 elif isinstance(decl, VarType): 379 storage = None 380 typequal, typespec, abstract = decl 381 elif isinstance(decl, TypeDef): 382 storage = None 383 typequal, typespec, abstract = decl.vartype 384 elif isinstance(decl, Variable): 385 storage = decl.storage 386 typequal, typespec, abstract = decl.vartype 387 elif isinstance(decl, Signature): 388 storage = None 389 typequal, typespec, abstract = decl.returntype 390 elif isinstance(decl, Function): 391 storage = decl.storage 392 typequal, typespec, abstract = decl.signature.returntype 393 elif isinstance(decl, str): 394 vartype, storage = VarType.from_str(decl) 395 typequal, typespec, abstract = vartype 396 else: 397 raise NotImplementedError(decl) 398 return kind, storage, typequal, typespec, abstract 399 400 401def get_default_storage(decl): 402 if decl.kind not in (KIND.VARIABLE, KIND.FUNCTION): 403 return None 404 return 'extern' if decl.parent is None else 'auto' 405 406 407def get_effective_storage(decl, *, default=None): 408 # Note that "static" limits access to just that C module 409 # and "extern" (the default for module-level) allows access 410 # outside the C module. 411 if default is None: 412 default = get_default_storage(decl) 413 if default is None: 414 return None 415 try: 416 storage = decl.storage 417 except AttributeError: 418 storage, _ = _get_vartype(decl.data) 419 return storage or default 420 421 422############################# 423# high-level 424 425class HighlevelParsedItem: 426 427 kind = None 428 429 FIELDS = ('file', 'parent', 'name', 'data') 430 431 @classmethod 432 def from_parsed(cls, parsed): 433 if parsed.kind is not cls.kind: 434 raise TypeError(f'kind mismatch ({parsed.kind.value} != {cls.kind.value})') 435 data, extra = cls._resolve_data(parsed.data) 436 self = cls( 437 cls._resolve_file(parsed), 438 parsed.name, 439 data, 440 cls._resolve_parent(parsed) if parsed.parent else None, 441 **extra or {} 442 ) 443 self._parsed = parsed 444 return self 445 446 @classmethod 447 def _resolve_file(cls, parsed): 448 fileinfo = FileInfo.from_raw(parsed.file) 449 if not fileinfo: 450 raise NotImplementedError(parsed) 451 return fileinfo 452 453 @classmethod 454 def _resolve_data(cls, data): 455 return data, None 456 457 @classmethod 458 def _raw_data(cls, data, extra): 459 if isinstance(data, str): 460 return data 461 else: 462 raise NotImplementedError(data) 463 464 @classmethod 465 def _data_as_row(cls, data, extra, colnames): 466 row = {} 467 for colname in colnames: 468 if colname in row: 469 continue 470 rendered = cls._render_data_row_item(colname, data, extra) 471 if rendered is iter(rendered): 472 rendered, = rendered 473 row[colname] = rendered 474 return row 475 476 @classmethod 477 def _render_data_row_item(cls, colname, data, extra): 478 if colname == 'data': 479 return str(data) 480 else: 481 return None 482 483 @classmethod 484 def _render_data_row(cls, fmt, data, extra, colnames): 485 if fmt != 'row': 486 raise NotImplementedError 487 datarow = cls._data_as_row(data, extra, colnames) 488 unresolved = [c for c, v in datarow.items() if v is None] 489 if unresolved: 490 raise NotImplementedError(unresolved) 491 for colname, value in datarow.items(): 492 if type(value) != str: 493 if colname == 'kind': 494 datarow[colname] = value.value 495 else: 496 datarow[colname] = str(value) 497 return datarow 498 499 @classmethod 500 def _render_data(cls, fmt, data, extra): 501 row = cls._render_data_row(fmt, data, extra, ['data']) 502 yield ' '.join(row.values()) 503 504 @classmethod 505 def _resolve_parent(cls, parsed, *, _kind=None): 506 fileinfo = FileInfo(parsed.file.filename, -1) 507 if isinstance(parsed.parent, str): 508 if parsed.parent.isidentifier(): 509 name = parsed.parent 510 else: 511 # XXX It could be something like "<kind> <name>". 512 raise NotImplementedError(repr(parsed.parent)) 513 parent = ParsedItem(fileinfo, _kind, None, name, None) 514 elif type(parsed.parent) is tuple: 515 # XXX It could be something like (kind, name). 516 raise NotImplementedError(repr(parsed.parent)) 517 else: 518 return parsed.parent 519 Parent = KIND_CLASSES.get(_kind, Declaration) 520 return Parent.from_parsed(parent) 521 522 @classmethod 523 def _parse_columns(cls, columns): 524 colnames = {} # {requested -> actual} 525 columns = list(columns or cls.FIELDS) 526 datacolumns = [] 527 for i, colname in enumerate(columns): 528 if colname == 'file': 529 columns[i] = 'filename' 530 colnames['file'] = 'filename' 531 elif colname == 'lno': 532 columns[i] = 'line' 533 colnames['lno'] = 'line' 534 elif colname in ('filename', 'line'): 535 colnames[colname] = colname 536 elif colname == 'data': 537 datacolumns.append(colname) 538 colnames[colname] = None 539 elif colname in cls.FIELDS or colname == 'kind': 540 colnames[colname] = colname 541 else: 542 datacolumns.append(colname) 543 colnames[colname] = None 544 return columns, datacolumns, colnames 545 546 def __init__(self, file, name, data, parent=None, *, 547 _extra=None, 548 _shortkey=None, 549 _key=None, 550 ): 551 self.file = file 552 self.parent = parent or None 553 self.name = name 554 self.data = data 555 self._extra = _extra or {} 556 self._shortkey = _shortkey 557 self._key = _key 558 559 def __repr__(self): 560 args = [f'{n}={getattr(self, n)!r}' 561 for n in ['file', 'name', 'data', 'parent', *(self._extra or ())]] 562 return f'{type(self).__name__}({", ".join(args)})' 563 564 def __str__(self): 565 try: 566 return self._str 567 except AttributeError: 568 self._str = next(self.render()) 569 return self._str 570 571 def __getattr__(self, name): 572 try: 573 return self._extra[name] 574 except KeyError: 575 raise AttributeError(name) 576 577 def __hash__(self): 578 return hash(self._key) 579 580 def __eq__(self, other): 581 if isinstance(other, HighlevelParsedItem): 582 return self._key == other._key 583 elif type(other) is tuple: 584 return self._key == other 585 else: 586 return NotImplemented 587 588 def __gt__(self, other): 589 if isinstance(other, HighlevelParsedItem): 590 return self._key > other._key 591 elif type(other) is tuple: 592 return self._key > other 593 else: 594 return NotImplemented 595 596 @property 597 def id(self): 598 return self.parsed.id 599 600 @property 601 def shortkey(self): 602 return self._shortkey 603 604 @property 605 def key(self): 606 return self._key 607 608 @property 609 def filename(self): 610 if not self.file: 611 return None 612 return self.file.filename 613 614 @property 615 def parsed(self): 616 try: 617 return self._parsed 618 except AttributeError: 619 parent = self.parent 620 if parent is not None and not isinstance(parent, str): 621 parent = parent.name 622 self._parsed = ParsedItem( 623 self.file, 624 self.kind, 625 parent, 626 self.name, 627 self._raw_data(), 628 ) 629 return self._parsed 630 631 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): 632 if self.file: 633 self.file = self.file.fix_filename(relroot, **kwargs) 634 return self 635 636 def as_rowdata(self, columns=None): 637 columns, datacolumns, colnames = self._parse_columns(columns) 638 return self._as_row(colnames, datacolumns, self._data_as_row) 639 640 def render_rowdata(self, columns=None): 641 columns, datacolumns, colnames = self._parse_columns(columns) 642 def data_as_row(data, ext, cols): 643 return self._render_data_row('row', data, ext, cols) 644 rowdata = self._as_row(colnames, datacolumns, data_as_row) 645 for column, value in rowdata.items(): 646 colname = colnames.get(column) 647 if not colname: 648 continue 649 if column == 'kind': 650 value = value.value 651 else: 652 if column == 'parent': 653 if self.parent: 654 value = f'({self.parent.kind.value} {self.parent.name})' 655 if not value: 656 value = '-' 657 elif type(value) is VarType: 658 value = repr(str(value)) 659 else: 660 value = str(value) 661 rowdata[column] = value 662 return rowdata 663 664 def _as_row(self, colnames, datacolumns, data_as_row): 665 try: 666 data = data_as_row(self.data, self._extra, datacolumns) 667 except NotImplementedError: 668 data = None 669 row = data or {} 670 for column, colname in colnames.items(): 671 if colname == 'filename': 672 value = self.file.filename if self.file else None 673 elif colname == 'line': 674 value = self.file.lno if self.file else None 675 elif colname is None: 676 value = getattr(self, column, None) 677 else: 678 value = getattr(self, colname, None) 679 row.setdefault(column, value) 680 return row 681 682 def render(self, fmt='line'): 683 fmt = fmt or 'line' 684 try: 685 render = _FORMATS[fmt] 686 except KeyError: 687 raise TypeError(f'unsupported fmt {fmt!r}') 688 try: 689 data = self._render_data(fmt, self.data, self._extra) 690 except NotImplementedError: 691 data = '-' 692 yield from render(self, data) 693 694 695### formats ### 696 697def _fmt_line(parsed, data=None): 698 parts = [ 699 f'<{parsed.kind.value}>', 700 ] 701 parent = '' 702 if parsed.parent: 703 parent = parsed.parent 704 if not isinstance(parent, str): 705 if parent.kind is KIND.FUNCTION: 706 parent = f'{parent.name}()' 707 else: 708 parent = parent.name 709 name = f'<{parent}>.{parsed.name}' 710 else: 711 name = parsed.name 712 if data is None: 713 data = parsed.data 714 elif data is iter(data): 715 data, = data 716 parts.extend([ 717 name, 718 f'<{data}>' if data else '-', 719 f'({str(parsed.file or "<unknown file>")})', 720 ]) 721 yield '\t'.join(parts) 722 723 724def _fmt_full(parsed, data=None): 725 if parsed.kind is KIND.VARIABLE and parsed.parent: 726 prefix = 'local ' 727 suffix = f' ({parsed.parent.name})' 728 else: 729 # XXX Show other prefixes (e.g. global, public) 730 prefix = suffix = '' 731 yield f'{prefix}{parsed.kind.value} {parsed.name!r}{suffix}' 732 for column, info in parsed.render_rowdata().items(): 733 if column == 'kind': 734 continue 735 if column == 'name': 736 continue 737 if column == 'parent' and parsed.kind is not KIND.VARIABLE: 738 continue 739 if column == 'data': 740 if parsed.kind in (KIND.STRUCT, KIND.UNION): 741 column = 'members' 742 elif parsed.kind is KIND.ENUM: 743 column = 'enumerators' 744 elif parsed.kind is KIND.STATEMENT: 745 column = 'text' 746 data, = data 747 else: 748 column = 'signature' 749 data, = data 750 if not data: 751# yield f'\t{column}:\t-' 752 continue 753 elif isinstance(data, str): 754 yield f'\t{column}:\t{data!r}' 755 else: 756 yield f'\t{column}:' 757 for line in data: 758 yield f'\t\t- {line}' 759 else: 760 yield f'\t{column}:\t{info}' 761 762 763_FORMATS = { 764 'raw': (lambda v, _d: [repr(v)]), 765 'brief': _fmt_line, 766 'line': _fmt_line, 767 'full': _fmt_full, 768} 769 770 771### declarations ## 772 773class Declaration(HighlevelParsedItem): 774 775 @classmethod 776 def from_row(cls, row, **markers): 777 fixed = tuple(_tables.fix_row(row, **markers)) 778 if cls is Declaration: 779 _, _, _, kind, _ = fixed 780 sub = KIND_CLASSES.get(KIND(kind)) 781 if not sub or not issubclass(sub, Declaration): 782 raise TypeError(f'unsupported kind, got {row!r}') 783 else: 784 sub = cls 785 return sub._from_row(fixed) 786 787 @classmethod 788 def _from_row(cls, row): 789 filename, funcname, name, kind, data = row 790 kind = KIND._from_raw(kind) 791 if kind is not cls.kind: 792 raise TypeError(f'expected kind {cls.kind.value!r}, got {row!r}') 793 fileinfo = FileInfo.from_raw(filename) 794 extra = None 795 if isinstance(data, str): 796 data, extra = cls._parse_data(data, fmt='row') 797 if extra: 798 return cls(fileinfo, name, data, funcname, _extra=extra) 799 else: 800 return cls(fileinfo, name, data, funcname) 801 802 @classmethod 803 def _resolve_parent(cls, parsed, *, _kind=None): 804 if _kind is None: 805 raise TypeError(f'{cls.kind.value} declarations do not have parents ({parsed})') 806 return super()._resolve_parent(parsed, _kind=_kind) 807 808 @classmethod 809 def _render_data(cls, fmt, data, extra): 810 if not data: 811 # XXX There should be some! Forward? 812 yield '???' 813 else: 814 yield from cls._format_data(fmt, data, extra) 815 816 @classmethod 817 def _render_data_row_item(cls, colname, data, extra): 818 if colname == 'data': 819 return cls._format_data('row', data, extra) 820 else: 821 return None 822 823 @classmethod 824 def _format_data(cls, fmt, data, extra): 825 raise NotImplementedError(fmt) 826 827 @classmethod 828 def _parse_data(cls, datastr, fmt=None): 829 """This is the reverse of _render_data.""" 830 if not datastr or datastr is _tables.UNKNOWN or datastr == '???': 831 return None, None 832 elif datastr is _tables.EMPTY or datastr == '-': 833 # All the kinds have *something* even it is unknown. 834 raise TypeError('all declarations have data of some sort, got none') 835 else: 836 return cls._unformat_data(datastr, fmt) 837 838 @classmethod 839 def _unformat_data(cls, datastr, fmt=None): 840 raise NotImplementedError(fmt) 841 842 843class VarType(namedtuple('VarType', 'typequal typespec abstract')): 844 845 @classmethod 846 def from_str(cls, text): 847 orig = text 848 storage, sep, text = text.strip().partition(' ') 849 if not sep: 850 text = storage 851 storage = None 852 elif storage not in ('auto', 'register', 'static', 'extern'): 853 text = orig 854 storage = None 855 return cls._from_str(text), storage 856 857 @classmethod 858 def _from_str(cls, text): 859 orig = text 860 if text.startswith(('const ', 'volatile ')): 861 typequal, _, text = text.partition(' ') 862 else: 863 typequal = None 864 865 # Extract a series of identifiers/keywords. 866 m = re.match(r"^ *'?([a-zA-Z_]\w*(?:\s+[a-zA-Z_]\w*)*)\s*(.*?)'?\s*$", text) 867 if not m: 868 raise ValueError(f'invalid vartype text {orig!r}') 869 typespec, abstract = m.groups() 870 871 return cls(typequal, typespec, abstract or None) 872 873 def __str__(self): 874 parts = [] 875 if self.qualifier: 876 parts.append(self.qualifier) 877 parts.append(self.spec + (self.abstract or '')) 878 return ' '.join(parts) 879 880 @property 881 def qualifier(self): 882 return self.typequal 883 884 @property 885 def spec(self): 886 return self.typespec 887 888 889class Variable(Declaration): 890 kind = KIND.VARIABLE 891 892 @classmethod 893 def _resolve_parent(cls, parsed): 894 return super()._resolve_parent(parsed, _kind=KIND.FUNCTION) 895 896 @classmethod 897 def _resolve_data(cls, data): 898 if not data: 899 return None, None 900 storage, vartype = _get_vartype(data) 901 return VarType(**vartype), {'storage': storage} 902 903 @classmethod 904 def _raw_data(self, data, extra): 905 vartype = data._asdict() 906 return { 907 'storage': extra['storage'], 908 'vartype': vartype, 909 } 910 911 @classmethod 912 def _format_data(cls, fmt, data, extra): 913 storage = extra.get('storage') 914 text = f'{storage} {data}' if storage else str(data) 915 if fmt in ('line', 'brief'): 916 yield text 917 #elif fmt == 'full': 918 elif fmt == 'row': 919 yield text 920 else: 921 raise NotImplementedError(fmt) 922 923 @classmethod 924 def _unformat_data(cls, datastr, fmt=None): 925 if fmt in ('line', 'brief'): 926 vartype, storage = VarType.from_str(datastr) 927 return vartype, {'storage': storage} 928 #elif fmt == 'full': 929 elif fmt == 'row': 930 vartype, storage = VarType.from_str(datastr) 931 return vartype, {'storage': storage} 932 else: 933 raise NotImplementedError(fmt) 934 935 def __init__(self, file, name, data, parent=None, storage=None): 936 super().__init__(file, name, data, parent, 937 _extra={'storage': storage or None}, 938 _shortkey=f'({parent.name}).{name}' if parent else name, 939 _key=(str(file), 940 # Tilde comes after all other ascii characters. 941 f'~{parent or ""}~', 942 name, 943 ), 944 ) 945 if storage: 946 if storage not in STORAGE: 947 # The parser must need an update. 948 raise NotImplementedError(storage) 949 # Otherwise we trust the compiler to have validated it. 950 951 @property 952 def vartype(self): 953 return self.data 954 955 956class Signature(namedtuple('Signature', 'params returntype inline isforward')): 957 958 @classmethod 959 def from_str(cls, text): 960 orig = text 961 storage, sep, text = text.strip().partition(' ') 962 if not sep: 963 text = storage 964 storage = None 965 elif storage not in ('auto', 'register', 'static', 'extern'): 966 text = orig 967 storage = None 968 return cls._from_str(text), storage 969 970 @classmethod 971 def _from_str(cls, text): 972 orig = text 973 inline, sep, text = text.partition('|') 974 if not sep: 975 text = inline 976 inline = None 977 978 isforward = False 979 if text.endswith(';'): 980 text = text[:-1] 981 isforward = True 982 elif text.endswith('{}'): 983 text = text[:-2] 984 985 index = text.rindex('(') 986 if index < 0: 987 raise ValueError(f'bad signature text {orig!r}') 988 params = text[index:] 989 while params.count('(') <= params.count(')'): 990 index = text.rindex('(', 0, index) 991 if index < 0: 992 raise ValueError(f'bad signature text {orig!r}') 993 params = text[index:] 994 text = text[:index] 995 996 returntype = VarType._from_str(text.rstrip()) 997 998 return cls(params, returntype, inline, isforward) 999 1000 def __str__(self): 1001 parts = [] 1002 if self.inline: 1003 parts.extend([ 1004 self.inline, 1005 '|', 1006 ]) 1007 parts.extend([ 1008 str(self.returntype), 1009 self.params, 1010 ';' if self.isforward else '{}', 1011 ]) 1012 return ' '.join(parts) 1013 1014 @property 1015 def returns(self): 1016 return self.returntype 1017 1018 @property 1019 def typequal(self): 1020 return self.returntype.typequal 1021 1022 @property 1023 def typespec(self): 1024 return self.returntype.typespec 1025 1026 @property 1027 def abstract(self): 1028 return self.returntype.abstract 1029 1030 1031class Function(Declaration): 1032 kind = KIND.FUNCTION 1033 1034 @classmethod 1035 def _resolve_data(cls, data): 1036 if not data: 1037 return None, None 1038 kwargs = dict(data) 1039 returntype = dict(data['returntype']) 1040 del returntype['storage'] 1041 kwargs['returntype'] = VarType(**returntype) 1042 storage = kwargs.pop('storage') 1043 return Signature(**kwargs), {'storage': storage} 1044 1045 @classmethod 1046 def _raw_data(self, data): 1047 # XXX finish! 1048 return data 1049 1050 @classmethod 1051 def _format_data(cls, fmt, data, extra): 1052 storage = extra.get('storage') 1053 text = f'{storage} {data}' if storage else str(data) 1054 if fmt in ('line', 'brief'): 1055 yield text 1056 #elif fmt == 'full': 1057 elif fmt == 'row': 1058 yield text 1059 else: 1060 raise NotImplementedError(fmt) 1061 1062 @classmethod 1063 def _unformat_data(cls, datastr, fmt=None): 1064 if fmt in ('line', 'brief'): 1065 sig, storage = Signature.from_str(sig) 1066 return sig, {'storage': storage} 1067 #elif fmt == 'full': 1068 elif fmt == 'row': 1069 sig, storage = Signature.from_str(sig) 1070 return sig, {'storage': storage} 1071 else: 1072 raise NotImplementedError(fmt) 1073 1074 def __init__(self, file, name, data, parent=None, storage=None): 1075 super().__init__(file, name, data, parent, _extra={'storage': storage}) 1076 self._shortkey = f'~{name}~ {self.data}' 1077 self._key = ( 1078 str(file), 1079 self._shortkey, 1080 ) 1081 1082 @property 1083 def signature(self): 1084 return self.data 1085 1086 1087class TypeDeclaration(Declaration): 1088 1089 def __init__(self, file, name, data, parent=None, *, _shortkey=None): 1090 if not _shortkey: 1091 _shortkey = f'{self.kind.value} {name}' 1092 super().__init__(file, name, data, parent, 1093 _shortkey=_shortkey, 1094 _key=( 1095 str(file), 1096 _shortkey, 1097 ), 1098 ) 1099 1100 1101class POTSType(TypeDeclaration): 1102 1103 def __init__(self, name): 1104 _file = _data = _parent = None 1105 super().__init__(_file, name, _data, _parent, _shortkey=name) 1106 1107 1108class FuncPtr(TypeDeclaration): 1109 1110 def __init__(self, vartype): 1111 _file = _name = _parent = None 1112 data = vartype 1113 self.vartype = vartype 1114 super().__init__(_file, _name, data, _parent, _shortkey=f'<{vartype}>') 1115 1116 1117class TypeDef(TypeDeclaration): 1118 kind = KIND.TYPEDEF 1119 1120 @classmethod 1121 def _resolve_data(cls, data): 1122 if not data: 1123 raise NotImplementedError(data) 1124 kwargs = dict(data) 1125 del kwargs['storage'] 1126 if 'returntype' in kwargs: 1127 vartype = kwargs['returntype'] 1128 del vartype['storage'] 1129 kwargs['returntype'] = VarType(**vartype) 1130 datacls = Signature 1131 else: 1132 datacls = VarType 1133 return datacls(**kwargs), None 1134 1135 @classmethod 1136 def _raw_data(self, data): 1137 # XXX finish! 1138 return data 1139 1140 @classmethod 1141 def _format_data(cls, fmt, data, extra): 1142 text = str(data) 1143 if fmt in ('line', 'brief'): 1144 yield text 1145 elif fmt == 'full': 1146 yield text 1147 elif fmt == 'row': 1148 yield text 1149 else: 1150 raise NotImplementedError(fmt) 1151 1152 @classmethod 1153 def _unformat_data(cls, datastr, fmt=None): 1154 if fmt in ('line', 'brief'): 1155 vartype, _ = VarType.from_str(datastr) 1156 return vartype, None 1157 #elif fmt == 'full': 1158 elif fmt == 'row': 1159 vartype, _ = VarType.from_str(datastr) 1160 return vartype, None 1161 else: 1162 raise NotImplementedError(fmt) 1163 1164 def __init__(self, file, name, data, parent=None): 1165 super().__init__(file, name, data, parent, _shortkey=name) 1166 1167 @property 1168 def vartype(self): 1169 return self.data 1170 1171 1172class Member(namedtuple('Member', 'name vartype size')): 1173 1174 @classmethod 1175 def from_data(cls, raw, index): 1176 name = raw.name if raw.name else index 1177 vartype = size = None 1178 if type(raw.data) is int: 1179 size = raw.data 1180 elif isinstance(raw.data, str): 1181 size = int(raw.data) 1182 elif raw.data: 1183 vartype = dict(raw.data) 1184 del vartype['storage'] 1185 if 'size' in vartype: 1186 size = vartype.pop('size') 1187 if isinstance(size, str) and size.isdigit(): 1188 size = int(size) 1189 vartype = VarType(**vartype) 1190 return cls(name, vartype, size) 1191 1192 @classmethod 1193 def from_str(cls, text): 1194 name, _, vartype = text.partition(': ') 1195 if name.startswith('#'): 1196 name = int(name[1:]) 1197 if vartype.isdigit(): 1198 size = int(vartype) 1199 vartype = None 1200 else: 1201 vartype, _ = VarType.from_str(vartype) 1202 size = None 1203 return cls(name, vartype, size) 1204 1205 def __str__(self): 1206 name = self.name if isinstance(self.name, str) else f'#{self.name}' 1207 return f'{name}: {self.vartype or self.size}' 1208 1209 1210class _StructUnion(TypeDeclaration): 1211 1212 @classmethod 1213 def _resolve_data(cls, data): 1214 if not data: 1215 # XXX There should be some! Forward? 1216 return None, None 1217 return [Member.from_data(v, i) for i, v in enumerate(data)], None 1218 1219 @classmethod 1220 def _raw_data(self, data): 1221 # XXX finish! 1222 return data 1223 1224 @classmethod 1225 def _format_data(cls, fmt, data, extra): 1226 if fmt in ('line', 'brief'): 1227 members = ', '.join(f'<{m}>' for m in data) 1228 yield f'[{members}]' 1229 elif fmt == 'full': 1230 for member in data: 1231 yield f'{member}' 1232 elif fmt == 'row': 1233 members = ', '.join(f'<{m}>' for m in data) 1234 yield f'[{members}]' 1235 else: 1236 raise NotImplementedError(fmt) 1237 1238 @classmethod 1239 def _unformat_data(cls, datastr, fmt=None): 1240 if fmt in ('line', 'brief'): 1241 members = [Member.from_str(m[1:-1]) 1242 for m in datastr[1:-1].split(', ')] 1243 return members, None 1244 #elif fmt == 'full': 1245 elif fmt == 'row': 1246 members = [Member.from_str(m.rstrip('>').lstrip('<')) 1247 for m in datastr[1:-1].split('>, <')] 1248 return members, None 1249 else: 1250 raise NotImplementedError(fmt) 1251 1252 def __init__(self, file, name, data, parent=None): 1253 super().__init__(file, name, data, parent) 1254 1255 @property 1256 def members(self): 1257 return self.data 1258 1259 1260class Struct(_StructUnion): 1261 kind = KIND.STRUCT 1262 1263 1264class Union(_StructUnion): 1265 kind = KIND.UNION 1266 1267 1268class Enum(TypeDeclaration): 1269 kind = KIND.ENUM 1270 1271 @classmethod 1272 def _resolve_data(cls, data): 1273 if not data: 1274 # XXX There should be some! Forward? 1275 return None, None 1276 enumerators = [e if isinstance(e, str) else e.name 1277 for e in data] 1278 return enumerators, None 1279 1280 @classmethod 1281 def _raw_data(self, data): 1282 # XXX finish! 1283 return data 1284 1285 @classmethod 1286 def _format_data(cls, fmt, data, extra): 1287 if fmt in ('line', 'brief'): 1288 yield repr(data) 1289 elif fmt == 'full': 1290 for enumerator in data: 1291 yield f'{enumerator}' 1292 elif fmt == 'row': 1293 # XXX This won't work with CSV... 1294 yield ','.join(data) 1295 else: 1296 raise NotImplementedError(fmt) 1297 1298 @classmethod 1299 def _unformat_data(cls, datastr, fmt=None): 1300 if fmt in ('line', 'brief'): 1301 return _strutil.unrepr(datastr), None 1302 #elif fmt == 'full': 1303 elif fmt == 'row': 1304 return datastr.split(','), None 1305 else: 1306 raise NotImplementedError(fmt) 1307 1308 def __init__(self, file, name, data, parent=None): 1309 super().__init__(file, name, data, parent) 1310 1311 @property 1312 def enumerators(self): 1313 return self.data 1314 1315 1316### statements ### 1317 1318class Statement(HighlevelParsedItem): 1319 kind = KIND.STATEMENT 1320 1321 @classmethod 1322 def _resolve_data(cls, data): 1323 # XXX finish! 1324 return data, None 1325 1326 @classmethod 1327 def _raw_data(self, data): 1328 # XXX finish! 1329 return data 1330 1331 @classmethod 1332 def _render_data(cls, fmt, data, extra): 1333 # XXX Handle other formats? 1334 return repr(data) 1335 1336 @classmethod 1337 def _parse_data(self, datastr, fmt=None): 1338 # XXX Handle other formats? 1339 return _strutil.unrepr(datastr), None 1340 1341 def __init__(self, file, name, data, parent=None): 1342 super().__init__(file, name, data, parent, 1343 _shortkey=data or '', 1344 _key=( 1345 str(file), 1346 file.lno, 1347 # XXX Only one stmt per line? 1348 ), 1349 ) 1350 1351 @property 1352 def text(self): 1353 return self.data 1354 1355 1356### 1357 1358KIND_CLASSES = {cls.kind: cls for cls in [ 1359 Variable, 1360 Function, 1361 TypeDef, 1362 Struct, 1363 Union, 1364 Enum, 1365 Statement, 1366]} 1367 1368 1369def resolve_parsed(parsed): 1370 if isinstance(parsed, HighlevelParsedItem): 1371 return parsed 1372 try: 1373 cls = KIND_CLASSES[parsed.kind] 1374 except KeyError: 1375 raise ValueError(f'unsupported kind in {parsed!r}') 1376 return cls.from_parsed(parsed) 1377 1378 1379def set_flag(item, name, value): 1380 try: 1381 setattr(item, name, value) 1382 except AttributeError: 1383 object.__setattr__(item, name, value) 1384 1385 1386############################# 1387# composite 1388 1389class Declarations: 1390 1391 @classmethod 1392 def from_decls(cls, decls): 1393 return cls(decls) 1394 1395 @classmethod 1396 def from_parsed(cls, items): 1397 decls = (resolve_parsed(item) 1398 for item in items 1399 if item.kind is not KIND.STATEMENT) 1400 return cls.from_decls(decls) 1401 1402 @classmethod 1403 def _resolve_key(cls, raw): 1404 if isinstance(raw, str): 1405 raw = [raw] 1406 elif isinstance(raw, Declaration): 1407 raw = ( 1408 raw.filename if cls._is_public(raw) else None, 1409 # `raw.parent` is always None for types and functions. 1410 raw.parent if raw.kind is KIND.VARIABLE else None, 1411 raw.name, 1412 ) 1413 1414 extra = None 1415 if len(raw) == 1: 1416 name, = raw 1417 if name: 1418 name = str(name) 1419 if name.endswith(('.c', '.h')): 1420 # This is only legit as a query. 1421 key = (name, None, None) 1422 else: 1423 key = (None, None, name) 1424 else: 1425 key = (None, None, None) 1426 elif len(raw) == 2: 1427 parent, name = raw 1428 name = str(name) 1429 if isinstance(parent, Declaration): 1430 key = (None, parent.name, name) 1431 elif not parent: 1432 key = (None, None, name) 1433 else: 1434 parent = str(parent) 1435 if parent.endswith(('.c', '.h')): 1436 key = (parent, None, name) 1437 else: 1438 key = (None, parent, name) 1439 else: 1440 key, extra = raw[:3], raw[3:] 1441 filename, funcname, name = key 1442 filename = str(filename) if filename else None 1443 if isinstance(funcname, Declaration): 1444 funcname = funcname.name 1445 else: 1446 funcname = str(funcname) if funcname else None 1447 name = str(name) if name else None 1448 key = (filename, funcname, name) 1449 return key, extra 1450 1451 @classmethod 1452 def _is_public(cls, decl): 1453 # For .c files don't we need info from .h files to make this decision? 1454 # XXX Check for "extern". 1455 # For now we treat all decls a "private" (have filename set). 1456 return False 1457 1458 def __init__(self, decls): 1459 # (file, func, name) -> decl 1460 # "public": 1461 # * (None, None, name) 1462 # "private", "global": 1463 # * (file, None, name) 1464 # "private", "local": 1465 # * (file, func, name) 1466 if hasattr(decls, 'items'): 1467 self._decls = decls 1468 else: 1469 self._decls = {} 1470 self._extend(decls) 1471 1472 # XXX always validate? 1473 1474 def validate(self): 1475 for key, decl in self._decls.items(): 1476 if type(key) is not tuple or len(key) != 3: 1477 raise ValueError(f'expected 3-tuple key, got {key!r} (for decl {decl!r})') 1478 filename, funcname, name = key 1479 if not name: 1480 raise ValueError(f'expected name in key, got {key!r} (for decl {decl!r})') 1481 elif type(name) is not str: 1482 raise ValueError(f'expected name in key to be str, got {key!r} (for decl {decl!r})') 1483 # XXX Check filename type? 1484 # XXX Check funcname type? 1485 1486 if decl.kind is KIND.STATEMENT: 1487 raise ValueError(f'expected a declaration, got {decl!r}') 1488 1489 def __repr__(self): 1490 return f'{type(self).__name__}({list(self)})' 1491 1492 def __len__(self): 1493 return len(self._decls) 1494 1495 def __iter__(self): 1496 yield from self._decls 1497 1498 def __getitem__(self, key): 1499 # XXX Be more exact for the 3-tuple case? 1500 if type(key) not in (str, tuple): 1501 raise KeyError(f'unsupported key {key!r}') 1502 resolved, extra = self._resolve_key(key) 1503 if extra: 1504 raise KeyError(f'key must have at most 3 parts, got {key!r}') 1505 if not resolved[2]: 1506 raise ValueError(f'expected name in key, got {key!r}') 1507 try: 1508 return self._decls[resolved] 1509 except KeyError: 1510 if type(key) is tuple and len(key) == 3: 1511 filename, funcname, name = key 1512 else: 1513 filename, funcname, name = resolved 1514 if filename and not filename.endswith(('.c', '.h')): 1515 raise KeyError(f'invalid filename in key {key!r}') 1516 elif funcname and funcname.endswith(('.c', '.h')): 1517 raise KeyError(f'invalid funcname in key {key!r}') 1518 elif name and name.endswith(('.c', '.h')): 1519 raise KeyError(f'invalid name in key {key!r}') 1520 else: 1521 raise # re-raise 1522 1523 @property 1524 def types(self): 1525 return self._find(kind=KIND.TYPES) 1526 1527 @property 1528 def functions(self): 1529 return self._find(None, None, None, KIND.FUNCTION) 1530 1531 @property 1532 def variables(self): 1533 return self._find(None, None, None, KIND.VARIABLE) 1534 1535 def iter_all(self): 1536 yield from self._decls.values() 1537 1538 def get(self, key, default=None): 1539 try: 1540 return self[key] 1541 except KeyError: 1542 return default 1543 1544 #def add_decl(self, decl, key=None): 1545 # decl = _resolve_parsed(decl) 1546 # self._add_decl(decl, key) 1547 1548 def find(self, *key, **explicit): 1549 if not key: 1550 if not explicit: 1551 return iter(self) 1552 return self._find(**explicit) 1553 1554 resolved, extra = self._resolve_key(key) 1555 filename, funcname, name = resolved 1556 if not extra: 1557 kind = None 1558 elif len(extra) == 1: 1559 kind, = extra 1560 else: 1561 raise KeyError(f'key must have at most 4 parts, got {key!r}') 1562 1563 implicit= {} 1564 if filename: 1565 implicit['filename'] = filename 1566 if funcname: 1567 implicit['funcname'] = funcname 1568 if name: 1569 implicit['name'] = name 1570 if kind: 1571 implicit['kind'] = kind 1572 return self._find(**implicit, **explicit) 1573 1574 def _find(self, filename=None, funcname=None, name=None, kind=None): 1575 for decl in self._decls.values(): 1576 if filename and decl.filename != filename: 1577 continue 1578 if funcname: 1579 if decl.kind is not KIND.VARIABLE: 1580 continue 1581 if decl.parent.name != funcname: 1582 continue 1583 if name and decl.name != name: 1584 continue 1585 if kind: 1586 kinds = KIND.resolve_group(kind) 1587 if decl.kind not in kinds: 1588 continue 1589 yield decl 1590 1591 def _add_decl(self, decl, key=None): 1592 if key: 1593 if type(key) not in (str, tuple): 1594 raise NotImplementedError((key, decl)) 1595 # Any partial key will be turned into a full key, but that 1596 # same partial key will still match a key lookup. 1597 resolved, _ = self._resolve_key(key) 1598 if not resolved[2]: 1599 raise ValueError(f'expected name in key, got {key!r}') 1600 key = resolved 1601 # XXX Also add with the decl-derived key if not the same? 1602 else: 1603 key, _ = self._resolve_key(decl) 1604 self._decls[key] = decl 1605 1606 def _extend(self, decls): 1607 decls = iter(decls) 1608 # Check only the first item. 1609 for decl in decls: 1610 if isinstance(decl, Declaration): 1611 self._add_decl(decl) 1612 # Add the rest without checking. 1613 for decl in decls: 1614 self._add_decl(decl) 1615 elif isinstance(decl, HighlevelParsedItem): 1616 raise NotImplementedError(decl) 1617 else: 1618 try: 1619 key, decl = decl 1620 except ValueError: 1621 raise NotImplementedError(decl) 1622 if not isinstance(decl, Declaration): 1623 raise NotImplementedError(decl) 1624 self._add_decl(decl, key) 1625 # Add the rest without checking. 1626 for key, decl in decls: 1627 self._add_decl(decl, key) 1628 # The iterator will be exhausted at this point. 1629