1from collections import namedtuple 2import enum 3import os.path 4import re 5 6from c_common import fsutil 7from c_common.clsutil import classonly 8import c_common.misc as _misc 9import c_common.strutil as _strutil 10import c_common.tables as _tables 11from .parser._regexes import SIMPLE_TYPE, _STORAGE 12 13 14FIXED_TYPE = _misc.Labeled('FIXED_TYPE') 15 16STORAGE = frozenset(_STORAGE) 17 18 19############################# 20# kinds 21 22@enum.unique 23class KIND(enum.Enum): 24 25 # XXX Use these in the raw parser code. 26 TYPEDEF = 'typedef' 27 STRUCT = 'struct' 28 UNION = 'union' 29 ENUM = 'enum' 30 FUNCTION = 'function' 31 VARIABLE = 'variable' 32 STATEMENT = 'statement' 33 34 @classonly 35 def _from_raw(cls, raw): 36 if raw is None: 37 return None 38 elif isinstance(raw, cls): 39 return raw 40 elif type(raw) is str: 41 # We could use cls[raw] for the upper-case form, 42 # but there's no need to go to the trouble. 43 return cls(raw.lower()) 44 else: 45 raise NotImplementedError(raw) 46 47 @classonly 48 def by_priority(cls, group=None): 49 if group is None: 50 return cls._ALL_BY_PRIORITY.copy() 51 elif group == 'type': 52 return cls._TYPE_DECLS_BY_PRIORITY.copy() 53 elif group == 'decl': 54 return cls._ALL_DECLS_BY_PRIORITY.copy() 55 elif isinstance(group, str): 56 raise NotImplementedError(group) 57 else: 58 # XXX Treat group as a set of kinds & return in priority order? 59 raise NotImplementedError(group) 60 61 @classonly 62 def is_type_decl(cls, kind): 63 if kind in cls.TYPES: 64 return True 65 if not isinstance(kind, cls): 66 raise TypeError(f'expected KIND, got {kind!r}') 67 return False 68 69 @classonly 70 def is_decl(cls, kind): 71 if kind in cls.DECLS: 72 return True 73 if not isinstance(kind, cls): 74 raise TypeError(f'expected KIND, got {kind!r}') 75 return False 76 77 @classonly 78 def get_group(cls, kind, *, groups=None): 79 if not isinstance(kind, cls): 80 raise TypeError(f'expected KIND, got {kind!r}') 81 if groups is None: 82 groups = ['type'] 83 elif not groups: 84 groups = () 85 elif isinstance(groups, str): 86 group = groups 87 if group not in cls._GROUPS: 88 raise ValueError(f'unsupported group {group!r}') 89 groups = [group] 90 else: 91 unsupported = [g for g in groups if g not in cls._GROUPS] 92 if unsupported: 93 raise ValueError(f'unsupported groups {", ".join(repr(unsupported))}') 94 for group in groups: 95 if kind in cls._GROUPS[group]: 96 return group 97 else: 98 return kind.value 99 100 @classonly 101 def resolve_group(cls, group): 102 if isinstance(group, cls): 103 return {group} 104 elif isinstance(group, str): 105 try: 106 return cls._GROUPS[group].copy() 107 except KeyError: 108 raise ValueError(f'unsupported group {group!r}') 109 else: 110 resolved = set() 111 for gr in group: 112 resolve.update(cls.resolve_group(gr)) 113 return resolved 114 #return {*cls.resolve_group(g) for g in group} 115 116 117KIND._TYPE_DECLS_BY_PRIORITY = [ 118 # These are in preferred order. 119 KIND.TYPEDEF, 120 KIND.STRUCT, 121 KIND.UNION, 122 KIND.ENUM, 123] 124KIND._ALL_DECLS_BY_PRIORITY = [ 125 # These are in preferred order. 126 *KIND._TYPE_DECLS_BY_PRIORITY, 127 KIND.FUNCTION, 128 KIND.VARIABLE, 129] 130KIND._ALL_BY_PRIORITY = [ 131 # These are in preferred order. 132 *KIND._ALL_DECLS_BY_PRIORITY, 133 KIND.STATEMENT, 134] 135 136KIND.TYPES = frozenset(KIND._TYPE_DECLS_BY_PRIORITY) 137KIND.DECLS = frozenset(KIND._ALL_DECLS_BY_PRIORITY) 138KIND._GROUPS = { 139 'type': KIND.TYPES, 140 'decl': KIND.DECLS, 141} 142KIND._GROUPS.update((k.value, {k}) for k in KIND) 143 144 145def get_kind_group(item): 146 return KIND.get_group(item.kind) 147 148 149############################# 150# low-level 151 152def _fix_filename(filename, relroot, *, 153 formatted=True, 154 **kwargs): 155 if formatted: 156 fix = fsutil.format_filename 157 else: 158 fix = fsutil.fix_filename 159 return fix(filename, relroot=relroot, **kwargs) 160 161 162class FileInfo(namedtuple('FileInfo', 'filename lno')): 163 @classmethod 164 def from_raw(cls, raw): 165 if isinstance(raw, cls): 166 return raw 167 elif isinstance(raw, tuple): 168 return cls(*raw) 169 elif not raw: 170 return None 171 elif isinstance(raw, str): 172 return cls(raw, -1) 173 else: 174 raise TypeError(f'unsupported "raw": {raw:!r}') 175 176 def __str__(self): 177 return self.filename 178 179 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): 180 filename = _fix_filename(self.filename, relroot, **kwargs) 181 if filename == self.filename: 182 return self 183 return self._replace(filename=filename) 184 185 186class SourceLine(namedtuple('Line', 'file kind data conditions')): 187 KINDS = ( 188 #'directive', # data is ... 189 'source', # "data" is the line 190 #'comment', # "data" is the text, including comment markers 191 ) 192 193 @property 194 def filename(self): 195 return self.file.filename 196 197 @property 198 def lno(self): 199 return self.file.lno 200 201 202class DeclID(namedtuple('DeclID', 'filename funcname name')): 203 """The globally-unique identifier for a declaration.""" 204 205 @classmethod 206 def from_row(cls, row, **markers): 207 row = _tables.fix_row(row, **markers) 208 return cls(*row) 209 210 # We have to provde _make() becaose we implemented __new__(). 211 212 @classmethod 213 def _make(cls, iterable): 214 try: 215 return cls(*iterable) 216 except Exception: 217 super()._make(iterable) 218 raise # re-raise 219 220 def __new__(cls, filename, funcname, name): 221 self = super().__new__( 222 cls, 223 filename=str(filename) if filename else None, 224 funcname=str(funcname) if funcname else None, 225 name=str(name) if name else None, 226 ) 227 self._compare = tuple(v or '' for v in self) 228 return self 229 230 def __hash__(self): 231 return super().__hash__() 232 233 def __eq__(self, other): 234 try: 235 other = tuple(v or '' for v in other) 236 except TypeError: 237 return NotImplemented 238 return self._compare == other 239 240 def __gt__(self, other): 241 try: 242 other = tuple(v or '' for v in other) 243 except TypeError: 244 return NotImplemented 245 return self._compare > other 246 247 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): 248 filename = _fix_filename(self.filename, relroot, **kwargs) 249 if filename == self.filename: 250 return self 251 return self._replace(filename=filename) 252 253 254class ParsedItem(namedtuple('ParsedItem', 'file kind parent name data')): 255 256 @classmethod 257 def from_raw(cls, raw): 258 if isinstance(raw, cls): 259 return raw 260 elif isinstance(raw, tuple): 261 return cls(*raw) 262 else: 263 raise TypeError(f'unsupported "raw": {raw:!r}') 264 265 @classmethod 266 def from_row(cls, row, columns=None): 267 if not columns: 268 colnames = 'filename funcname name kind data'.split() 269 else: 270 colnames = list(columns) 271 for i, column in enumerate(colnames): 272 if column == 'file': 273 colnames[i] = 'filename' 274 elif column == 'funcname': 275 colnames[i] = 'parent' 276 if len(row) != len(set(colnames)): 277 raise NotImplementedError(columns, row) 278 kwargs = {} 279 for column, value in zip(colnames, row): 280 if column == 'filename': 281 kwargs['file'] = FileInfo.from_raw(value) 282 elif column == 'kind': 283 kwargs['kind'] = KIND(value) 284 elif column in cls._fields: 285 kwargs[column] = value 286 else: 287 raise NotImplementedError(column) 288 return cls(**kwargs) 289 290 @property 291 def id(self): 292 try: 293 return self._id 294 except AttributeError: 295 if self.kind is KIND.STATEMENT: 296 self._id = None 297 else: 298 self._id = DeclID(str(self.file), self.funcname, self.name) 299 return self._id 300 301 @property 302 def filename(self): 303 if not self.file: 304 return None 305 return self.file.filename 306 307 @property 308 def lno(self): 309 if not self.file: 310 return -1 311 return self.file.lno 312 313 @property 314 def funcname(self): 315 if not self.parent: 316 return None 317 if type(self.parent) is str: 318 return self.parent 319 else: 320 return self.parent.name 321 322 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): 323 fixed = self.file.fix_filename(relroot, **kwargs) 324 if fixed == self.file: 325 return self 326 return self._replace(file=fixed) 327 328 def as_row(self, columns=None): 329 if not columns: 330 columns = self._fields 331 row = [] 332 for column in columns: 333 if column == 'file': 334 value = self.filename 335 elif column == 'kind': 336 value = self.kind.value 337 elif column == 'data': 338 value = self._render_data() 339 else: 340 value = getattr(self, column) 341 row.append(value) 342 return row 343 344 def _render_data(self): 345 if not self.data: 346 return None 347 elif isinstance(self.data, str): 348 return self.data 349 else: 350 # XXX 351 raise NotImplementedError 352 353 354def _get_vartype(data): 355 try: 356 vartype = dict(data['vartype']) 357 except KeyError: 358 vartype = dict(data) 359 storage = data.get('storage') 360 else: 361 storage = data.get('storage') or vartype.get('storage') 362 del vartype['storage'] 363 return storage, vartype 364 365 366def get_parsed_vartype(decl): 367 kind = getattr(decl, 'kind', None) 368 if isinstance(decl, ParsedItem): 369 storage, vartype = _get_vartype(decl.data) 370 typequal = vartype['typequal'] 371 typespec = vartype['typespec'] 372 abstract = vartype['abstract'] 373 elif isinstance(decl, dict): 374 kind = decl.get('kind') 375 storage, vartype = _get_vartype(decl) 376 typequal = vartype['typequal'] 377 typespec = vartype['typespec'] 378 abstract = vartype['abstract'] 379 elif isinstance(decl, VarType): 380 storage = None 381 typequal, typespec, abstract = decl 382 elif isinstance(decl, TypeDef): 383 storage = None 384 typequal, typespec, abstract = decl.vartype 385 elif isinstance(decl, Variable): 386 storage = decl.storage 387 typequal, typespec, abstract = decl.vartype 388 elif isinstance(decl, Function): 389 storage = decl.storage 390 typequal, typespec, abstract = decl.signature.returntype 391 elif isinstance(decl, str): 392 vartype, storage = VarType.from_str(decl) 393 typequal, typespec, abstract = vartype 394 else: 395 raise NotImplementedError(decl) 396 return kind, storage, typequal, typespec, abstract 397 398 399def get_default_storage(decl): 400 if decl.kind not in (KIND.VARIABLE, KIND.FUNCTION): 401 return None 402 return 'extern' if decl.parent is None else 'auto' 403 404 405def get_effective_storage(decl, *, default=None): 406 # Note that "static" limits access to just that C module 407 # and "extern" (the default for module-level) allows access 408 # outside the C module. 409 if default is None: 410 default = get_default_storage(decl) 411 if default is None: 412 return None 413 try: 414 storage = decl.storage 415 except AttributeError: 416 storage, _ = _get_vartype(decl.data) 417 return storage or default 418 419 420############################# 421# high-level 422 423class HighlevelParsedItem: 424 425 kind = None 426 427 FIELDS = ('file', 'parent', 'name', 'data') 428 429 @classmethod 430 def from_parsed(cls, parsed): 431 if parsed.kind is not cls.kind: 432 raise TypeError(f'kind mismatch ({parsed.kind.value} != {cls.kind.value})') 433 data, extra = cls._resolve_data(parsed.data) 434 self = cls( 435 cls._resolve_file(parsed), 436 parsed.name, 437 data, 438 cls._resolve_parent(parsed) if parsed.parent else None, 439 **extra or {} 440 ) 441 self._parsed = parsed 442 return self 443 444 @classmethod 445 def _resolve_file(cls, parsed): 446 fileinfo = FileInfo.from_raw(parsed.file) 447 if not fileinfo: 448 raise NotImplementedError(parsed) 449 return fileinfo 450 451 @classmethod 452 def _resolve_data(cls, data): 453 return data, None 454 455 @classmethod 456 def _raw_data(cls, data, extra): 457 if isinstance(data, str): 458 return data 459 else: 460 raise NotImplementedError(data) 461 462 @classmethod 463 def _data_as_row(cls, data, extra, colnames): 464 row = {} 465 for colname in colnames: 466 if colname in row: 467 continue 468 rendered = cls._render_data_row_item(colname, data, extra) 469 if rendered is iter(rendered): 470 rendered, = rendered 471 row[colname] = rendered 472 return row 473 474 @classmethod 475 def _render_data_row_item(cls, colname, data, extra): 476 if colname == 'data': 477 return str(data) 478 else: 479 return None 480 481 @classmethod 482 def _render_data_row(cls, fmt, data, extra, colnames): 483 if fmt != 'row': 484 raise NotImplementedError 485 datarow = cls._data_as_row(data, extra, colnames) 486 unresolved = [c for c, v in datarow.items() if v is None] 487 if unresolved: 488 raise NotImplementedError(unresolved) 489 for colname, value in datarow.items(): 490 if type(value) != str: 491 if colname == 'kind': 492 datarow[colname] = value.value 493 else: 494 datarow[colname] = str(value) 495 return datarow 496 497 @classmethod 498 def _render_data(cls, fmt, data, extra): 499 row = cls._render_data_row(fmt, data, extra, ['data']) 500 yield ' '.join(row.values()) 501 502 @classmethod 503 def _resolve_parent(cls, parsed, *, _kind=None): 504 fileinfo = FileInfo(parsed.file.filename, -1) 505 if isinstance(parsed.parent, str): 506 if parsed.parent.isidentifier(): 507 name = parsed.parent 508 else: 509 # XXX It could be something like "<kind> <name>". 510 raise NotImplementedError(repr(parsed.parent)) 511 parent = ParsedItem(fileinfo, _kind, None, name, None) 512 elif type(parsed.parent) is tuple: 513 # XXX It could be something like (kind, name). 514 raise NotImplementedError(repr(parsed.parent)) 515 else: 516 return parsed.parent 517 Parent = KIND_CLASSES.get(_kind, Declaration) 518 return Parent.from_parsed(parent) 519 520 @classmethod 521 def _parse_columns(cls, columns): 522 colnames = {} # {requested -> actual} 523 columns = list(columns or cls.FIELDS) 524 datacolumns = [] 525 for i, colname in enumerate(columns): 526 if colname == 'file': 527 columns[i] = 'filename' 528 colnames['file'] = 'filename' 529 elif colname == 'lno': 530 columns[i] = 'line' 531 colnames['lno'] = 'line' 532 elif colname in ('filename', 'line'): 533 colnames[colname] = colname 534 elif colname == 'data': 535 datacolumns.append(colname) 536 colnames[colname] = None 537 elif colname in cls.FIELDS or colname == 'kind': 538 colnames[colname] = colname 539 else: 540 datacolumns.append(colname) 541 colnames[colname] = None 542 return columns, datacolumns, colnames 543 544 def __init__(self, file, name, data, parent=None, *, 545 _extra=None, 546 _shortkey=None, 547 _key=None, 548 ): 549 self.file = file 550 self.parent = parent or None 551 self.name = name 552 self.data = data 553 self._extra = _extra or {} 554 self._shortkey = _shortkey 555 self._key = _key 556 557 def __repr__(self): 558 args = [f'{n}={getattr(self, n)!r}' 559 for n in ['file', 'name', 'data', 'parent', *(self._extra or ())]] 560 return f'{type(self).__name__}({", ".join(args)})' 561 562 def __str__(self): 563 try: 564 return self._str 565 except AttributeError: 566 self._str = next(self.render()) 567 return self._str 568 569 def __getattr__(self, name): 570 try: 571 return self._extra[name] 572 except KeyError: 573 raise AttributeError(name) 574 575 def __hash__(self): 576 return hash(self._key) 577 578 def __eq__(self, other): 579 if isinstance(other, HighlevelParsedItem): 580 return self._key == other._key 581 elif type(other) is tuple: 582 return self._key == other 583 else: 584 return NotImplemented 585 586 def __gt__(self, other): 587 if isinstance(other, HighlevelParsedItem): 588 return self._key > other._key 589 elif type(other) is tuple: 590 return self._key > other 591 else: 592 return NotImplemented 593 594 @property 595 def id(self): 596 return self.parsed.id 597 598 @property 599 def shortkey(self): 600 return self._shortkey 601 602 @property 603 def key(self): 604 return self._key 605 606 @property 607 def filename(self): 608 if not self.file: 609 return None 610 return self.file.filename 611 612 @property 613 def parsed(self): 614 try: 615 return self._parsed 616 except AttributeError: 617 parent = self.parent 618 if parent is not None and not isinstance(parent, str): 619 parent = parent.name 620 self._parsed = ParsedItem( 621 self.file, 622 self.kind, 623 parent, 624 self.name, 625 self._raw_data(), 626 ) 627 return self._parsed 628 629 def fix_filename(self, relroot=fsutil.USE_CWD, **kwargs): 630 if self.file: 631 self.file = self.file.fix_filename(relroot, **kwargs) 632 return self 633 634 def as_rowdata(self, columns=None): 635 columns, datacolumns, colnames = self._parse_columns(columns) 636 return self._as_row(colnames, datacolumns, self._data_as_row) 637 638 def render_rowdata(self, columns=None): 639 columns, datacolumns, colnames = self._parse_columns(columns) 640 def data_as_row(data, ext, cols): 641 return self._render_data_row('row', data, ext, cols) 642 rowdata = self._as_row(colnames, datacolumns, data_as_row) 643 for column, value in rowdata.items(): 644 colname = colnames.get(column) 645 if not colname: 646 continue 647 if column == 'kind': 648 value = value.value 649 else: 650 if column == 'parent': 651 if self.parent: 652 value = f'({self.parent.kind.value} {self.parent.name})' 653 if not value: 654 value = '-' 655 elif type(value) is VarType: 656 value = repr(str(value)) 657 else: 658 value = str(value) 659 rowdata[column] = value 660 return rowdata 661 662 def _as_row(self, colnames, datacolumns, data_as_row): 663 try: 664 data = data_as_row(self.data, self._extra, datacolumns) 665 except NotImplementedError: 666 data = None 667 row = data or {} 668 for column, colname in colnames.items(): 669 if colname == 'filename': 670 value = self.file.filename if self.file else None 671 elif colname == 'line': 672 value = self.file.lno if self.file else None 673 elif colname is None: 674 value = getattr(self, column, None) 675 else: 676 value = getattr(self, colname, None) 677 row.setdefault(column, value) 678 return row 679 680 def render(self, fmt='line'): 681 fmt = fmt or 'line' 682 try: 683 render = _FORMATS[fmt] 684 except KeyError: 685 raise TypeError(f'unsupported fmt {fmt!r}') 686 try: 687 data = self._render_data(fmt, self.data, self._extra) 688 except NotImplementedError: 689 data = '-' 690 yield from render(self, data) 691 692 693### formats ### 694 695def _fmt_line(parsed, data=None): 696 parts = [ 697 f'<{parsed.kind.value}>', 698 ] 699 parent = '' 700 if parsed.parent: 701 parent = parsed.parent 702 if not isinstance(parent, str): 703 if parent.kind is KIND.FUNCTION: 704 parent = f'{parent.name}()' 705 else: 706 parent = parent.name 707 name = f'<{parent}>.{parsed.name}' 708 else: 709 name = parsed.name 710 if data is None: 711 data = parsed.data 712 elif data is iter(data): 713 data, = data 714 parts.extend([ 715 name, 716 f'<{data}>' if data else '-', 717 f'({str(parsed.file or "<unknown file>")})', 718 ]) 719 yield '\t'.join(parts) 720 721 722def _fmt_full(parsed, data=None): 723 if parsed.kind is KIND.VARIABLE and parsed.parent: 724 prefix = 'local ' 725 suffix = f' ({parsed.parent.name})' 726 else: 727 # XXX Show other prefixes (e.g. global, public) 728 prefix = suffix = '' 729 yield f'{prefix}{parsed.kind.value} {parsed.name!r}{suffix}' 730 for column, info in parsed.render_rowdata().items(): 731 if column == 'kind': 732 continue 733 if column == 'name': 734 continue 735 if column == 'parent' and parsed.kind is not KIND.VARIABLE: 736 continue 737 if column == 'data': 738 if parsed.kind in (KIND.STRUCT, KIND.UNION): 739 column = 'members' 740 elif parsed.kind is KIND.ENUM: 741 column = 'enumerators' 742 elif parsed.kind is KIND.STATEMENT: 743 column = 'text' 744 data, = data 745 else: 746 column = 'signature' 747 data, = data 748 if not data: 749# yield f'\t{column}:\t-' 750 continue 751 elif isinstance(data, str): 752 yield f'\t{column}:\t{data!r}' 753 else: 754 yield f'\t{column}:' 755 for line in data: 756 yield f'\t\t- {line}' 757 else: 758 yield f'\t{column}:\t{info}' 759 760 761_FORMATS = { 762 'raw': (lambda v, _d: [repr(v)]), 763 'brief': _fmt_line, 764 'line': _fmt_line, 765 'full': _fmt_full, 766} 767 768 769### declarations ## 770 771class Declaration(HighlevelParsedItem): 772 773 @classmethod 774 def from_row(cls, row, **markers): 775 fixed = tuple(_tables.fix_row(row, **markers)) 776 if cls is Declaration: 777 _, _, _, kind, _ = fixed 778 sub = KIND_CLASSES.get(KIND(kind)) 779 if not sub or not issubclass(sub, Declaration): 780 raise TypeError(f'unsupported kind, got {row!r}') 781 else: 782 sub = cls 783 return sub._from_row(fixed) 784 785 @classmethod 786 def _from_row(cls, row): 787 filename, funcname, name, kind, data = row 788 kind = KIND._from_raw(kind) 789 if kind is not cls.kind: 790 raise TypeError(f'expected kind {cls.kind.value!r}, got {row!r}') 791 fileinfo = FileInfo.from_raw(filename) 792 if isinstance(data, str): 793 data, extra = cls._parse_data(data, fmt='row') 794 if extra: 795 return cls(fileinfo, name, data, funcname, _extra=extra) 796 else: 797 return cls(fileinfo, name, data, funcname) 798 799 @classmethod 800 def _resolve_parent(cls, parsed, *, _kind=None): 801 if _kind is None: 802 raise TypeError(f'{cls.kind.value} declarations do not have parents ({parsed})') 803 return super()._resolve_parent(parsed, _kind=_kind) 804 805 @classmethod 806 def _render_data(cls, fmt, data, extra): 807 if not data: 808 # XXX There should be some! Forward? 809 yield '???' 810 else: 811 yield from cls._format_data(fmt, data, extra) 812 813 @classmethod 814 def _render_data_row_item(cls, colname, data, extra): 815 if colname == 'data': 816 return cls._format_data('row', data, extra) 817 else: 818 return None 819 820 @classmethod 821 def _format_data(cls, fmt, data, extra): 822 raise NotImplementedError(fmt) 823 824 @classmethod 825 def _parse_data(cls, datastr, fmt=None): 826 """This is the reverse of _render_data.""" 827 if not datastr or datastr is _tables.UNKNOWN or datastr == '???': 828 return None, None 829 elif datastr is _tables.EMPTY or datastr == '-': 830 # All the kinds have *something* even it is unknown. 831 raise TypeError('all declarations have data of some sort, got none') 832 else: 833 return cls._unformat_data(datastr, fmt) 834 835 @classmethod 836 def _unformat_data(cls, datastr, fmt=None): 837 raise NotImplementedError(fmt) 838 839 840class VarType(namedtuple('VarType', 'typequal typespec abstract')): 841 842 @classmethod 843 def from_str(cls, text): 844 orig = text 845 storage, sep, text = text.strip().partition(' ') 846 if not sep: 847 text = storage 848 storage = None 849 elif storage not in ('auto', 'register', 'static', 'extern'): 850 text = orig 851 storage = None 852 return cls._from_str(text), storage 853 854 @classmethod 855 def _from_str(cls, text): 856 orig = text 857 if text.startswith(('const ', 'volatile ')): 858 typequal, _, text = text.partition(' ') 859 else: 860 typequal = None 861 862 # Extract a series of identifiers/keywords. 863 m = re.match(r"^ *'?([a-zA-Z_]\w*(?:\s+[a-zA-Z_]\w*)*)\s*(.*?)'?\s*$", text) 864 if not m: 865 raise ValueError(f'invalid vartype text {orig!r}') 866 typespec, abstract = m.groups() 867 868 return cls(typequal, typespec, abstract or None) 869 870 def __str__(self): 871 parts = [] 872 if self.qualifier: 873 parts.append(self.qualifier) 874 parts.append(self.spec + (self.abstract or '')) 875 return ' '.join(parts) 876 877 @property 878 def qualifier(self): 879 return self.typequal 880 881 @property 882 def spec(self): 883 return self.typespec 884 885 886class Variable(Declaration): 887 kind = KIND.VARIABLE 888 889 @classmethod 890 def _resolve_parent(cls, parsed): 891 return super()._resolve_parent(parsed, _kind=KIND.FUNCTION) 892 893 @classmethod 894 def _resolve_data(cls, data): 895 if not data: 896 return None, None 897 storage, vartype = _get_vartype(data) 898 return VarType(**vartype), {'storage': storage} 899 900 @classmethod 901 def _raw_data(self, data, extra): 902 vartype = data._asdict() 903 return { 904 'storage': extra['storage'], 905 'vartype': vartype, 906 } 907 908 @classmethod 909 def _format_data(cls, fmt, data, extra): 910 storage = extra.get('storage') 911 text = f'{storage} {data}' if storage else str(data) 912 if fmt in ('line', 'brief'): 913 yield text 914 #elif fmt == 'full': 915 elif fmt == 'row': 916 yield text 917 else: 918 raise NotImplementedError(fmt) 919 920 @classmethod 921 def _unformat_data(cls, datastr, fmt=None): 922 if fmt in ('line', 'brief'): 923 vartype, storage = VarType.from_str(datastr) 924 return vartype, {'storage': storage} 925 #elif fmt == 'full': 926 elif fmt == 'row': 927 vartype, storage = VarType.from_str(datastr) 928 return vartype, {'storage': storage} 929 else: 930 raise NotImplementedError(fmt) 931 932 def __init__(self, file, name, data, parent=None, storage=None): 933 super().__init__(file, name, data, parent, 934 _extra={'storage': storage or None}, 935 _shortkey=f'({parent.name}).{name}' if parent else name, 936 _key=(str(file), 937 # Tilde comes after all other ascii characters. 938 f'~{parent or ""}~', 939 name, 940 ), 941 ) 942 if storage: 943 if storage not in STORAGE: 944 # The parser must need an update. 945 raise NotImplementedError(storage) 946 # Otherwise we trust the compiler to have validated it. 947 948 @property 949 def vartype(self): 950 return self.data 951 952 953class Signature(namedtuple('Signature', 'params returntype inline isforward')): 954 955 @classmethod 956 def from_str(cls, text): 957 orig = text 958 storage, sep, text = text.strip().partition(' ') 959 if not sep: 960 text = storage 961 storage = None 962 elif storage not in ('auto', 'register', 'static', 'extern'): 963 text = orig 964 storage = None 965 return cls._from_str(text), storage 966 967 @classmethod 968 def _from_str(cls, text): 969 orig = text 970 inline, sep, text = text.partition('|') 971 if not sep: 972 text = inline 973 inline = None 974 975 isforward = False 976 if text.endswith(';'): 977 text = text[:-1] 978 isforward = True 979 elif text.endswith('{}'): 980 text = text[:-2] 981 982 index = text.rindex('(') 983 if index < 0: 984 raise ValueError(f'bad signature text {orig!r}') 985 params = text[index:] 986 while params.count('(') <= params.count(')'): 987 index = text.rindex('(', 0, index) 988 if index < 0: 989 raise ValueError(f'bad signature text {orig!r}') 990 params = text[index:] 991 text = text[:index] 992 993 returntype = VarType._from_str(text.rstrip()) 994 995 return cls(params, returntype, inline, isforward) 996 997 def __str__(self): 998 parts = [] 999 if self.inline: 1000 parts.extend([ 1001 self.inline, 1002 '|', 1003 ]) 1004 parts.extend([ 1005 str(self.returntype), 1006 self.params, 1007 ';' if self.isforward else '{}', 1008 ]) 1009 return ' '.join(parts) 1010 1011 @property 1012 def returns(self): 1013 return self.returntype 1014 1015 1016class Function(Declaration): 1017 kind = KIND.FUNCTION 1018 1019 @classmethod 1020 def _resolve_data(cls, data): 1021 if not data: 1022 return None, None 1023 kwargs = dict(data) 1024 returntype = dict(data['returntype']) 1025 del returntype['storage'] 1026 kwargs['returntype'] = VarType(**returntype) 1027 storage = kwargs.pop('storage') 1028 return Signature(**kwargs), {'storage': storage} 1029 1030 @classmethod 1031 def _raw_data(self, data): 1032 # XXX finish! 1033 return data 1034 1035 @classmethod 1036 def _format_data(cls, fmt, data, extra): 1037 storage = extra.get('storage') 1038 text = f'{storage} {data}' if storage else str(data) 1039 if fmt in ('line', 'brief'): 1040 yield text 1041 #elif fmt == 'full': 1042 elif fmt == 'row': 1043 yield text 1044 else: 1045 raise NotImplementedError(fmt) 1046 1047 @classmethod 1048 def _unformat_data(cls, datastr, fmt=None): 1049 if fmt in ('line', 'brief'): 1050 sig, storage = Signature.from_str(sig) 1051 return sig, {'storage': storage} 1052 #elif fmt == 'full': 1053 elif fmt == 'row': 1054 sig, storage = Signature.from_str(sig) 1055 return sig, {'storage': storage} 1056 else: 1057 raise NotImplementedError(fmt) 1058 1059 def __init__(self, file, name, data, parent=None, storage=None): 1060 super().__init__(file, name, data, parent, _extra={'storage': storage}) 1061 self._shortkey = f'~{name}~ {self.data}' 1062 self._key = ( 1063 str(file), 1064 self._shortkey, 1065 ) 1066 1067 @property 1068 def signature(self): 1069 return self.data 1070 1071 1072class TypeDeclaration(Declaration): 1073 1074 def __init__(self, file, name, data, parent=None, *, _shortkey=None): 1075 if not _shortkey: 1076 _shortkey = f'{self.kind.value} {name}' 1077 super().__init__(file, name, data, parent, 1078 _shortkey=_shortkey, 1079 _key=( 1080 str(file), 1081 _shortkey, 1082 ), 1083 ) 1084 1085 1086class POTSType(TypeDeclaration): 1087 1088 def __init__(self, name): 1089 _file = _data = _parent = None 1090 super().__init__(_file, name, _data, _parent, _shortkey=name) 1091 1092 1093class FuncPtr(TypeDeclaration): 1094 1095 def __init__(self, vartype): 1096 _file = _name = _parent = None 1097 data = vartype 1098 self.vartype = vartype 1099 super().__init__(_file, _name, data, _parent, _shortkey=f'<{vartype}>') 1100 1101 1102class TypeDef(TypeDeclaration): 1103 kind = KIND.TYPEDEF 1104 1105 @classmethod 1106 def _resolve_data(cls, data): 1107 if not data: 1108 raise NotImplementedError(data) 1109 vartype = dict(data) 1110 del vartype['storage'] 1111 return VarType(**vartype), None 1112 1113 @classmethod 1114 def _raw_data(self, data): 1115 # XXX finish! 1116 return data 1117 1118 @classmethod 1119 def _format_data(cls, fmt, data, extra): 1120 text = str(data) 1121 if fmt in ('line', 'brief'): 1122 yield text 1123 elif fmt == 'full': 1124 yield text 1125 elif fmt == 'row': 1126 yield text 1127 else: 1128 raise NotImplementedError(fmt) 1129 1130 @classmethod 1131 def _unformat_data(cls, datastr, fmt=None): 1132 if fmt in ('line', 'brief'): 1133 vartype, _ = VarType.from_str(datastr) 1134 return vartype, None 1135 #elif fmt == 'full': 1136 elif fmt == 'row': 1137 vartype, _ = VarType.from_str(datastr) 1138 return vartype, None 1139 else: 1140 raise NotImplementedError(fmt) 1141 1142 def __init__(self, file, name, data, parent=None): 1143 super().__init__(file, name, data, parent, _shortkey=name) 1144 1145 @property 1146 def vartype(self): 1147 return self.data 1148 1149 1150class Member(namedtuple('Member', 'name vartype size')): 1151 1152 @classmethod 1153 def from_data(cls, raw, index): 1154 name = raw.name if raw.name else index 1155 vartype = size = None 1156 if type(raw.data) is int: 1157 size = raw.data 1158 elif isinstance(raw.data, str): 1159 size = int(raw.data) 1160 elif raw.data: 1161 vartype = dict(raw.data) 1162 del vartype['storage'] 1163 if 'size' in vartype: 1164 size = int(vartype.pop('size')) 1165 vartype = VarType(**vartype) 1166 return cls(name, vartype, size) 1167 1168 @classmethod 1169 def from_str(cls, text): 1170 name, _, vartype = text.partition(': ') 1171 if name.startswith('#'): 1172 name = int(name[1:]) 1173 if vartype.isdigit(): 1174 size = int(vartype) 1175 vartype = None 1176 else: 1177 vartype, _ = VarType.from_str(vartype) 1178 size = None 1179 return cls(name, vartype, size) 1180 1181 def __str__(self): 1182 name = self.name if isinstance(self.name, str) else f'#{self.name}' 1183 return f'{name}: {self.vartype or self.size}' 1184 1185 1186class _StructUnion(TypeDeclaration): 1187 1188 @classmethod 1189 def _resolve_data(cls, data): 1190 if not data: 1191 # XXX There should be some! Forward? 1192 return None, None 1193 return [Member.from_data(v, i) for i, v in enumerate(data)], None 1194 1195 @classmethod 1196 def _raw_data(self, data): 1197 # XXX finish! 1198 return data 1199 1200 @classmethod 1201 def _format_data(cls, fmt, data, extra): 1202 if fmt in ('line', 'brief'): 1203 members = ', '.join(f'<{m}>' for m in data) 1204 yield f'[{members}]' 1205 elif fmt == 'full': 1206 for member in data: 1207 yield f'{member}' 1208 elif fmt == 'row': 1209 members = ', '.join(f'<{m}>' for m in data) 1210 yield f'[{members}]' 1211 else: 1212 raise NotImplementedError(fmt) 1213 1214 @classmethod 1215 def _unformat_data(cls, datastr, fmt=None): 1216 if fmt in ('line', 'brief'): 1217 members = [Member.from_str(m[1:-1]) 1218 for m in datastr[1:-1].split(', ')] 1219 return members, None 1220 #elif fmt == 'full': 1221 elif fmt == 'row': 1222 members = [Member.from_str(m.rstrip('>').lstrip('<')) 1223 for m in datastr[1:-1].split('>, <')] 1224 return members, None 1225 else: 1226 raise NotImplementedError(fmt) 1227 1228 def __init__(self, file, name, data, parent=None): 1229 super().__init__(file, name, data, parent) 1230 1231 @property 1232 def members(self): 1233 return self.data 1234 1235 1236class Struct(_StructUnion): 1237 kind = KIND.STRUCT 1238 1239 1240class Union(_StructUnion): 1241 kind = KIND.UNION 1242 1243 1244class Enum(TypeDeclaration): 1245 kind = KIND.ENUM 1246 1247 @classmethod 1248 def _resolve_data(cls, data): 1249 if not data: 1250 # XXX There should be some! Forward? 1251 return None, None 1252 enumerators = [e if isinstance(e, str) else e.name 1253 for e in data] 1254 return enumerators, None 1255 1256 @classmethod 1257 def _raw_data(self, data): 1258 # XXX finish! 1259 return data 1260 1261 @classmethod 1262 def _format_data(cls, fmt, data, extra): 1263 if fmt in ('line', 'brief'): 1264 yield repr(data) 1265 elif fmt == 'full': 1266 for enumerator in data: 1267 yield f'{enumerator}' 1268 elif fmt == 'row': 1269 # XXX This won't work with CSV... 1270 yield ','.join(data) 1271 else: 1272 raise NotImplementedError(fmt) 1273 1274 @classmethod 1275 def _unformat_data(cls, datastr, fmt=None): 1276 if fmt in ('line', 'brief'): 1277 return _strutil.unrepr(datastr), None 1278 #elif fmt == 'full': 1279 elif fmt == 'row': 1280 return datastr.split(','), None 1281 else: 1282 raise NotImplementedError(fmt) 1283 1284 def __init__(self, file, name, data, parent=None): 1285 super().__init__(file, name, data, parent) 1286 1287 @property 1288 def enumerators(self): 1289 return self.data 1290 1291 1292### statements ### 1293 1294class Statement(HighlevelParsedItem): 1295 kind = KIND.STATEMENT 1296 1297 @classmethod 1298 def _resolve_data(cls, data): 1299 # XXX finish! 1300 return data, None 1301 1302 @classmethod 1303 def _raw_data(self, data): 1304 # XXX finish! 1305 return data 1306 1307 @classmethod 1308 def _render_data(cls, fmt, data, extra): 1309 # XXX Handle other formats? 1310 return repr(data) 1311 1312 @classmethod 1313 def _parse_data(self, datastr, fmt=None): 1314 # XXX Handle other formats? 1315 return _strutil.unrepr(datastr), None 1316 1317 def __init__(self, file, name, data, parent=None): 1318 super().__init__(file, name, data, parent, 1319 _shortkey=data or '', 1320 _key=( 1321 str(file), 1322 file.lno, 1323 # XXX Only one stmt per line? 1324 ), 1325 ) 1326 1327 @property 1328 def text(self): 1329 return self.data 1330 1331 1332### 1333 1334KIND_CLASSES = {cls.kind: cls for cls in [ 1335 Variable, 1336 Function, 1337 TypeDef, 1338 Struct, 1339 Union, 1340 Enum, 1341 Statement, 1342]} 1343 1344 1345def resolve_parsed(parsed): 1346 if isinstance(parsed, HighlevelParsedItem): 1347 return parsed 1348 try: 1349 cls = KIND_CLASSES[parsed.kind] 1350 except KeyError: 1351 raise ValueError(f'unsupported kind in {parsed!r}') 1352 return cls.from_parsed(parsed) 1353 1354 1355def set_flag(item, name, value): 1356 try: 1357 setattr(item, name, value) 1358 except AttributeError: 1359 object.__setattr__(item, name, value) 1360 1361 1362############################# 1363# composite 1364 1365class Declarations: 1366 1367 @classmethod 1368 def from_decls(cls, decls): 1369 return cls(decls) 1370 1371 @classmethod 1372 def from_parsed(cls, items): 1373 decls = (resolve_parsed(item) 1374 for item in items 1375 if item.kind is not KIND.STATEMENT) 1376 return cls.from_decls(decls) 1377 1378 @classmethod 1379 def _resolve_key(cls, raw): 1380 if isinstance(raw, str): 1381 raw = [raw] 1382 elif isinstance(raw, Declaration): 1383 raw = ( 1384 raw.filename if cls._is_public(raw) else None, 1385 # `raw.parent` is always None for types and functions. 1386 raw.parent if raw.kind is KIND.VARIABLE else None, 1387 raw.name, 1388 ) 1389 1390 extra = None 1391 if len(raw) == 1: 1392 name, = raw 1393 if name: 1394 name = str(name) 1395 if name.endswith(('.c', '.h')): 1396 # This is only legit as a query. 1397 key = (name, None, None) 1398 else: 1399 key = (None, None, name) 1400 else: 1401 key = (None, None, None) 1402 elif len(raw) == 2: 1403 parent, name = raw 1404 name = str(name) 1405 if isinstance(parent, Declaration): 1406 key = (None, parent.name, name) 1407 elif not parent: 1408 key = (None, None, name) 1409 else: 1410 parent = str(parent) 1411 if parent.endswith(('.c', '.h')): 1412 key = (parent, None, name) 1413 else: 1414 key = (None, parent, name) 1415 else: 1416 key, extra = raw[:3], raw[3:] 1417 filename, funcname, name = key 1418 filename = str(filename) if filename else None 1419 if isinstance(funcname, Declaration): 1420 funcname = funcname.name 1421 else: 1422 funcname = str(funcname) if funcname else None 1423 name = str(name) if name else None 1424 key = (filename, funcname, name) 1425 return key, extra 1426 1427 @classmethod 1428 def _is_public(cls, decl): 1429 # For .c files don't we need info from .h files to make this decision? 1430 # XXX Check for "extern". 1431 # For now we treat all decls a "private" (have filename set). 1432 return False 1433 1434 def __init__(self, decls): 1435 # (file, func, name) -> decl 1436 # "public": 1437 # * (None, None, name) 1438 # "private", "global": 1439 # * (file, None, name) 1440 # "private", "local": 1441 # * (file, func, name) 1442 if hasattr(decls, 'items'): 1443 self._decls = decls 1444 else: 1445 self._decls = {} 1446 self._extend(decls) 1447 1448 # XXX always validate? 1449 1450 def validate(self): 1451 for key, decl in self._decls.items(): 1452 if type(key) is not tuple or len(key) != 3: 1453 raise ValueError(f'expected 3-tuple key, got {key!r} (for decl {decl!r})') 1454 filename, funcname, name = key 1455 if not name: 1456 raise ValueError(f'expected name in key, got {key!r} (for decl {decl!r})') 1457 elif type(name) is not str: 1458 raise ValueError(f'expected name in key to be str, got {key!r} (for decl {decl!r})') 1459 # XXX Check filename type? 1460 # XXX Check funcname type? 1461 1462 if decl.kind is KIND.STATEMENT: 1463 raise ValueError(f'expected a declaration, got {decl!r}') 1464 1465 def __repr__(self): 1466 return f'{type(self).__name__}({list(self)})' 1467 1468 def __len__(self): 1469 return len(self._decls) 1470 1471 def __iter__(self): 1472 yield from self._decls 1473 1474 def __getitem__(self, key): 1475 # XXX Be more exact for the 3-tuple case? 1476 if type(key) not in (str, tuple): 1477 raise KeyError(f'unsupported key {key!r}') 1478 resolved, extra = self._resolve_key(key) 1479 if extra: 1480 raise KeyError(f'key must have at most 3 parts, got {key!r}') 1481 if not resolved[2]: 1482 raise ValueError(f'expected name in key, got {key!r}') 1483 try: 1484 return self._decls[resolved] 1485 except KeyError: 1486 if type(key) is tuple and len(key) == 3: 1487 filename, funcname, name = key 1488 else: 1489 filename, funcname, name = resolved 1490 if filename and not filename.endswith(('.c', '.h')): 1491 raise KeyError(f'invalid filename in key {key!r}') 1492 elif funcname and funcname.endswith(('.c', '.h')): 1493 raise KeyError(f'invalid funcname in key {key!r}') 1494 elif name and name.endswith(('.c', '.h')): 1495 raise KeyError(f'invalid name in key {key!r}') 1496 else: 1497 raise # re-raise 1498 1499 @property 1500 def types(self): 1501 return self._find(kind=KIND.TYPES) 1502 1503 @property 1504 def functions(self): 1505 return self._find(None, None, None, KIND.FUNCTION) 1506 1507 @property 1508 def variables(self): 1509 return self._find(None, None, None, KIND.VARIABLE) 1510 1511 def iter_all(self): 1512 yield from self._decls.values() 1513 1514 def get(self, key, default=None): 1515 try: 1516 return self[key] 1517 except KeyError: 1518 return default 1519 1520 #def add_decl(self, decl, key=None): 1521 # decl = _resolve_parsed(decl) 1522 # self._add_decl(decl, key) 1523 1524 def find(self, *key, **explicit): 1525 if not key: 1526 if not explicit: 1527 return iter(self) 1528 return self._find(**explicit) 1529 1530 resolved, extra = self._resolve_key(key) 1531 filename, funcname, name = resolved 1532 if not extra: 1533 kind = None 1534 elif len(extra) == 1: 1535 kind, = extra 1536 else: 1537 raise KeyError(f'key must have at most 4 parts, got {key!r}') 1538 1539 implicit= {} 1540 if filename: 1541 implicit['filename'] = filename 1542 if funcname: 1543 implicit['funcname'] = funcname 1544 if name: 1545 implicit['name'] = name 1546 if kind: 1547 implicit['kind'] = kind 1548 return self._find(**implicit, **explicit) 1549 1550 def _find(self, filename=None, funcname=None, name=None, kind=None): 1551 for decl in self._decls.values(): 1552 if filename and decl.filename != filename: 1553 continue 1554 if funcname: 1555 if decl.kind is not KIND.VARIABLE: 1556 continue 1557 if decl.parent.name != funcname: 1558 continue 1559 if name and decl.name != name: 1560 continue 1561 if kind: 1562 kinds = KIND.resolve_group(kind) 1563 if decl.kind not in kinds: 1564 continue 1565 yield decl 1566 1567 def _add_decl(self, decl, key=None): 1568 if key: 1569 if type(key) not in (str, tuple): 1570 raise NotImplementedError((key, decl)) 1571 # Any partial key will be turned into a full key, but that 1572 # same partial key will still match a key lookup. 1573 resolved, _ = self._resolve_key(key) 1574 if not resolved[2]: 1575 raise ValueError(f'expected name in key, got {key!r}') 1576 key = resolved 1577 # XXX Also add with the decl-derived key if not the same? 1578 else: 1579 key, _ = self._resolve_key(decl) 1580 self._decls[key] = decl 1581 1582 def _extend(self, decls): 1583 decls = iter(decls) 1584 # Check only the first item. 1585 for decl in decls: 1586 if isinstance(decl, Declaration): 1587 self._add_decl(decl) 1588 # Add the rest without checking. 1589 for decl in decls: 1590 self._add_decl(decl) 1591 elif isinstance(decl, HighlevelParsedItem): 1592 raise NotImplementedError(decl) 1593 else: 1594 try: 1595 key, decl = decl 1596 except ValueError: 1597 raise NotImplementedError(decl) 1598 if not isinstance(decl, Declaration): 1599 raise NotImplementedError(decl) 1600 self._add_decl(decl, key) 1601 # Add the rest without checking. 1602 for key, decl in decls: 1603 self._add_decl(decl, key) 1604 # The iterator will be exhausted at this point. 1605