• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1from collections import namedtuple
2import logging
3import os
4import os.path
5import re
6import textwrap
7
8from c_common.tables import build_table, resolve_columns
9from c_parser.parser._regexes import _ind
10from ._files import iter_header_files, resolve_filename
11from . import REPO_ROOT
12
13
14logger = logging.getLogger(__name__)
15
16
17INCLUDE_ROOT = os.path.join(REPO_ROOT, 'Include')
18INCLUDE_CPYTHON = os.path.join(INCLUDE_ROOT, 'cpython')
19INCLUDE_INTERNAL = os.path.join(INCLUDE_ROOT, 'internal')
20
21_MAYBE_NESTED_PARENS = textwrap.dedent(r'''
22    (?:
23        (?: [^(]* [(] [^()]* [)] )* [^(]*
24    )
25''')
26
27CAPI_FUNC = textwrap.dedent(rf'''
28    (?:
29        ^
30        \s*
31        PyAPI_FUNC \s*
32        [(]
33        {_ind(_MAYBE_NESTED_PARENS, 2)}
34        [)] \s*
35        (\w+)  # <func>
36        \s* [(]
37    )
38''')
39CAPI_DATA = textwrap.dedent(rf'''
40    (?:
41        ^
42        \s*
43        PyAPI_DATA \s*
44        [(]
45        {_ind(_MAYBE_NESTED_PARENS, 2)}
46        [)] \s*
47        (\w+)  # <data>
48        \b [^(]
49    )
50''')
51CAPI_INLINE = textwrap.dedent(r'''
52    (?:
53        ^
54        \s*
55        static \s+ inline \s+
56        .*?
57        \s+
58        ( \w+ )  # <inline>
59        \s* [(]
60    )
61''')
62CAPI_MACRO = textwrap.dedent(r'''
63    (?:
64        (\w+)  # <macro>
65        [(]
66    )
67''')
68CAPI_CONSTANT = textwrap.dedent(r'''
69    (?:
70        (\w+)  # <constant>
71        \s+ [^(]
72    )
73''')
74CAPI_DEFINE = textwrap.dedent(rf'''
75    (?:
76        ^
77        \s* [#] \s* define \s+
78        (?:
79            {_ind(CAPI_MACRO, 3)}
80            |
81            {_ind(CAPI_CONSTANT, 3)}
82            |
83            (?:
84                # ignored
85                \w+   # <defined_name>
86                \s*
87                $
88            )
89        )
90    )
91''')
92CAPI_RE = re.compile(textwrap.dedent(rf'''
93    (?:
94        {_ind(CAPI_FUNC, 2)}
95        |
96        {_ind(CAPI_DATA, 2)}
97        |
98        {_ind(CAPI_INLINE, 2)}
99        |
100        {_ind(CAPI_DEFINE, 2)}
101    )
102'''), re.VERBOSE)
103
104KINDS = [
105    'func',
106    'data',
107    'inline',
108    'macro',
109    'constant',
110]
111
112
113def _parse_line(line, prev=None):
114    last = line
115    if prev:
116        if not prev.endswith(os.linesep):
117            prev += os.linesep
118        line = prev + line
119    m = CAPI_RE.match(line)
120    if not m:
121        if not prev and line.startswith('static inline '):
122            return line  # the new "prev"
123        #if 'PyAPI_' in line or '#define ' in line or ' define ' in line:
124        #    print(line)
125        return None
126    results = zip(KINDS, m.groups())
127    for kind, name in results:
128        if name:
129            clean = last.split('//')[0].rstrip()
130            if clean.endswith('*/'):
131                clean = clean.split('/*')[0].rstrip()
132
133            if kind == 'macro' or kind == 'constant':
134                if not clean.endswith('\\'):
135                    return name, kind
136            elif kind == 'inline':
137                if clean.endswith('}'):
138                    if not prev or clean == '}':
139                        return name, kind
140            elif kind == 'func' or kind == 'data':
141                if clean.endswith(';'):
142                    return name, kind
143            else:
144                # This should not be reached.
145                raise NotImplementedError
146            return line  # the new "prev"
147    # It was a plain #define.
148    return None
149
150
151LEVELS = [
152    'stable',
153    'cpython',
154    'private',
155    'internal',
156]
157
158def _get_level(filename, name, *,
159               _cpython=INCLUDE_CPYTHON + os.path.sep,
160               _internal=INCLUDE_INTERNAL + os.path.sep,
161               ):
162    if filename.startswith(_internal):
163        return 'internal'
164    elif name.startswith('_'):
165        return 'private'
166    elif os.path.dirname(filename) == INCLUDE_ROOT:
167        return 'stable'
168    elif filename.startswith(_cpython):
169        return 'cpython'
170    else:
171        raise NotImplementedError
172    #return '???'
173
174
175GROUPINGS = {
176    'kind': KINDS,
177    'level': LEVELS,
178}
179
180
181class CAPIItem(namedtuple('CAPIItem', 'file lno name kind level')):
182
183    @classmethod
184    def from_line(cls, line, filename, lno, prev=None):
185        parsed = _parse_line(line, prev)
186        if not parsed:
187            return None, None
188        if isinstance(parsed, str):
189            # incomplete
190            return None, parsed
191        name, kind = parsed
192        level = _get_level(filename, name)
193        self = cls(filename, lno, name, kind, level)
194        if prev:
195            self._text = (prev + line).rstrip().splitlines()
196        else:
197            self._text = [line.rstrip()]
198        return self, None
199
200    @property
201    def relfile(self):
202        return self.file[len(REPO_ROOT) + 1:]
203
204    @property
205    def text(self):
206        try:
207            return self._text
208        except AttributeError:
209            # XXX Actually ready the text from disk?.
210            self._text = []
211            if self.kind == 'data':
212                self._text = [
213                    f'PyAPI_DATA(...) {self.name}',
214                ]
215            elif self.kind == 'func':
216                self._text = [
217                    f'PyAPI_FUNC(...) {self.name}(...);',
218                ]
219            elif self.kind == 'inline':
220                self._text = [
221                    f'static inline {self.name}(...);',
222                ]
223            elif self.kind == 'macro':
224                self._text = [
225                    f'#define {self.name}(...) \\',
226                    f'    ...',
227                ]
228            elif self.kind == 'constant':
229                self._text = [
230                    f'#define {self.name} ...',
231                ]
232            else:
233                raise NotImplementedError
234
235            return self._text
236
237
238def _parse_groupby(raw):
239    if not raw:
240        raw = 'kind'
241
242    if isinstance(raw, str):
243        groupby = raw.replace(',', ' ').strip().split()
244    else:
245        raise NotImplementedError
246
247    if not all(v in GROUPINGS for v in groupby):
248        raise ValueError(f'invalid groupby value {raw!r}')
249    return groupby
250
251
252def _resolve_full_groupby(groupby):
253    if isinstance(groupby, str):
254        groupby = [groupby]
255    groupings = []
256    for grouping in groupby + list(GROUPINGS):
257        if grouping not in groupings:
258            groupings.append(grouping)
259    return groupings
260
261
262def summarize(items, *, groupby='kind', includeempty=True, minimize=None):
263    if minimize is None:
264        if includeempty is None:
265            minimize = True
266            includeempty = False
267        else:
268            minimize = includeempty
269    elif includeempty is None:
270        includeempty = minimize
271    elif minimize and includeempty:
272        raise ValueError(f'cannot minimize and includeempty at the same time')
273
274    groupby = _parse_groupby(groupby)[0]
275    _outer, _inner = _resolve_full_groupby(groupby)
276    outers = GROUPINGS[_outer]
277    inners = GROUPINGS[_inner]
278
279    summary = {
280        'totals': {
281            'all': 0,
282            'subs': {o: 0 for o in outers},
283            'bygroup': {o: {i: 0 for i in inners}
284                        for o in outers},
285        },
286    }
287
288    for item in items:
289        outer = getattr(item, _outer)
290        inner = getattr(item, _inner)
291        # Update totals.
292        summary['totals']['all'] += 1
293        summary['totals']['subs'][outer] += 1
294        summary['totals']['bygroup'][outer][inner] += 1
295
296    if not includeempty:
297        subtotals = summary['totals']['subs']
298        bygroup = summary['totals']['bygroup']
299        for outer in outers:
300            if subtotals[outer] == 0:
301                del subtotals[outer]
302                del bygroup[outer]
303                continue
304
305            for inner in inners:
306                if bygroup[outer][inner] == 0:
307                    del bygroup[outer][inner]
308            if minimize:
309                if len(bygroup[outer]) == 1:
310                    del bygroup[outer]
311
312    return summary
313
314
315def _parse_capi(lines, filename):
316    if isinstance(lines, str):
317        lines = lines.splitlines()
318    prev = None
319    for lno, line in enumerate(lines, 1):
320        parsed, prev = CAPIItem.from_line(line, filename, lno, prev)
321        if parsed:
322            yield parsed
323    if prev:
324        parsed, prev = CAPIItem.from_line('', filename, lno, prev)
325        if parsed:
326            yield parsed
327        if prev:
328            print('incomplete match:')
329            print(filename)
330            print(prev)
331            raise Exception
332
333
334def iter_capi(filenames=None):
335    for filename in iter_header_files(filenames):
336        with open(filename) as infile:
337            for item in _parse_capi(infile, filename):
338                yield item
339
340
341def resolve_filter(ignored):
342    if not ignored:
343        return None
344    ignored = set(_resolve_ignored(ignored))
345    def filter(item, *, log=None):
346        if item.name not in ignored:
347            return True
348        if log is not None:
349            log(f'ignored {item.name!r}')
350        return False
351    return filter
352
353
354def _resolve_ignored(ignored):
355    if isinstance(ignored, str):
356        ignored = [ignored]
357    for raw in ignored:
358        if isinstance(raw, str):
359            if raw.startswith('|'):
360                yield raw[1:]
361            elif raw.startswith('<') and raw.endswith('>'):
362                filename = raw[1:-1]
363                try:
364                    infile = open(filename)
365                except Exception as exc:
366                    logger.error(f'ignore file failed: {exc}')
367                    continue
368                logger.log(1, f'reading ignored names from {filename!r}')
369                with infile:
370                    for line in infile:
371                        if not line:
372                            continue
373                        if line[0].isspace():
374                            continue
375                        line = line.partition('#')[0].rstrip()
376                        if line:
377                            # XXX Recurse?
378                            yield line
379            else:
380                raw = raw.strip()
381                if raw:
382                    yield raw
383        else:
384            raise NotImplementedError
385
386
387def _collate(items, groupby, includeempty):
388    groupby = _parse_groupby(groupby)[0]
389    maxfilename = maxname = maxkind = maxlevel = 0
390
391    collated = {}
392    groups = GROUPINGS[groupby]
393    for group in groups:
394        collated[group] = []
395
396    for item in items:
397        key = getattr(item, groupby)
398        collated[key].append(item)
399        maxfilename = max(len(item.relfile), maxfilename)
400        maxname = max(len(item.name), maxname)
401        maxkind = max(len(item.kind), maxkind)
402        maxlevel = max(len(item.level), maxlevel)
403    if not includeempty:
404        for group in groups:
405            if not collated[group]:
406                del collated[group]
407    maxextra = {
408        'kind': maxkind,
409        'level': maxlevel,
410    }
411    return collated, groupby, maxfilename, maxname, maxextra
412
413
414def _get_sortkey(sort, _groupby, _columns):
415    if sort is True or sort is None:
416        # For now:
417        def sortkey(item):
418            return (
419                item.level == 'private',
420                LEVELS.index(item.level),
421                KINDS.index(item.kind),
422                os.path.dirname(item.file),
423                os.path.basename(item.file),
424                item.name,
425            )
426        return sortkey
427
428        sortfields = 'not-private level kind dirname basename name'.split()
429    elif isinstance(sort, str):
430        sortfields = sort.replace(',', ' ').strip().split()
431    elif callable(sort):
432        return sort
433    else:
434        raise NotImplementedError
435
436    # XXX Build a sortkey func from sortfields.
437    raise NotImplementedError
438
439
440##################################
441# CLI rendering
442
443_MARKERS = {
444    'level': {
445        'S': 'stable',
446        'C': 'cpython',
447        'P': 'private',
448        'I': 'internal',
449    },
450    'kind': {
451        'F': 'func',
452        'D': 'data',
453        'I': 'inline',
454        'M': 'macro',
455        'C': 'constant',
456    },
457}
458
459
460def resolve_format(format):
461    if not format:
462        return 'table'
463    elif isinstance(format, str) and format in _FORMATS:
464        return format
465    else:
466        return resolve_columns(format)
467
468
469def get_renderer(format):
470    format = resolve_format(format)
471    if isinstance(format, str):
472        try:
473            return _FORMATS[format]
474        except KeyError:
475            raise ValueError(f'unsupported format {format!r}')
476    else:
477        def render(items, **kwargs):
478            return render_table(items, columns=format, **kwargs)
479        return render
480
481
482def render_table(items, *,
483                 columns=None,
484                 groupby='kind',
485                 sort=True,
486                 showempty=False,
487                 verbose=False,
488                 ):
489    if groupby is None:
490        groupby = 'kind'
491    if showempty is None:
492        showempty = False
493
494    if groupby:
495        (collated, groupby, maxfilename, maxname, maxextra,
496         ) = _collate(items, groupby, showempty)
497        for grouping in GROUPINGS:
498            maxextra[grouping] = max(len(g) for g in GROUPINGS[grouping])
499
500        _, extra = _resolve_full_groupby(groupby)
501        extras = [extra]
502        markers = {extra: _MARKERS[extra]}
503
504        groups = GROUPINGS[groupby]
505    else:
506        # XXX Support no grouping?
507        raise NotImplementedError
508
509    if columns:
510        def get_extra(item):
511            return {extra: getattr(item, extra)
512                    for extra in ('kind', 'level')}
513    else:
514        if verbose:
515            extracols = [f'{extra}:{maxextra[extra]}'
516                         for extra in extras]
517            def get_extra(item):
518                return {extra: getattr(item, extra)
519                        for extra in extras}
520        elif len(extras) == 1:
521            extra, = extras
522            extracols = [f'{m}:1' for m in markers[extra]]
523            def get_extra(item):
524                return {m: m if getattr(item, extra) == markers[extra][m] else ''
525                        for m in markers[extra]}
526        else:
527            raise NotImplementedError
528            #extracols = [[f'{m}:1' for m in markers[extra]]
529            #             for extra in extras]
530            #def get_extra(item):
531            #    values = {}
532            #    for extra in extras:
533            #        cur = markers[extra]
534            #        for m in cur:
535            #            values[m] = m if getattr(item, m) == cur[m] else ''
536            #    return values
537        columns = [
538            f'filename:{maxfilename}',
539            f'name:{maxname}',
540            *extracols,
541        ]
542    header, div, fmt = build_table(columns)
543
544    if sort:
545        sortkey = _get_sortkey(sort, groupby, columns)
546
547    total = 0
548    for group, grouped in collated.items():
549        if not showempty and group not in collated:
550            continue
551        yield ''
552        yield f' === {group} ==='
553        yield ''
554        yield header
555        yield div
556        if grouped:
557            if sort:
558                grouped = sorted(grouped, key=sortkey)
559            for item in grouped:
560                yield fmt.format(
561                    filename=item.relfile,
562                    name=item.name,
563                    **get_extra(item),
564                )
565        yield div
566        subtotal = len(grouped)
567        yield f'  sub-total: {subtotal}'
568        total += subtotal
569    yield ''
570    yield f'total: {total}'
571
572
573def render_full(items, *,
574                groupby='kind',
575                sort=None,
576                showempty=None,
577                verbose=False,
578                ):
579    if groupby is None:
580        groupby = 'kind'
581    if showempty is None:
582        showempty = False
583
584    if sort:
585        sortkey = _get_sortkey(sort, groupby, None)
586
587    if groupby:
588        collated, groupby, _, _, _ = _collate(items, groupby, showempty)
589        for group, grouped in collated.items():
590            yield '#' * 25
591            yield f'# {group} ({len(grouped)})'
592            yield '#' * 25
593            yield ''
594            if not grouped:
595                continue
596            if sort:
597                grouped = sorted(grouped, key=sortkey)
598            for item in grouped:
599                yield from _render_item_full(item, groupby, verbose)
600                yield ''
601    else:
602        if sort:
603            items = sorted(items, key=sortkey)
604        for item in items:
605            yield from _render_item_full(item, None, verbose)
606            yield ''
607
608
609def _render_item_full(item, groupby, verbose):
610    yield item.name
611    yield f'  {"filename:":10} {item.relfile}'
612    for extra in ('kind', 'level'):
613        #if groupby != extra:
614            yield f'  {extra+":":10} {getattr(item, extra)}'
615    if verbose:
616        print('  ---------------------------------------')
617        for lno, line in enumerate(item.text, item.lno):
618            print(f'  | {lno:3} {line}')
619        print('  ---------------------------------------')
620
621
622def render_summary(items, *,
623                   groupby='kind',
624                   sort=None,
625                   showempty=None,
626                   verbose=False,
627                   ):
628    if groupby is None:
629        groupby = 'kind'
630    summary = summarize(
631        items,
632        groupby=groupby,
633        includeempty=showempty,
634        minimize=None if showempty else not verbose,
635    )
636
637    subtotals = summary['totals']['subs']
638    bygroup = summary['totals']['bygroup']
639    lastempty = False
640    for outer, subtotal in subtotals.items():
641        if bygroup:
642            subtotal = f'({subtotal})'
643            yield f'{outer + ":":20} {subtotal:>8}'
644        else:
645            yield f'{outer + ":":10} {subtotal:>8}'
646        if outer in bygroup:
647            for inner, count in bygroup[outer].items():
648                yield f'   {inner + ":":9} {count}'
649            lastempty = False
650        else:
651            lastempty = True
652
653    total = f'*{summary["totals"]["all"]}*'
654    label = '*total*:'
655    if bygroup:
656        yield f'{label:20} {total:>8}'
657    else:
658        yield f'{label:10} {total:>9}'
659
660
661_FORMATS = {
662    'table': render_table,
663    'full': render_full,
664    'summary': render_summary,
665}
666