• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import io
2import logging
3import os
4import os.path
5import re
6import sys
7
8from c_common import fsutil
9from c_common.logging import VERBOSITY, Printer
10from c_common.scriptutil import (
11    add_verbosity_cli,
12    add_traceback_cli,
13    add_sepval_cli,
14    add_progress_cli,
15    add_files_cli,
16    add_commands_cli,
17    process_args_by_key,
18    configure_logger,
19    get_prog,
20    filter_filenames,
21    iter_marks,
22)
23from c_parser.info import KIND
24from c_parser.match import is_type_decl
25from .match import filter_forward
26from . import (
27    analyze as _analyze,
28    datafiles as _datafiles,
29    check_all as _check_all,
30)
31
32
33KINDS = [
34    KIND.TYPEDEF,
35    KIND.STRUCT,
36    KIND.UNION,
37    KIND.ENUM,
38    KIND.FUNCTION,
39    KIND.VARIABLE,
40    KIND.STATEMENT,
41]
42
43logger = logging.getLogger(__name__)
44
45
46#######################################
47# table helpers
48
49TABLE_SECTIONS = {
50    'types': (
51        ['kind', 'name', 'data', 'file'],
52        KIND.is_type_decl,
53        (lambda v: (v.kind.value, v.filename or '', v.name)),
54    ),
55    'typedefs': 'types',
56    'structs': 'types',
57    'unions': 'types',
58    'enums': 'types',
59    'functions': (
60        ['name', 'data', 'file'],
61        (lambda kind: kind is KIND.FUNCTION),
62        (lambda v: (v.filename or '', v.name)),
63    ),
64    'variables': (
65        ['name', 'parent', 'data', 'file'],
66        (lambda kind: kind is KIND.VARIABLE),
67        (lambda v: (v.filename or '', str(v.parent) if v.parent else '', v.name)),
68    ),
69    'statements': (
70        ['file', 'parent', 'data'],
71        (lambda kind: kind is KIND.STATEMENT),
72        (lambda v: (v.filename or '', str(v.parent) if v.parent else '', v.name)),
73    ),
74    KIND.TYPEDEF: 'typedefs',
75    KIND.STRUCT: 'structs',
76    KIND.UNION: 'unions',
77    KIND.ENUM: 'enums',
78    KIND.FUNCTION: 'functions',
79    KIND.VARIABLE: 'variables',
80    KIND.STATEMENT: 'statements',
81}
82
83
84def _render_table(items, columns, relroot=None):
85    # XXX improve this
86    header = '\t'.join(columns)
87    div = '--------------------'
88    yield header
89    yield div
90    total = 0
91    for item in items:
92        rowdata = item.render_rowdata(columns)
93        row = [rowdata[c] for c in columns]
94        if relroot and 'file' in columns:
95            index = columns.index('file')
96            row[index] = os.path.relpath(row[index], relroot)
97        yield '\t'.join(row)
98        total += 1
99    yield div
100    yield f'total: {total}'
101
102
103def build_section(name, groupitems, *, relroot=None):
104    info = TABLE_SECTIONS[name]
105    while type(info) is not tuple:
106        if name in KINDS:
107            name = info
108        info = TABLE_SECTIONS[info]
109
110    columns, match_kind, sortkey = info
111    items = (v for v in groupitems if match_kind(v.kind))
112    items = sorted(items, key=sortkey)
113    def render():
114        yield ''
115        yield f'{name}:'
116        yield ''
117        for line in _render_table(items, columns, relroot):
118            yield line
119    return items, render
120
121
122#######################################
123# the checks
124
125CHECKS = {
126    #'globals': _check_globals,
127}
128
129
130def add_checks_cli(parser, checks=None, *, add_flags=None):
131    default = False
132    if not checks:
133        checks = list(CHECKS)
134        default = True
135    elif isinstance(checks, str):
136        checks = [checks]
137    if (add_flags is None and len(checks) > 1) or default:
138        add_flags = True
139
140    process_checks = add_sepval_cli(parser, '--check', 'checks', checks)
141    if add_flags:
142        for check in checks:
143            parser.add_argument(f'--{check}', dest='checks',
144                                action='append_const', const=check)
145    return [
146        process_checks,
147    ]
148
149
150def _get_check_handlers(fmt, printer, verbosity=VERBOSITY):
151    div = None
152    def handle_after():
153        pass
154    if not fmt:
155        div = ''
156        def handle_failure(failure, data):
157            data = repr(data)
158            if verbosity >= 3:
159                logger.info(f'failure: {failure}')
160                logger.info(f'data:    {data}')
161            else:
162                logger.warn(f'failure: {failure} (data: {data})')
163    elif fmt == 'raw':
164        def handle_failure(failure, data):
165            print(f'{failure!r} {data!r}')
166    elif fmt == 'brief':
167        def handle_failure(failure, data):
168            parent = data.parent or ''
169            funcname = parent if isinstance(parent, str) else parent.name
170            name = f'({funcname}).{data.name}' if funcname else data.name
171            failure = failure.split('\t')[0]
172            print(f'{data.filename}:{name} - {failure}')
173    elif fmt == 'summary':
174        def handle_failure(failure, data):
175            print(_fmt_one_summary(data, failure))
176    elif fmt == 'full':
177        div = ''
178        def handle_failure(failure, data):
179            name = data.shortkey if data.kind is KIND.VARIABLE else data.name
180            parent = data.parent or ''
181            funcname = parent if isinstance(parent, str) else parent.name
182            known = 'yes' if data.is_known else '*** NO ***'
183            print(f'{data.kind.value} {name!r} failed ({failure})')
184            print(f'  file:         {data.filename}')
185            print(f'  func:         {funcname or "-"}')
186            print(f'  name:         {data.name}')
187            print(f'  data:         ...')
188            print(f'  type unknown: {known}')
189    else:
190        if fmt in FORMATS:
191            raise NotImplementedError(fmt)
192        raise ValueError(f'unsupported fmt {fmt!r}')
193    return handle_failure, handle_after, div
194
195
196#######################################
197# the formats
198
199def fmt_raw(analysis):
200    for item in analysis:
201        yield from item.render('raw')
202
203
204def fmt_brief(analysis):
205    # XXX Support sorting.
206    items = sorted(analysis)
207    for kind in KINDS:
208        if kind is KIND.STATEMENT:
209            continue
210        for item in items:
211            if item.kind is not kind:
212                continue
213            yield from item.render('brief')
214    yield f'  total: {len(items)}'
215
216
217def fmt_summary(analysis):
218    # XXX Support sorting and grouping.
219    items = list(analysis)
220    total = len(items)
221
222    def section(name):
223        _, render = build_section(name, items)
224        yield from render()
225
226    yield from section('types')
227    yield from section('functions')
228    yield from section('variables')
229    yield from section('statements')
230
231    yield ''
232#    yield f'grand total: {len(supported) + len(unsupported)}'
233    yield f'grand total: {total}'
234
235
236def _fmt_one_summary(item, extra=None):
237    parent = item.parent or ''
238    funcname = parent if isinstance(parent, str) else parent.name
239    if extra:
240        return f'{item.filename:35}\t{funcname or "-":35}\t{item.name:40}\t{extra}'
241    else:
242        return f'{item.filename:35}\t{funcname or "-":35}\t{item.name}'
243
244
245def fmt_full(analysis):
246    # XXX Support sorting.
247    items = sorted(analysis, key=lambda v: v.key)
248    yield ''
249    for item in items:
250        yield from item.render('full')
251        yield ''
252    yield f'total: {len(items)}'
253
254
255FORMATS = {
256    'raw': fmt_raw,
257    'brief': fmt_brief,
258    'summary': fmt_summary,
259    'full': fmt_full,
260}
261
262
263def add_output_cli(parser, *, default='summary'):
264    parser.add_argument('--format', dest='fmt', default=default, choices=tuple(FORMATS))
265
266    def process_args(args, *, argv=None):
267        pass
268    return process_args
269
270
271#######################################
272# the commands
273
274def _cli_check(parser, checks=None, **kwargs):
275    if isinstance(checks, str):
276        checks = [checks]
277    if checks is False:
278        process_checks = None
279    elif checks is None:
280        process_checks = add_checks_cli(parser)
281    elif len(checks) == 1 and type(checks) is not dict and re.match(r'^<.*>$', checks[0]):
282        check = checks[0][1:-1]
283        def process_checks(args, *, argv=None):
284            args.checks = [check]
285    else:
286        process_checks = add_checks_cli(parser, checks=checks)
287    process_progress = add_progress_cli(parser)
288    process_output = add_output_cli(parser, default=None)
289    process_files = add_files_cli(parser, **kwargs)
290    return [
291        process_checks,
292        process_progress,
293        process_output,
294        process_files,
295    ]
296
297
298def cmd_check(filenames, *,
299              checks=None,
300              ignored=None,
301              fmt=None,
302              failfast=False,
303              iter_filenames=None,
304              relroot=fsutil.USE_CWD,
305              track_progress=None,
306              verbosity=VERBOSITY,
307              _analyze=_analyze,
308              _CHECKS=CHECKS,
309              **kwargs
310              ):
311    if not checks:
312        checks = _CHECKS
313    elif isinstance(checks, str):
314        checks = [checks]
315    checks = [_CHECKS[c] if isinstance(c, str) else c
316              for c in checks]
317    printer = Printer(verbosity)
318    (handle_failure, handle_after, div
319     ) = _get_check_handlers(fmt, printer, verbosity)
320
321    filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
322    filenames = filter_filenames(filenames, iter_filenames, relroot)
323    if track_progress:
324        filenames = track_progress(filenames)
325
326    logger.info('analyzing files...')
327    analyzed = _analyze(filenames, **kwargs)
328    analyzed.fix_filenames(relroot, normalize=False)
329    decls = filter_forward(analyzed, markpublic=True)
330
331    logger.info('checking analysis results...')
332    failed = []
333    for data, failure in _check_all(decls, checks, failfast=failfast):
334        if data is None:
335            printer.info('stopping after one failure')
336            break
337        if div is not None and len(failed) > 0:
338            printer.info(div)
339        failed.append(data)
340        handle_failure(failure, data)
341    handle_after()
342
343    printer.info('-------------------------')
344    logger.info(f'total failures: {len(failed)}')
345    logger.info('done checking')
346
347    if fmt == 'summary':
348        print('Categorized by storage:')
349        print()
350        from .match import group_by_storage
351        grouped = group_by_storage(failed, ignore_non_match=False)
352        for group, decls in grouped.items():
353            print()
354            print(group)
355            for decl in decls:
356                print(' ', _fmt_one_summary(decl))
357            print(f'subtotal: {len(decls)}')
358
359    if len(failed) > 0:
360        sys.exit(len(failed))
361
362
363def _cli_analyze(parser, **kwargs):
364    process_progress = add_progress_cli(parser)
365    process_output = add_output_cli(parser)
366    process_files = add_files_cli(parser, **kwargs)
367    return [
368        process_progress,
369        process_output,
370        process_files,
371    ]
372
373
374# XXX Support filtering by kind.
375def cmd_analyze(filenames, *,
376                fmt=None,
377                iter_filenames=None,
378                relroot=fsutil.USE_CWD,
379                track_progress=None,
380                verbosity=None,
381                _analyze=_analyze,
382                formats=FORMATS,
383                **kwargs
384                ):
385    verbosity = verbosity if verbosity is not None else 3
386
387    try:
388        do_fmt = formats[fmt]
389    except KeyError:
390        raise ValueError(f'unsupported fmt {fmt!r}')
391
392    filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
393    filenames = filter_filenames(filenames, iter_filenames, relroot)
394    if track_progress:
395        filenames = track_progress(filenames)
396
397    logger.info('analyzing files...')
398    analyzed = _analyze(filenames, **kwargs)
399    analyzed.fix_filenames(relroot, normalize=False)
400    decls = filter_forward(analyzed, markpublic=True)
401
402    for line in do_fmt(decls):
403        print(line)
404
405
406def _cli_data(parser, filenames=None, known=None):
407    ArgumentParser = type(parser)
408    common = ArgumentParser(add_help=False)
409    # These flags will get processed by the top-level parse_args().
410    add_verbosity_cli(common)
411    add_traceback_cli(common)
412
413    subs = parser.add_subparsers(dest='datacmd')
414
415    sub = subs.add_parser('show', parents=[common])
416    if known is None:
417        sub.add_argument('--known', required=True)
418    if filenames is None:
419        sub.add_argument('filenames', metavar='FILE', nargs='+')
420
421    sub = subs.add_parser('dump', parents=[common])
422    if known is None:
423        sub.add_argument('--known')
424    sub.add_argument('--show', action='store_true')
425    process_progress = add_progress_cli(sub)
426
427    sub = subs.add_parser('check', parents=[common])
428    if known is None:
429        sub.add_argument('--known', required=True)
430
431    def process_args(args, *, argv):
432        if args.datacmd == 'dump':
433            process_progress(args, argv)
434    return process_args
435
436
437def cmd_data(datacmd, filenames, known=None, *,
438             _analyze=_analyze,
439             formats=FORMATS,
440             extracolumns=None,
441             relroot=fsutil.USE_CWD,
442             track_progress=None,
443             **kwargs
444             ):
445    kwargs.pop('verbosity', None)
446    usestdout = kwargs.pop('show', None)
447    if datacmd == 'show':
448        do_fmt = formats['summary']
449        if isinstance(known, str):
450            known, _ = _datafiles.get_known(known, extracolumns, relroot)
451        for line in do_fmt(known):
452            print(line)
453    elif datacmd == 'dump':
454        filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
455        if track_progress:
456            filenames = track_progress(filenames)
457        analyzed = _analyze(filenames, **kwargs)
458        analyzed.fix_filenames(relroot, normalize=False)
459        if known is None or usestdout:
460            outfile = io.StringIO()
461            _datafiles.write_known(analyzed, outfile, extracolumns,
462                                   relroot=relroot)
463            print(outfile.getvalue())
464        else:
465            _datafiles.write_known(analyzed, known, extracolumns,
466                                   relroot=relroot)
467    elif datacmd == 'check':
468        raise NotImplementedError(datacmd)
469    else:
470        raise ValueError(f'unsupported data command {datacmd!r}')
471
472
473COMMANDS = {
474    'check': (
475        'analyze and fail if the given C source/header files have any problems',
476        [_cli_check],
477        cmd_check,
478    ),
479    'analyze': (
480        'report on the state of the given C source/header files',
481        [_cli_analyze],
482        cmd_analyze,
483    ),
484    'data': (
485        'check/manage local data (e.g. known types, ignored vars, caches)',
486        [_cli_data],
487        cmd_data,
488    ),
489}
490
491
492#######################################
493# the script
494
495def parse_args(argv=sys.argv[1:], prog=sys.argv[0], *, subset=None):
496    import argparse
497    parser = argparse.ArgumentParser(
498        prog=prog or get_prog(),
499    )
500
501    processors = add_commands_cli(
502        parser,
503        commands={k: v[1] for k, v in COMMANDS.items()},
504        commonspecs=[
505            add_verbosity_cli,
506            add_traceback_cli,
507        ],
508        subset=subset,
509    )
510
511    args = parser.parse_args(argv)
512    ns = vars(args)
513
514    cmd = ns.pop('cmd')
515
516    verbosity, traceback_cm = process_args_by_key(
517        args,
518        argv,
519        processors[cmd],
520        ['verbosity', 'traceback_cm'],
521    )
522    # "verbosity" is sent to the commands, so we put it back.
523    args.verbosity = verbosity
524
525    return cmd, ns, verbosity, traceback_cm
526
527
528def main(cmd, cmd_kwargs):
529    try:
530        run_cmd = COMMANDS[cmd][0]
531    except KeyError:
532        raise ValueError(f'unsupported cmd {cmd!r}')
533    run_cmd(**cmd_kwargs)
534
535
536if __name__ == '__main__':
537    cmd, cmd_kwargs, verbosity, traceback_cm = parse_args()
538    configure_logger(verbosity)
539    with traceback_cm:
540        main(cmd, cmd_kwargs)
541