• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import logging
2import sys
3
4from c_common.fsutil import expand_filenames, iter_files_by_suffix
5from c_common.scriptutil import (
6    VERBOSITY,
7    add_verbosity_cli,
8    add_traceback_cli,
9    add_commands_cli,
10    add_kind_filtering_cli,
11    add_files_cli,
12    add_progress_cli,
13    main_for_filenames,
14    process_args_by_key,
15    configure_logger,
16    get_prog,
17)
18from c_parser.info import KIND
19import c_parser.__main__ as c_parser
20import c_analyzer.__main__ as c_analyzer
21import c_analyzer as _c_analyzer
22from c_analyzer.info import UNKNOWN
23from . import _analyzer, _capi, _files, _parser, REPO_ROOT
24
25
26logger = logging.getLogger(__name__)
27
28
29def _resolve_filenames(filenames):
30    if filenames:
31        resolved = (_files.resolve_filename(f) for f in filenames)
32    else:
33        resolved = _files.iter_filenames()
34    return resolved
35
36
37#######################################
38# the formats
39
40def fmt_summary(analysis):
41    # XXX Support sorting and grouping.
42    supported = []
43    unsupported = []
44    for item in analysis:
45        if item.supported:
46            supported.append(item)
47        else:
48            unsupported.append(item)
49    total = 0
50
51    def section(name, groupitems):
52        nonlocal total
53        items, render = c_analyzer.build_section(name, groupitems,
54                                                 relroot=REPO_ROOT)
55        yield from render()
56        total += len(items)
57
58    yield ''
59    yield '===================='
60    yield 'supported'
61    yield '===================='
62
63    yield from section('types', supported)
64    yield from section('variables', supported)
65
66    yield ''
67    yield '===================='
68    yield 'unsupported'
69    yield '===================='
70
71    yield from section('types', unsupported)
72    yield from section('variables', unsupported)
73
74    yield ''
75    yield f'grand total: {total}'
76
77
78#######################################
79# the checks
80
81CHECKS = dict(c_analyzer.CHECKS, **{
82    'globals': _analyzer.check_globals,
83})
84
85#######################################
86# the commands
87
88FILES_KWARGS = dict(excluded=_parser.EXCLUDED, nargs='*')
89
90
91def _cli_parse(parser):
92    process_output = c_parser.add_output_cli(parser)
93    process_kind = add_kind_filtering_cli(parser)
94    process_preprocessor = c_parser.add_preprocessor_cli(
95        parser,
96        get_preprocessor=_parser.get_preprocessor,
97    )
98    process_files = add_files_cli(parser, **FILES_KWARGS)
99    return [
100        process_output,
101        process_kind,
102        process_preprocessor,
103        process_files,
104    ]
105
106
107def cmd_parse(filenames=None, **kwargs):
108    filenames = _resolve_filenames(filenames)
109    if 'get_file_preprocessor' not in kwargs:
110        kwargs['get_file_preprocessor'] = _parser.get_preprocessor()
111    c_parser.cmd_parse(
112        filenames,
113        relroot=REPO_ROOT,
114        **kwargs
115    )
116
117
118def _cli_check(parser, **kwargs):
119    return c_analyzer._cli_check(parser, CHECKS, **kwargs, **FILES_KWARGS)
120
121
122def cmd_check(filenames=None, **kwargs):
123    filenames = _resolve_filenames(filenames)
124    kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
125    c_analyzer.cmd_check(
126        filenames,
127        relroot=REPO_ROOT,
128        _analyze=_analyzer.analyze,
129        _CHECKS=CHECKS,
130        **kwargs
131    )
132
133
134def cmd_analyze(filenames=None, **kwargs):
135    formats = dict(c_analyzer.FORMATS)
136    formats['summary'] = fmt_summary
137    filenames = _resolve_filenames(filenames)
138    kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
139    c_analyzer.cmd_analyze(
140        filenames,
141        relroot=REPO_ROOT,
142        _analyze=_analyzer.analyze,
143        formats=formats,
144        **kwargs
145    )
146
147
148def _cli_data(parser):
149    filenames = False
150    known = True
151    return c_analyzer._cli_data(parser, filenames, known)
152
153
154def cmd_data(datacmd, **kwargs):
155    formats = dict(c_analyzer.FORMATS)
156    formats['summary'] = fmt_summary
157    filenames = (file
158                 for file in _resolve_filenames(None)
159                 if file not in _parser.EXCLUDED)
160    kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
161    if datacmd == 'show':
162        types = _analyzer.read_known()
163        results = []
164        for decl, info in types.items():
165            if info is UNKNOWN:
166                if decl.kind in (KIND.STRUCT, KIND.UNION):
167                    extra = {'unsupported': ['type unknown'] * len(decl.members)}
168                else:
169                    extra = {'unsupported': ['type unknown']}
170                info = (info, extra)
171            results.append((decl, info))
172            if decl.shortkey == 'struct _object':
173                tempinfo = info
174        known = _analyzer.Analysis.from_results(results)
175        analyze = None
176    elif datacmd == 'dump':
177        known = _analyzer.KNOWN_FILE
178        def analyze(files, **kwargs):
179            decls = []
180            for decl in _analyzer.iter_decls(files, **kwargs):
181                if not KIND.is_type_decl(decl.kind):
182                    continue
183                if not decl.filename.endswith('.h'):
184                    if decl.shortkey not in _analyzer.KNOWN_IN_DOT_C:
185                        continue
186                decls.append(decl)
187            results = _c_analyzer.analyze_decls(
188                decls,
189                known={},
190                analyze_resolved=_analyzer.analyze_resolved,
191            )
192            return _analyzer.Analysis.from_results(results)
193    else:  # check
194        known = _analyzer.read_known()
195        def analyze(files, **kwargs):
196            return _analyzer.iter_decls(files, **kwargs)
197    extracolumns = None
198    c_analyzer.cmd_data(
199        datacmd,
200        filenames,
201        known,
202        _analyze=analyze,
203        formats=formats,
204        extracolumns=extracolumns,
205        relroot=REPO_ROOT,
206        **kwargs
207    )
208
209
210def _cli_capi(parser):
211    parser.add_argument('--levels', action='append', metavar='LEVEL[,...]')
212    parser.add_argument(f'--public', dest='levels',
213                        action='append_const', const='public')
214    parser.add_argument(f'--no-public', dest='levels',
215                        action='append_const', const='no-public')
216    for level in _capi.LEVELS:
217        parser.add_argument(f'--{level}', dest='levels',
218                            action='append_const', const=level)
219    def process_levels(args, *, argv=None):
220        levels = []
221        for raw in args.levels or ():
222            for level in raw.replace(',', ' ').strip().split():
223                if level == 'public':
224                    levels.append('stable')
225                    levels.append('cpython')
226                elif level == 'no-public':
227                    levels.append('private')
228                    levels.append('internal')
229                elif level in _capi.LEVELS:
230                    levels.append(level)
231                else:
232                    parser.error(f'expected LEVEL to be one of {sorted(_capi.LEVELS)}, got {level!r}')
233        args.levels = set(levels)
234
235    parser.add_argument('--kinds', action='append', metavar='KIND[,...]')
236    for kind in _capi.KINDS:
237        parser.add_argument(f'--{kind}', dest='kinds',
238                            action='append_const', const=kind)
239    def process_kinds(args, *, argv=None):
240        kinds = []
241        for raw in args.kinds or ():
242            for kind in raw.replace(',', ' ').strip().split():
243                if kind in _capi.KINDS:
244                    kinds.append(kind)
245                else:
246                    parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}')
247        args.kinds = set(kinds)
248
249    parser.add_argument('--group-by', dest='groupby',
250                        choices=['level', 'kind'])
251
252    parser.add_argument('--format', default='table')
253    parser.add_argument('--summary', dest='format',
254                        action='store_const', const='summary')
255    def process_format(args, *, argv=None):
256        orig = args.format
257        args.format = _capi.resolve_format(args.format)
258        if isinstance(args.format, str):
259            if args.format not in _capi._FORMATS:
260                parser.error(f'unsupported format {orig!r}')
261
262    parser.add_argument('--show-empty', dest='showempty', action='store_true')
263    parser.add_argument('--no-show-empty', dest='showempty', action='store_false')
264    parser.set_defaults(showempty=None)
265
266    # XXX Add --sort-by, --sort and --no-sort.
267
268    parser.add_argument('--ignore', dest='ignored', action='append')
269    def process_ignored(args, *, argv=None):
270        ignored = []
271        for raw in args.ignored or ():
272            ignored.extend(raw.replace(',', ' ').strip().split())
273        args.ignored = ignored or None
274
275    parser.add_argument('filenames', nargs='*', metavar='FILENAME')
276    process_progress = add_progress_cli(parser)
277
278    return [
279        process_levels,
280        process_kinds,
281        process_format,
282        process_ignored,
283        process_progress,
284    ]
285
286
287def cmd_capi(filenames=None, *,
288             levels=None,
289             kinds=None,
290             groupby='kind',
291             format='table',
292             showempty=None,
293             ignored=None,
294             track_progress=None,
295             verbosity=VERBOSITY,
296             **kwargs
297             ):
298    render = _capi.get_renderer(format)
299
300    filenames = _files.iter_header_files(filenames, levels=levels)
301    #filenames = (file for file, _ in main_for_filenames(filenames))
302    if track_progress:
303        filenames = track_progress(filenames)
304    items = _capi.iter_capi(filenames)
305    if levels:
306        items = (item for item in items if item.level in levels)
307    if kinds:
308        items = (item for item in items if item.kind in kinds)
309
310    filter = _capi.resolve_filter(ignored)
311    if filter:
312        items = (item for item in items if filter(item, log=lambda msg: logger.log(1, msg)))
313
314    lines = render(
315        items,
316        groupby=groupby,
317        showempty=showempty,
318        verbose=verbosity > VERBOSITY,
319    )
320    print()
321    for line in lines:
322        print(line)
323
324
325# We do not define any other cmd_*() handlers here,
326# favoring those defined elsewhere.
327
328COMMANDS = {
329    'check': (
330        'analyze and fail if the CPython source code has any problems',
331        [_cli_check],
332        cmd_check,
333    ),
334    'analyze': (
335        'report on the state of the CPython source code',
336        [(lambda p: c_analyzer._cli_analyze(p, **FILES_KWARGS))],
337        cmd_analyze,
338    ),
339    'parse': (
340        'parse the CPython source files',
341        [_cli_parse],
342        cmd_parse,
343    ),
344    'data': (
345        'check/manage local data (e.g. known types, ignored vars, caches)',
346        [_cli_data],
347        cmd_data,
348    ),
349    'capi': (
350        'inspect the C-API',
351        [_cli_capi],
352        cmd_capi,
353    ),
354}
355
356
357#######################################
358# the script
359
360def parse_args(argv=sys.argv[1:], prog=None, *, subset=None):
361    import argparse
362    parser = argparse.ArgumentParser(
363        prog=prog or get_prog(),
364    )
365
366#    if subset == 'check' or subset == ['check']:
367#        if checks is not None:
368#            commands = dict(COMMANDS)
369#            commands['check'] = list(commands['check'])
370#            cli = commands['check'][1][0]
371#            commands['check'][1][0] = (lambda p: cli(p, checks=checks))
372    processors = add_commands_cli(
373        parser,
374        commands=COMMANDS,
375        commonspecs=[
376            add_verbosity_cli,
377            add_traceback_cli,
378        ],
379        subset=subset,
380    )
381
382    args = parser.parse_args(argv)
383    ns = vars(args)
384
385    cmd = ns.pop('cmd')
386
387    verbosity, traceback_cm = process_args_by_key(
388        args,
389        argv,
390        processors[cmd],
391        ['verbosity', 'traceback_cm'],
392    )
393    if cmd != 'parse':
394        # "verbosity" is sent to the commands, so we put it back.
395        args.verbosity = verbosity
396
397    return cmd, ns, verbosity, traceback_cm
398
399
400def main(cmd, cmd_kwargs):
401    try:
402        run_cmd = COMMANDS[cmd][-1]
403    except KeyError:
404        raise ValueError(f'unsupported cmd {cmd!r}')
405    run_cmd(**cmd_kwargs)
406
407
408if __name__ == '__main__':
409    cmd, cmd_kwargs, verbosity, traceback_cm = parse_args()
410    configure_logger(verbosity)
411    with traceback_cm:
412        main(cmd, cmd_kwargs)
413