• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1import os.path
2
3from c_common.clsutil import classonly
4from c_parser.info import (
5    KIND,
6    Declaration,
7    TypeDeclaration,
8    Member,
9    FIXED_TYPE,
10)
11from c_parser.match import (
12    is_pots,
13    is_funcptr,
14)
15from c_analyzer.match import (
16    is_system_type,
17    is_process_global,
18    is_fixed_type,
19    is_immutable,
20)
21import c_analyzer as _c_analyzer
22import c_analyzer.info as _info
23import c_analyzer.datafiles as _datafiles
24from . import _parser, REPO_ROOT
25
26
27_DATA_DIR = os.path.dirname(__file__)
28KNOWN_FILE = os.path.join(_DATA_DIR, 'known.tsv')
29IGNORED_FILE = os.path.join(_DATA_DIR, 'ignored.tsv')
30NEED_FIX_FILE = os.path.join(_DATA_DIR, 'globals-to-fix.tsv')
31KNOWN_IN_DOT_C = {
32    'struct _odictobject': False,
33    'PyTupleObject': False,
34    'struct _typeobject': False,
35    'struct _arena': True,  # ???
36    'struct _frame': False,
37    'struct _ts': True,  # ???
38    'struct PyCodeObject': False,
39    'struct _is': True,  # ???
40    'PyWideStringList': True,  # ???
41    # recursive
42    'struct _dictkeysobject': False,
43}
44# These are loaded from the respective .tsv files upon first use.
45_KNOWN = {
46    # {(file, ID) | ID => info | bool}
47    #'PyWideStringList': True,
48}
49#_KNOWN = {(Struct(None, typeid.partition(' ')[-1], None)
50#           if typeid.startswith('struct ')
51#           else TypeDef(None, typeid, None)
52#           ): ([], {'unsupported': None if supported else True})
53#          for typeid, supported in _KNOWN_IN_DOT_C.items()}
54_IGNORED = {
55    # {ID => reason}
56}
57
58# XXX We should be handling these through known.tsv.
59_OTHER_SUPPORTED_TYPES = {
60    # Holds tuple of strings, which we statically initialize:
61    '_PyArg_Parser',
62    # Uses of these should be const, but we don't worry about it.
63    'PyModuleDef',
64    'PyModuleDef_Slot[]',
65    'PyType_Spec',
66    'PyType_Slot[]',
67    'PyMethodDef',
68    'PyMethodDef[]',
69    'PyMemberDef[]',
70    'PyGetSetDef[]',
71    'PyNumberMethods',
72    'PySequenceMethods',
73    'PyMappingMethods',
74    'PyAsyncMethods',
75    'PyBufferProcs',
76    'PyStructSequence_Field[]',
77    'PyStructSequence_Desc',
78}
79
80# XXX We should normalize all cases to a single name,
81# e.g. "kwlist" (currently the most common).
82_KWLIST_VARIANTS = [
83    ('*', 'kwlist'),
84    ('*', 'keywords'),
85    ('*', 'kwargs'),
86    ('Modules/_csv.c', 'dialect_kws'),
87    ('Modules/_datetimemodule.c', 'date_kws'),
88    ('Modules/_datetimemodule.c', 'datetime_kws'),
89    ('Modules/_datetimemodule.c', 'time_kws'),
90    ('Modules/_datetimemodule.c', 'timezone_kws'),
91    ('Modules/_lzmamodule.c', 'optnames'),
92    ('Modules/_lzmamodule.c', 'arg_names'),
93    ('Modules/cjkcodecs/multibytecodec.c', 'incnewkwarglist'),
94    ('Modules/cjkcodecs/multibytecodec.c', 'streamkwarglist'),
95    ('Modules/socketmodule.c', 'kwnames'),
96]
97
98KINDS = frozenset((*KIND.TYPES, KIND.VARIABLE))
99
100
101def read_known():
102    if not _KNOWN:
103        # Cache a copy the first time.
104        extracols = None  # XXX
105        #extracols = ['unsupported']
106        known = _datafiles.read_known(KNOWN_FILE, extracols, REPO_ROOT)
107        # For now we ignore known.values() (i.e. "extra").
108        types, _ = _datafiles.analyze_known(
109            known,
110            analyze_resolved=analyze_resolved,
111        )
112        _KNOWN.update(types)
113    return _KNOWN.copy()
114
115
116def write_known():
117    raise NotImplementedError
118    datafiles.write_known(decls, IGNORED_FILE, ['unsupported'], relroot=REPO_ROOT)
119
120
121def read_ignored():
122    if not _IGNORED:
123        _IGNORED.update(_datafiles.read_ignored(IGNORED_FILE, relroot=REPO_ROOT))
124        _IGNORED.update(_datafiles.read_ignored(NEED_FIX_FILE, relroot=REPO_ROOT))
125    return dict(_IGNORED)
126
127
128def write_ignored():
129    raise NotImplementedError
130    _datafiles.write_ignored(variables, IGNORED_FILE, relroot=REPO_ROOT)
131
132
133def analyze(filenames, *,
134            skip_objects=False,
135            **kwargs
136            ):
137    if skip_objects:
138        # XXX Set up a filter.
139        raise NotImplementedError
140
141    known = read_known()
142
143    decls = iter_decls(filenames)
144    results = _c_analyzer.analyze_decls(
145        decls,
146        known,
147        analyze_resolved=analyze_resolved,
148    )
149    analysis = Analysis.from_results(results)
150
151    return analysis
152
153
154def iter_decls(filenames, **kwargs):
155    decls = _c_analyzer.iter_decls(
156        filenames,
157        # We ignore functions (and statements).
158        kinds=KINDS,
159        parse_files=_parser.parse_files,
160        **kwargs
161    )
162    for decl in decls:
163        if not decl.data:
164            # Ignore forward declarations.
165            continue
166        yield decl
167
168
169def analyze_resolved(resolved, decl, types, knowntypes, extra=None):
170    if decl.kind not in KINDS:
171        # Skip it!
172        return None
173
174    typedeps = resolved
175    if typedeps is _info.UNKNOWN:
176        if decl.kind in (KIND.STRUCT, KIND.UNION):
177            typedeps = [typedeps] * len(decl.members)
178        else:
179            typedeps = [typedeps]
180    #assert isinstance(typedeps, (list, TypeDeclaration)), typedeps
181
182    if extra is None:
183        extra = {}
184    elif 'unsupported' in extra:
185        raise NotImplementedError((decl, extra))
186
187    unsupported = _check_unsupported(decl, typedeps, types, knowntypes)
188    extra['unsupported'] = unsupported
189
190    return typedeps, extra
191
192
193def _check_unsupported(decl, typedeps, types, knowntypes):
194    if typedeps is None:
195        raise NotImplementedError(decl)
196
197    if decl.kind in (KIND.STRUCT, KIND.UNION):
198        return _check_members(decl, typedeps, types, knowntypes)
199    elif decl.kind is KIND.ENUM:
200        if typedeps:
201            raise NotImplementedError((decl, typedeps))
202        return None
203    else:
204        return _check_typedep(decl, typedeps, types, knowntypes)
205
206
207def _check_members(decl, typedeps, types, knowntypes):
208    if isinstance(typedeps, TypeDeclaration):
209        raise NotImplementedError((decl, typedeps))
210
211    #members = decl.members or ()  # A forward decl has no members.
212    members = decl.members
213    if not members:
214        # A forward decl has no members, but that shouldn't surface here..
215        raise NotImplementedError(decl)
216    if len(members) != len(typedeps):
217        raise NotImplementedError((decl, typedeps))
218
219    unsupported = []
220    for member, typedecl in zip(members, typedeps):
221        checked = _check_typedep(member, typedecl, types, knowntypes)
222        unsupported.append(checked)
223    if any(None if v is FIXED_TYPE else v for v in unsupported):
224        return unsupported
225    elif FIXED_TYPE in unsupported:
226        return FIXED_TYPE
227    else:
228        return None
229
230
231def _check_typedep(decl, typedecl, types, knowntypes):
232    if not isinstance(typedecl, TypeDeclaration):
233        if hasattr(type(typedecl), '__len__'):
234            if len(typedecl) == 1:
235                typedecl, = typedecl
236    if typedecl is None:
237        # XXX Fail?
238        return 'typespec (missing)'
239    elif typedecl is _info.UNKNOWN:
240        if _has_other_supported_type(decl):
241            return None
242        # XXX Is this right?
243        return 'typespec (unknown)'
244    elif not isinstance(typedecl, TypeDeclaration):
245        raise NotImplementedError((decl, typedecl))
246
247    if isinstance(decl, Member):
248        return _check_vartype(decl, typedecl, types, knowntypes)
249    elif not isinstance(decl, Declaration):
250        raise NotImplementedError(decl)
251    elif decl.kind is KIND.TYPEDEF:
252        return _check_vartype(decl, typedecl, types, knowntypes)
253    elif decl.kind is KIND.VARIABLE:
254        if not is_process_global(decl):
255            return None
256        if _is_kwlist(decl):
257            return None
258        if _has_other_supported_type(decl):
259            return None
260        checked = _check_vartype(decl, typedecl, types, knowntypes)
261        return 'mutable' if checked is FIXED_TYPE else checked
262    else:
263        raise NotImplementedError(decl)
264
265
266def _is_kwlist(decl):
267    # keywords for PyArg_ParseTupleAndKeywords()
268    # "static char *name[]" -> "static const char * const name[]"
269    # XXX These should be made const.
270    for relpath, name in _KWLIST_VARIANTS:
271        if decl.name == name:
272            if relpath == '*':
273                break
274            assert os.path.isabs(decl.file.filename)
275            relpath = os.path.normpath(relpath)
276            if decl.file.filename.endswith(os.path.sep + relpath):
277                break
278    else:
279        return False
280    vartype = ''.join(str(decl.vartype).split())
281    return vartype == 'char*[]'
282
283
284def _has_other_supported_type(decl):
285    if hasattr(decl, 'file') and decl.file.filename.endswith('.c.h'):
286        assert 'clinic' in decl.file.filename, (decl,)
287        if decl.name == '_kwtuple':
288            return True
289    vartype = str(decl.vartype).split()
290    if vartype[0] == 'struct':
291        vartype = vartype[1:]
292    vartype = ''.join(vartype)
293    return vartype in _OTHER_SUPPORTED_TYPES
294
295
296def _check_vartype(decl, typedecl, types, knowntypes):
297    """Return failure reason."""
298    checked = _check_typespec(decl, typedecl, types, knowntypes)
299    if checked:
300        return checked
301    if is_immutable(decl.vartype):
302        return None
303    if is_fixed_type(decl.vartype):
304        return FIXED_TYPE
305    return 'mutable'
306
307
308def _check_typespec(decl, typedecl, types, knowntypes):
309    typespec = decl.vartype.typespec
310    if typedecl is not None:
311        found = types.get(typedecl)
312        if found is None:
313            found = knowntypes.get(typedecl)
314
315        if found is not None:
316            _, extra = found
317            if extra is None:
318                # XXX Under what circumstances does this happen?
319                extra = {}
320            unsupported = extra.get('unsupported')
321            if unsupported is FIXED_TYPE:
322                unsupported = None
323            return 'typespec' if unsupported else None
324    # Fall back to default known types.
325    if is_pots(typespec):
326        return None
327    elif is_system_type(typespec):
328        return None
329    elif is_funcptr(decl.vartype):
330        return None
331    return 'typespec'
332
333
334class Analyzed(_info.Analyzed):
335
336    @classonly
337    def is_target(cls, raw):
338        if not super().is_target(raw):
339            return False
340        if raw.kind not in KINDS:
341            return False
342        return True
343
344    #@classonly
345    #def _parse_raw_result(cls, result, extra):
346    #    typedecl, extra = super()._parse_raw_result(result, extra)
347    #    if typedecl is None:
348    #        return None, extra
349    #    raise NotImplementedError
350
351    def __init__(self, item, typedecl=None, *, unsupported=None, **extra):
352        if 'unsupported' in extra:
353            raise NotImplementedError((item, typedecl, unsupported, extra))
354        if not unsupported:
355            unsupported = None
356        elif isinstance(unsupported, (str, TypeDeclaration)):
357            unsupported = (unsupported,)
358        elif unsupported is not FIXED_TYPE:
359            unsupported = tuple(unsupported)
360        self.unsupported = unsupported
361        extra['unsupported'] = self.unsupported  # ...for __repr__(), etc.
362        if self.unsupported is None:
363            #self.supported = None
364            self.supported = True
365        elif self.unsupported is FIXED_TYPE:
366            if item.kind is KIND.VARIABLE:
367                raise NotImplementedError(item, typedecl, unsupported)
368            self.supported = True
369        else:
370            self.supported = not self.unsupported
371        super().__init__(item, typedecl, **extra)
372
373    def render(self, fmt='line', *, itemonly=False):
374        if fmt == 'raw':
375            yield repr(self)
376            return
377        rendered = super().render(fmt, itemonly=itemonly)
378        # XXX ???
379        #if itemonly:
380        #    yield from rendered
381        supported = self.supported
382        if fmt in ('line', 'brief'):
383            rendered, = rendered
384            parts = [
385                '+' if supported else '-' if supported is False else '',
386                rendered,
387            ]
388            yield '\t'.join(parts)
389        elif fmt == 'summary':
390            raise NotImplementedError(fmt)
391        elif fmt == 'full':
392            yield from rendered
393            if supported:
394                yield f'\tsupported:\t{supported}'
395        else:
396            raise NotImplementedError(fmt)
397
398
399class Analysis(_info.Analysis):
400    _item_class = Analyzed
401
402    @classonly
403    def build_item(cls, info, result=None):
404        if not isinstance(info, Declaration) or info.kind not in KINDS:
405            raise NotImplementedError((info, result))
406        return super().build_item(info, result)
407
408
409def check_globals(analysis):
410    # yield (data, failure)
411    ignored = read_ignored()
412    for item in analysis:
413        if item.kind != KIND.VARIABLE:
414            continue
415        if item.supported:
416            continue
417        if item.id in ignored:
418            continue
419        reason = item.unsupported
420        if not reason:
421            reason = '???'
422        elif not isinstance(reason, str):
423            if len(reason) == 1:
424                reason, = reason
425        reason = f'({reason})'
426        yield item, f'not supported {reason:20}\t{item.storage or ""} {item.vartype}'
427