1import os.path 2import re 3 4from c_parser.preprocessor import ( 5 get_preprocessor as _get_preprocessor, 6) 7from c_parser import ( 8 parse_file as _parse_file, 9 parse_files as _parse_files, 10) 11from . import REPO_ROOT 12 13 14GLOB_ALL = '**/*' 15 16 17def clean_lines(text): 18 """Clear out comments, blank lines, and leading/trailing whitespace.""" 19 lines = (line.strip() for line in text.splitlines()) 20 lines = (line.partition('#')[0].rstrip() 21 for line in lines 22 if line and not line.startswith('#')) 23 glob_all = f'{GLOB_ALL} ' 24 lines = (re.sub(r'^[*] ', glob_all, line) for line in lines) 25 lines = (os.path.join(REPO_ROOT, line) for line in lines) 26 return list(lines) 27 28 29''' 30@begin=sh@ 31./python ../c-parser/cpython.py 32 --exclude '+../c-parser/EXCLUDED' 33 --macros '+../c-parser/MACROS' 34 --incldirs '+../c-parser/INCL_DIRS' 35 --same './Include/cpython/' 36 Include/*.h 37 Include/internal/*.h 38 Modules/**/*.c 39 Objects/**/*.c 40 Parser/**/*.c 41 Python/**/*.c 42@end=sh@ 43''' 44 45EXCLUDED = clean_lines(''' 46# @begin=conf@ 47 48# Rather than fixing for this one, we manually make sure it's okay. 49Modules/_sha3/kcp/KeccakP-1600-opt64.c 50 51# OSX 52#Modules/_ctypes/darwin/*.c 53#Modules/_ctypes/libffi_osx/*.c 54Modules/_scproxy.c # SystemConfiguration/SystemConfiguration.h 55 56# Windows 57Modules/_winapi.c # windows.h 58Modules/overlapped.c # winsock.h 59Python/dynload_win.c # windows.h 60Modules/expat/winconfig.h 61Python/thread_nt.h 62 63# other OS-dependent 64Python/dynload_dl.c # dl.h 65Python/dynload_hpux.c # dl.h 66Python/dynload_aix.c # sys/ldr.h 67Python/thread_pthread.h 68 69# only huge constants (safe but parsing is slow) 70Modules/_ssl_data.h 71Modules/unicodedata_db.h 72Modules/unicodename_db.h 73Modules/cjkcodecs/mappings_*.h 74Objects/unicodetype_db.h 75Python/importlib.h 76Python/importlib_external.h 77Python/importlib_zipimport.h 78 79# @end=conf@ 80''') 81 82# XXX Fix the parser. 83EXCLUDED += clean_lines(''' 84# The tool should be able to parse these... 85 86Modules/hashlib.h 87Objects/stringlib/codecs.h 88Objects/stringlib/count.h 89Objects/stringlib/ctype.h 90Objects/stringlib/fastsearch.h 91Objects/stringlib/find.h 92Objects/stringlib/find_max_char.h 93Objects/stringlib/partition.h 94Objects/stringlib/replace.h 95Objects/stringlib/split.h 96 97Modules/_dbmmodule.c 98Modules/cjkcodecs/_codecs_*.c 99Modules/expat/xmlrole.c 100Modules/expat/xmlparse.c 101Python/initconfig.c 102''') 103 104INCL_DIRS = clean_lines(''' 105# @begin=tsv@ 106 107glob dirname 108* . 109* ./Include 110* ./Include/internal 111 112Modules/_tkinter.c /usr/include/tcl8.6 113Modules/tkappinit.c /usr/include/tcl 114Modules/_decimal/**/*.c Modules/_decimal/libmpdec 115 116# @end=tsv@ 117''')[1:] 118 119MACROS = clean_lines(''' 120# @begin=tsv@ 121 122glob name value 123 124Include/internal/*.h Py_BUILD_CORE 1 125Python/**/*.c Py_BUILD_CORE 1 126Parser/**/*.c Py_BUILD_CORE 1 127Objects/**/*.c Py_BUILD_CORE 1 128 129Modules/faulthandler.c Py_BUILD_CORE 1 130Modules/_functoolsmodule.c Py_BUILD_CORE 1 131Modules/gcmodule.c Py_BUILD_CORE 1 132Modules/getpath.c Py_BUILD_CORE 1 133Modules/_io/*.c Py_BUILD_CORE 1 134Modules/itertoolsmodule.c Py_BUILD_CORE 1 135Modules/_localemodule.c Py_BUILD_CORE 1 136Modules/main.c Py_BUILD_CORE 1 137Modules/posixmodule.c Py_BUILD_CORE 1 138Modules/signalmodule.c Py_BUILD_CORE 1 139Modules/_threadmodule.c Py_BUILD_CORE 1 140Modules/_tracemalloc.c Py_BUILD_CORE 1 141Modules/_asynciomodule.c Py_BUILD_CORE 1 142Modules/mathmodule.c Py_BUILD_CORE 1 143Modules/cmathmodule.c Py_BUILD_CORE 1 144Modules/_weakref.c Py_BUILD_CORE 1 145Modules/sha256module.c Py_BUILD_CORE 1 146Modules/sha512module.c Py_BUILD_CORE 1 147Modules/_datetimemodule.c Py_BUILD_CORE 1 148Modules/_ctypes/cfield.c Py_BUILD_CORE 1 149Modules/_heapqmodule.c Py_BUILD_CORE 1 150Modules/_posixsubprocess.c Py_BUILD_CORE 1 151Modules/_sre.c Py_BUILD_CORE 1 152Modules/_collectionsmodule.c Py_BUILD_CORE 1 153Modules/_zoneinfo.c Py_BUILD_CORE 1 154Modules/unicodedata.c Py_BUILD_CORE 1 155Modules/_cursesmodule.c Py_BUILD_CORE 1 156Modules/_ctypes/_ctypes.c Py_BUILD_CORE 1 157Objects/stringlib/codecs.h Py_BUILD_CORE 1 158Python/ceval_gil.h Py_BUILD_CORE 1 159Python/condvar.h Py_BUILD_CORE 1 160 161Modules/_json.c Py_BUILD_CORE_BUILTIN 1 162Modules/_pickle.c Py_BUILD_CORE_BUILTIN 1 163Modules/_testinternalcapi.c Py_BUILD_CORE_BUILTIN 1 164 165Include/cpython/abstract.h Py_CPYTHON_ABSTRACTOBJECT_H 1 166Include/cpython/bytearrayobject.h Py_CPYTHON_BYTEARRAYOBJECT_H 1 167Include/cpython/bytesobject.h Py_CPYTHON_BYTESOBJECT_H 1 168Include/cpython/ceval.h Py_CPYTHON_CEVAL_H 1 169Include/cpython/code.h Py_CPYTHON_CODE_H 1 170Include/cpython/dictobject.h Py_CPYTHON_DICTOBJECT_H 1 171Include/cpython/fileobject.h Py_CPYTHON_FILEOBJECT_H 1 172Include/cpython/fileutils.h Py_CPYTHON_FILEUTILS_H 1 173Include/cpython/frameobject.h Py_CPYTHON_FRAMEOBJECT_H 1 174Include/cpython/import.h Py_CPYTHON_IMPORT_H 1 175Include/cpython/interpreteridobject.h Py_CPYTHON_INTERPRETERIDOBJECT_H 1 176Include/cpython/listobject.h Py_CPYTHON_LISTOBJECT_H 1 177Include/cpython/methodobject.h Py_CPYTHON_METHODOBJECT_H 1 178Include/cpython/object.h Py_CPYTHON_OBJECT_H 1 179Include/cpython/objimpl.h Py_CPYTHON_OBJIMPL_H 1 180Include/cpython/pyerrors.h Py_CPYTHON_ERRORS_H 1 181Include/cpython/pylifecycle.h Py_CPYTHON_PYLIFECYCLE_H 1 182Include/cpython/pymem.h Py_CPYTHON_PYMEM_H 1 183Include/cpython/pystate.h Py_CPYTHON_PYSTATE_H 1 184Include/cpython/sysmodule.h Py_CPYTHON_SYSMODULE_H 1 185Include/cpython/traceback.h Py_CPYTHON_TRACEBACK_H 1 186Include/cpython/tupleobject.h Py_CPYTHON_TUPLEOBJECT_H 1 187Include/cpython/unicodeobject.h Py_CPYTHON_UNICODEOBJECT_H 1 188 189# implied include of pyport.h 190Include/**/*.h PyAPI_DATA(RTYPE) extern RTYPE 191Include/**/*.h PyAPI_FUNC(RTYPE) RTYPE 192Include/**/*.h Py_DEPRECATED(VER) /* */ 193Include/**/*.h _Py_NO_RETURN /* */ 194Include/**/*.h PYLONG_BITS_IN_DIGIT 30 195Modules/**/*.c PyMODINIT_FUNC PyObject* 196Objects/unicodeobject.c PyMODINIT_FUNC PyObject* 197Python/marshal.c PyMODINIT_FUNC PyObject* 198Python/_warnings.c PyMODINIT_FUNC PyObject* 199Python/Python-ast.c PyMODINIT_FUNC PyObject* 200Python/import.c PyMODINIT_FUNC PyObject* 201Modules/_testcapimodule.c PyAPI_FUNC(RTYPE) RTYPE 202Python/getargs.c PyAPI_FUNC(RTYPE) RTYPE 203Objects/stringlib/unicode_format.h Py_LOCAL_INLINE(type) static inline type 204 205# implied include of pymacro.h 206*/clinic/*.c.h PyDoc_VAR(name) static const char name[] 207*/clinic/*.c.h PyDoc_STR(str) str 208*/clinic/*.c.h PyDoc_STRVAR(name,str) PyDoc_VAR(name) = PyDoc_STR(str) 209 210# implied include of exports.h 211#Modules/_io/bytesio.c Py_EXPORTED_SYMBOL /* */ 212 213# implied include of object.h 214Include/**/*.h PyObject_HEAD PyObject ob_base; 215Include/**/*.h PyObject_VAR_HEAD PyVarObject ob_base; 216 217# implied include of pyconfig.h 218Include/**/*.h SIZEOF_WCHAR_T 4 219 220# implied include of <unistd.h> 221Include/**/*.h _POSIX_THREADS 1 222 223# from Makefile 224Modules/getpath.c PYTHONPATH 1 225Modules/getpath.c PREFIX ... 226Modules/getpath.c EXEC_PREFIX ... 227Modules/getpath.c VERSION ... 228Modules/getpath.c VPATH ... 229 230# from Modules/_sha3/sha3module.c 231Modules/_sha3/kcp/KeccakP-1600-inplace32BI.c PLATFORM_BYTE_ORDER 4321 # force big-endian 232Modules/_sha3/kcp/*.c KeccakOpt 64 233Modules/_sha3/kcp/*.c KeccakP200_excluded 1 234Modules/_sha3/kcp/*.c KeccakP400_excluded 1 235Modules/_sha3/kcp/*.c KeccakP800_excluded 1 236 237# See: setup.py 238Modules/_decimal/**/*.c CONFIG_64 1 239Modules/_decimal/**/*.c ASM 1 240Modules/expat/xmlparse.c HAVE_EXPAT_CONFIG_H 1 241Modules/expat/xmlparse.c XML_POOR_ENTROPY 1 242Modules/_dbmmodule.c HAVE_GDBM_DASH_NDBM_H 1 243 244# others 245Modules/sre_lib.h LOCAL(type) static inline type 246Modules/sre_lib.h SRE(F) sre_ucs2_##F 247Objects/stringlib/codecs.h STRINGLIB_IS_UNICODE 1 248 249# @end=tsv@ 250''')[1:] 251 252# -pthread 253# -Wno-unused-result 254# -Wsign-compare 255# -g 256# -Og 257# -Wall 258# -std=c99 259# -Wextra 260# -Wno-unused-result -Wno-unused-parameter 261# -Wno-missing-field-initializers 262# -Werror=implicit-function-declaration 263 264SAME = [ 265 './Include/cpython/', 266] 267 268 269def get_preprocessor(*, 270 file_macros=None, 271 file_incldirs=None, 272 file_same=None, 273 **kwargs 274 ): 275 macros = tuple(MACROS) 276 if file_macros: 277 macros += tuple(file_macros) 278 incldirs = tuple(INCL_DIRS) 279 if file_incldirs: 280 incldirs += tuple(file_incldirs) 281 return _get_preprocessor( 282 file_macros=macros, 283 file_incldirs=incldirs, 284 file_same=file_same, 285 **kwargs 286 ) 287 288 289def parse_file(filename, *, 290 match_kind=None, 291 ignore_exc=None, 292 log_err=None, 293 ): 294 get_file_preprocessor = get_preprocessor( 295 ignore_exc=ignore_exc, 296 log_err=log_err, 297 ) 298 yield from _parse_file( 299 filename, 300 match_kind=match_kind, 301 get_file_preprocessor=get_file_preprocessor, 302 ) 303 304 305def parse_files(filenames=None, *, 306 match_kind=None, 307 ignore_exc=None, 308 log_err=None, 309 get_file_preprocessor=None, 310 **file_kwargs 311 ): 312 if get_file_preprocessor is None: 313 get_file_preprocessor = get_preprocessor( 314 ignore_exc=ignore_exc, 315 log_err=log_err, 316 ) 317 yield from _parse_files( 318 filenames, 319 match_kind=match_kind, 320 get_file_preprocessor=get_file_preprocessor, 321 **file_kwargs 322 ) 323