• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""
2Highly experimental script that compiles the CPython standard library using Cython.
3
4Execute the script either in the CPython 'Lib' directory or pass the
5option '--current-python' to compile the standard library of the running
6Python interpreter.
7
8Pass '-j N' to get a parallel build with N processes.
9
10Usage example::
11
12    $ python cystdlib.py --current-python build_ext -i
13"""
14
15import os
16import sys
17from distutils.core import setup
18from Cython.Build import cythonize
19from Cython.Compiler import Options
20
21# improve Python compatibility by allowing some broken code
22Options.error_on_unknown_names = False
23Options.error_on_uninitialized = False
24
25exclude_patterns = ['**/test/**/*.py', '**/tests/**/*.py', '**/__init__.py']
26broken = [
27    'idlelib/MultiCall.py',
28    'email/utils.py',
29    'multiprocessing/reduction.py',
30    'multiprocessing/util.py',
31    'threading.py',      # interrupt handling
32    'lib2to3/fixes/fix_sys_exc.py',
33    'traceback.py',
34    'types.py',
35    'enum.py',
36    'importlib/_bootstrap',
37]
38
39default_directives = dict(
40    auto_cpdef=False,   # enable when it's safe, see long list of failures below
41    binding=True,
42    set_initial_path='SOURCEFILE')
43default_directives['optimize.inline_defnode_calls'] = True
44
45special_directives = [
46    (['pkgutil.py',
47      'decimal.py',
48      'datetime.py',
49      'optparse.py',
50      'sndhdr.py',
51      'opcode.py',
52      'ntpath.py',
53      'urllib/request.py',
54      'plat-*/TYPES.py',
55      'plat-*/IN.py',
56      'tkinter/_fix.py',
57      'lib2to3/refactor.py',
58      'webbrowser.py',
59      'shutil.py',
60      'multiprocessing/forking.py',
61      'xml/sax/expatreader.py',
62      'xmlrpc/client.py',
63      'pydoc.py',
64      'xml/etree/ElementTree.py',
65      'posixpath.py',
66      'inspect.py',
67      'ctypes/util.py',
68      'urllib/parse.py',
69      'warnings.py',
70      'tempfile.py',
71      'trace.py',
72      'heapq.py',
73      'pickletools.py',
74      'multiprocessing/connection.py',
75      'hashlib.py',
76      'getopt.py',
77      'os.py',
78      'types.py',
79     ], dict(auto_cpdef=False)),
80]
81del special_directives[:]  # currently unused
82
83def build_extensions(includes='**/*.py',
84                     excludes=None,
85                     special_directives=special_directives,
86                     language_level=sys.version_info[0],
87                     parallel=None):
88    if isinstance(includes, str):
89        includes = [includes]
90    excludes = list(excludes or exclude_patterns) + broken
91
92    all_groups = (special_directives or []) + [(includes, {})]
93    extensions = []
94    for modules, directives in all_groups:
95        exclude_now = excludes[:]
96        for other_modules, _ in special_directives:
97            if other_modules != modules:
98                exclude_now.extend(other_modules)
99
100        d = dict(default_directives)
101        d.update(directives)
102
103        extensions.extend(
104            cythonize(
105                modules,
106                exclude=exclude_now,
107                exclude_failures=True,
108                language_level=language_level,
109                compiler_directives=d,
110                nthreads=parallel,
111            ))
112    return extensions
113
114
115def build(extensions):
116    try:
117        setup(ext_modules=extensions)
118        result = True
119    except:
120        import traceback
121        print('error building extensions %s' % (
122            [ext.name for ext in extensions],))
123        traceback.print_exc()
124        result = False
125    return extensions, result
126
127
128def _build(args):
129    sys_args, ext = args
130    sys.argv[1:] = sys_args
131    return build([ext])
132
133
134def parse_args():
135    from optparse import OptionParser
136    parser = OptionParser('%prog [options] [LIB_DIR (default: ./Lib)]')
137    parser.add_option(
138        '--current-python', dest='current_python', action='store_true',
139        help='compile the stdlib of the running Python')
140    parser.add_option(
141        '-j', '--jobs', dest='parallel_jobs', metavar='N',
142        type=int, default=1,
143        help='run builds in N parallel jobs (default: 1)')
144    parser.add_option(
145        '-x', '--exclude', dest='excludes', metavar='PATTERN',
146        action="append", help='exclude modules/packages matching PATTERN')
147    options, args = parser.parse_args()
148    if not args:
149        args = ['./Lib']
150    elif len(args) > 1:
151        parser.error('only one argument expected, got %d' % len(args))
152    return options, args
153
154
155if __name__ == '__main__':
156    options, args = parse_args()
157    if options.current_python:
158        # assume that the stdlib is where the "os" module lives
159        os.chdir(os.path.dirname(os.__file__))
160    else:
161        os.chdir(args[0])
162
163    pool = None
164    parallel_jobs = options.parallel_jobs
165    if options.parallel_jobs:
166        try:
167            import multiprocessing
168            pool = multiprocessing.Pool(parallel_jobs)
169            print("Building in %d parallel processes" % parallel_jobs)
170        except (ImportError, OSError):
171            print("Not building in parallel")
172            parallel_jobs = 0
173
174    extensions = build_extensions(
175        parallel=parallel_jobs,
176        excludes=options.excludes)
177    sys_args = ['build_ext', '-i']
178    if pool is not None:
179        results = pool.map(_build, [(sys_args, ext) for ext in extensions])
180        pool.close()
181        pool.join()
182        for ext, result in results:
183            if not result:
184                print("building extension %s failed" % (ext[0].name,))
185    else:
186        sys.argv[1:] = sys_args
187        build(extensions)
188