• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2018 The gRPC Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Patches the compile() to allow enable parallel compilation of C/C++.
15
16build_ext has lots of C/C++ files and normally them one by one.
17Enabling parallel build helps a lot.
18"""
19
20import distutils.ccompiler
21import os
22
23try:
24    BUILD_EXT_COMPILER_JOBS = int(
25        os.environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'])
26except KeyError:
27    import multiprocessing
28    BUILD_EXT_COMPILER_JOBS = multiprocessing.cpu_count()
29except ValueError:
30    BUILD_EXT_COMPILER_JOBS = 1
31
32
33# monkey-patch for parallel compilation
34def _parallel_compile(self,
35                      sources,
36                      output_dir=None,
37                      macros=None,
38                      include_dirs=None,
39                      debug=0,
40                      extra_preargs=None,
41                      extra_postargs=None,
42                      depends=None):
43    # setup the same way as distutils.ccompiler.CCompiler
44    # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564
45    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
46        str(output_dir), macros, include_dirs, sources, depends, extra_postargs)
47    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
48
49    def _compile_single_file(obj):
50        try:
51            src, ext = build[obj]
52        except KeyError:
53            return
54        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
55
56    # run compilation of individual files in parallel
57    import multiprocessing.pool
58    multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map(
59        _compile_single_file, objects)
60    return objects
61
62
63def monkeypatch_compile_maybe():
64    """Monkeypatching is dumb, but the build speed gain is worth it."""
65    if BUILD_EXT_COMPILER_JOBS > 1:
66        distutils.ccompiler.CCompiler.compile = _parallel_compile
67