• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2018 The gRPC Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Patches the compile() to allow enable parallel compilation of C/C++.
15
16build_ext has lots of C/C++ files and normally them one by one.
17Enabling parallel build helps a lot.
18"""
19
20import distutils.ccompiler
21import os
22
23try:
24    BUILD_EXT_COMPILER_JOBS = int(
25        os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1'))
26except ValueError:
27    BUILD_EXT_COMPILER_JOBS = 1
28
29
30# monkey-patch for parallel compilation
31def _parallel_compile(self,
32                      sources,
33                      output_dir=None,
34                      macros=None,
35                      include_dirs=None,
36                      debug=0,
37                      extra_preargs=None,
38                      extra_postargs=None,
39                      depends=None):
40    # setup the same way as distutils.ccompiler.CCompiler
41    # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564
42    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
43        output_dir, macros, include_dirs, sources, depends, extra_postargs)
44    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
45
46    def _compile_single_file(obj):
47        try:
48            src, ext = build[obj]
49        except KeyError:
50            return
51        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
52
53    # run compilation of individual files in parallel
54    import multiprocessing.pool
55    multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map(
56        _compile_single_file, objects)
57    return objects
58
59
60def monkeypatch_compile_maybe():
61    """Monkeypatching is dumb, but the build speed gain is worth it."""
62    if BUILD_EXT_COMPILER_JOBS > 1:
63        distutils.ccompiler.CCompiler.compile = _parallel_compile
64