• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2016 gRPC authors.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15from distutils import cygwinccompiler
16from distutils import extension
17from distutils import util
18import errno
19import os
20import os.path
21import platform
22import re
23import shlex
24import shutil
25import subprocess
26from subprocess import PIPE
27import sys
28import sysconfig
29
30import pkg_resources
31import setuptools
32from setuptools.command import build_ext
33
34# TODO(atash) add flag to disable Cython use
35
36_PACKAGE_PATH = os.path.realpath(os.path.dirname(__file__))
37_README_PATH = os.path.join(_PACKAGE_PATH, 'README.rst')
38
39os.chdir(os.path.dirname(os.path.abspath(__file__)))
40sys.path.insert(0, os.path.abspath('.'))
41
42import _parallel_compile_patch
43import protoc_lib_deps
44
45import grpc_version
46
47_EXT_INIT_SYMBOL = None
48if sys.version_info[0] == 2:
49    _EXT_INIT_SYMBOL = "init_protoc_compiler"
50else:
51    _EXT_INIT_SYMBOL = "PyInit__protoc_compiler"
52
53_parallel_compile_patch.monkeypatch_compile_maybe()
54
55CLASSIFIERS = [
56    'Development Status :: 5 - Production/Stable',
57    'Programming Language :: Python',
58    'Programming Language :: Python :: 3',
59    'License :: OSI Approved :: Apache Software License',
60]
61
62PY3 = sys.version_info.major == 3
63
64
65def _env_bool_value(env_name, default):
66    """Parses a bool option from an environment variable"""
67    return os.environ.get(env_name, default).upper() not in ['FALSE', '0', '']
68
69
70# Environment variable to determine whether or not the Cython extension should
71# *use* Cython or use the generated C files. Note that this requires the C files
72# to have been generated by building first *with* Cython support.
73BUILD_WITH_CYTHON = _env_bool_value('GRPC_PYTHON_BUILD_WITH_CYTHON', 'False')
74
75# Export this variable to force building the python extension with a statically linked libstdc++.
76# At least on linux, this is normally not needed as we can build manylinux-compatible wheels on linux just fine
77# without statically linking libstdc++ (which leads to a slight increase in the wheel size).
78# This option is useful when crosscompiling wheels for aarch64 where
79# it's difficult to ensure that the crosscompilation toolchain has a high-enough version
80# of GCC (we require >=5.1) but still uses old-enough libstdc++ symbols.
81# TODO(jtattermusch): remove this workaround once issues with crosscompiler version are resolved.
82BUILD_WITH_STATIC_LIBSTDCXX = _env_bool_value(
83    'GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX', 'False')
84
85
86def check_linker_need_libatomic():
87    """Test if linker on system needs libatomic."""
88    code_test = (b'#include <atomic>\n' +
89                 b'int main() { return std::atomic<int64_t>{}; }')
90    cxx = os.environ.get('CXX', 'c++')
91    cpp_test = subprocess.Popen([cxx, '-x', 'c++', '-std=c++14', '-'],
92                                stdin=PIPE,
93                                stdout=PIPE,
94                                stderr=PIPE)
95    cpp_test.communicate(input=code_test)
96    if cpp_test.returncode == 0:
97        return False
98    # Double-check to see if -latomic actually can solve the problem.
99    # https://github.com/grpc/grpc/issues/22491
100    cpp_test = subprocess.Popen(
101        [cxx, '-x', 'c++', '-std=c++14', '-', '-latomic'],
102        stdin=PIPE,
103        stdout=PIPE,
104        stderr=PIPE)
105    cpp_test.communicate(input=code_test)
106    return cpp_test.returncode == 0
107
108
109class BuildExt(build_ext.build_ext):
110    """Custom build_ext command."""
111
112    def get_ext_filename(self, ext_name):
113        # since python3.5, python extensions' shared libraries use a suffix that corresponds to the value
114        # of sysconfig.get_config_var('EXT_SUFFIX') and contains info about the architecture the library targets.
115        # E.g. on x64 linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so"
116        # When crosscompiling python wheels, we need to be able to override this suffix
117        # so that the resulting file name matches the target architecture and we end up with a well-formed
118        # wheel.
119        filename = build_ext.build_ext.get_ext_filename(self, ext_name)
120        orig_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
121        new_ext_suffix = os.getenv('GRPC_PYTHON_OVERRIDE_EXT_SUFFIX')
122        if new_ext_suffix and filename.endswith(orig_ext_suffix):
123            filename = filename[:-len(orig_ext_suffix)] + new_ext_suffix
124        return filename
125
126
127# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are
128# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
129# We use these environment variables to thus get around that without locking
130# ourselves in w.r.t. the multitude of operating systems this ought to build on.
131# We can also use these variables as a way to inject environment-specific
132# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a
133# reasonable default.
134EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None)
135EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None)
136if EXTRA_ENV_COMPILE_ARGS is None:
137    EXTRA_ENV_COMPILE_ARGS = '-std=c++14'
138    if 'win32' in sys.platform:
139        if sys.version_info < (3, 5):
140            # We use define flags here and don't directly add to DEFINE_MACROS below to
141            # ensure that the expert user/builder has a way of turning it off (via the
142            # envvars) without adding yet more GRPC-specific envvars.
143            # See https://sourceforge.net/p/mingw-w64/bugs/363/
144            if '32' in platform.architecture()[0]:
145                EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s -D_hypot=hypot'
146            else:
147                EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64 -D_hypot=hypot'
148        else:
149            # We need to statically link the C++ Runtime, only the C runtime is
150            # available dynamically
151            EXTRA_ENV_COMPILE_ARGS += ' /MT'
152    elif "linux" in sys.platform or "darwin" in sys.platform:
153        EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv -frtti'
154if EXTRA_ENV_LINK_ARGS is None:
155    EXTRA_ENV_LINK_ARGS = ''
156    # NOTE(rbellevi): Clang on Mac OS will make all static symbols (both
157    # variables and objects) global weak symbols. When a process loads the
158    # protobuf wheel's shared object library before loading *this* C extension,
159    # the runtime linker will prefer the protobuf module's version of symbols.
160    # This results in the process using a mixture of symbols from the protobuf
161    # wheel and this wheel, which may be using different versions of
162    # libprotobuf. In the case that they *are* using different versions of
163    # libprotobuf *and* there has been a change in data layout (or in other
164    # invariants) segfaults, data corruption, or "bad things" may happen.
165    #
166    # This flag ensures that on Mac, the only global symbol is the one loaded by
167    # the Python interpreter. The problematic global weak symbols become local
168    # weak symbols.  This is not required on Linux since the compiler does not
169    # produce global weak symbols. This is not required on Windows as our ".pyd"
170    # file does not contain any symbols.
171    #
172    # Finally, the leading underscore here is part of the Mach-O ABI. Unlike
173    # more modern ABIs (ELF et al.), Mach-O prepends an underscore to the names
174    # of C functions.
175    if "darwin" in sys.platform:
176        EXTRA_ENV_LINK_ARGS += ' -Wl,-exported_symbol,_{}'.format(
177            _EXT_INIT_SYMBOL)
178    if "linux" in sys.platform or "darwin" in sys.platform:
179        EXTRA_ENV_LINK_ARGS += ' -lpthread'
180        if check_linker_need_libatomic():
181            EXTRA_ENV_LINK_ARGS += ' -latomic'
182    elif "win32" in sys.platform and sys.version_info < (3, 5):
183        msvcr = cygwinccompiler.get_msvcr()[0]
184        EXTRA_ENV_LINK_ARGS += (
185            ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr}'
186            ' -static -lshlwapi'.format(msvcr=msvcr))
187
188EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
189EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
190
191if BUILD_WITH_STATIC_LIBSTDCXX:
192    EXTRA_LINK_ARGS.append('-static-libstdc++')
193
194CC_FILES = [os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES]
195PROTO_FILES = [
196    os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES
197]
198CC_INCLUDES = [
199    os.path.normpath(include_dir) for include_dir in protoc_lib_deps.CC_INCLUDES
200]
201PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE)
202
203GRPC_PYTHON_TOOLS_PACKAGE = 'grpc_tools'
204GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
205
206DEFINE_MACROS = ()
207if "win32" in sys.platform:
208    DEFINE_MACROS += (
209        ('WIN32_LEAN_AND_MEAN', 1),
210        # avoid https://github.com/abseil/abseil-cpp/issues/1425
211        ('NOMINMAX', 1),
212    )
213    if '64bit' in platform.architecture()[0]:
214        DEFINE_MACROS += (('MS_WIN64', 1),)
215elif "linux" in sys.platform or "darwin" in sys.platform:
216    DEFINE_MACROS += (('HAVE_PTHREAD', 1),)
217
218# By default, Python3 distutils enforces compatibility of
219# c plugins (.so files) with the OSX version Python was built with.
220# We need OSX 10.10, the oldest which supports C++ thread_local.
221if 'darwin' in sys.platform:
222    mac_target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
223    if mac_target and (pkg_resources.parse_version(mac_target) <
224                       pkg_resources.parse_version('10.10.0')):
225        os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.10'
226        os.environ['_PYTHON_HOST_PLATFORM'] = re.sub(
227            r'macosx-[0-9]+\.[0-9]+-(.+)', r'macosx-10.10-\1',
228            util.get_platform())
229
230
231def package_data():
232    tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep)
233    proto_resources_path = os.path.join(tools_path,
234                                        GRPC_PYTHON_PROTO_RESOURCES_NAME)
235    proto_files = []
236    for proto_file in PROTO_FILES:
237        source = os.path.join(PROTO_INCLUDE, proto_file)
238        target = os.path.join(proto_resources_path, proto_file)
239        relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME,
240                                       proto_file)
241        try:
242            os.makedirs(os.path.dirname(target))
243        except OSError as error:
244            if error.errno == errno.EEXIST:
245                pass
246            else:
247                raise
248        shutil.copy(source, target)
249        proto_files.append(relative_target)
250    return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files}
251
252
253def extension_modules():
254    if BUILD_WITH_CYTHON:
255        plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.pyx')]
256    else:
257        plugin_sources = [os.path.join('grpc_tools', '_protoc_compiler.cpp')]
258
259    plugin_sources += [
260        os.path.join('grpc_tools', 'main.cc'),
261        os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc'),
262        os.path.join('grpc_root', 'src', 'compiler', 'proto_parser_helper.cc')
263    ] + CC_FILES
264
265    plugin_ext = extension.Extension(
266        name='grpc_tools._protoc_compiler',
267        sources=plugin_sources,
268        include_dirs=[
269            '.',
270            'grpc_root',
271            os.path.join('grpc_root', 'include'),
272        ] + CC_INCLUDES,
273        language='c++',
274        define_macros=list(DEFINE_MACROS),
275        extra_compile_args=list(EXTRA_COMPILE_ARGS),
276        extra_link_args=list(EXTRA_LINK_ARGS),
277    )
278    extensions = [plugin_ext]
279    if BUILD_WITH_CYTHON:
280        from Cython import Build
281        return Build.cythonize(extensions)
282    else:
283        return extensions
284
285
286setuptools.setup(
287    name='grpcio-tools',
288    version=grpc_version.VERSION,
289    description='Protobuf code generator for gRPC',
290    long_description_content_type='text/x-rst',
291    long_description=open(_README_PATH, 'r').read(),
292    author='The gRPC Authors',
293    author_email='grpc-io@googlegroups.com',
294    url='https://grpc.io',
295    project_urls={
296        "Source Code":
297            "https://github.com/grpc/grpc/tree/master/tools/distrib/python/grpcio_tools",
298        "Bug Tracker":
299            "https://github.com/grpc/grpc/issues",
300    },
301    license='Apache License 2.0',
302    classifiers=CLASSIFIERS,
303    ext_modules=extension_modules(),
304    packages=setuptools.find_packages('.'),
305    python_requires='>=3.7',
306    install_requires=[
307        'protobuf>=4.21.6,<5.0dev',
308        'grpcio>={version}'.format(version=grpc_version.VERSION),
309        'setuptools',
310    ],
311    package_data=package_data(),
312    cmdclass={
313        'build_ext': BuildExt,
314    })
315