1# Copyright 2016 gRPC authors. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import errno 16import os 17import os.path 18import platform 19import re 20import shlex 21import shutil 22import subprocess 23from subprocess import PIPE 24import sys 25import sysconfig 26 27import setuptools 28from setuptools import Extension 29from setuptools.command import build_ext 30 31# TODO(atash) add flag to disable Cython use 32 33_PACKAGE_PATH = os.path.realpath(os.path.dirname(__file__)) 34_README_PATH = os.path.join(_PACKAGE_PATH, "README.rst") 35 36os.chdir(os.path.dirname(os.path.abspath(__file__))) 37sys.path.insert(0, os.path.abspath(".")) 38 39import _parallel_compile_patch 40import _spawn_patch 41import protoc_lib_deps 42import python_version 43 44import grpc_version 45 46_EXT_INIT_SYMBOL = None 47if sys.version_info[0] == 2: 48 _EXT_INIT_SYMBOL = "init_protoc_compiler" 49else: 50 _EXT_INIT_SYMBOL = "PyInit__protoc_compiler" 51 52_parallel_compile_patch.monkeypatch_compile_maybe() 53_spawn_patch.monkeypatch_spawn() 54 55CLASSIFIERS = [ 56 "Development Status :: 5 - Production/Stable", 57 "Programming Language :: Python", 58 "Programming Language :: Python :: 3", 59 "License :: OSI Approved :: Apache Software License", 60] 61 62PY3 = sys.version_info.major == 3 63 64 65def _env_bool_value(env_name, default): 66 """Parses a bool option from an environment variable""" 67 return os.environ.get(env_name, default).upper() not in ["FALSE", "0", ""] 68 69 70# Environment variable to determine whether or not the Cython extension should 71# *use* Cython or use the generated C files. Note that this requires the C files 72# to have been generated by building first *with* Cython support. 73BUILD_WITH_CYTHON = _env_bool_value("GRPC_PYTHON_BUILD_WITH_CYTHON", "False") 74 75# Export this variable to force building the python extension with a statically linked libstdc++. 76# At least on linux, this is normally not needed as we can build manylinux-compatible wheels on linux just fine 77# without statically linking libstdc++ (which leads to a slight increase in the wheel size). 78# This option is useful when crosscompiling wheels for aarch64 where 79# it's difficult to ensure that the crosscompilation toolchain has a high-enough version 80# of GCC (we require >=5.1) but still uses old-enough libstdc++ symbols. 81# TODO(jtattermusch): remove this workaround once issues with crosscompiler version are resolved. 82BUILD_WITH_STATIC_LIBSTDCXX = _env_bool_value( 83 "GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX", "False" 84) 85 86 87def check_linker_need_libatomic(): 88 """Test if linker on system needs libatomic.""" 89 code_test = ( 90 b"#include <atomic>\n" 91 + b"int main() { return std::atomic<int64_t>{}; }" 92 ) 93 cxx = os.environ.get("CXX", "c++") 94 cpp_test = subprocess.Popen( 95 [cxx, "-x", "c++", "-std=c++17", "-"], 96 stdin=PIPE, 97 stdout=PIPE, 98 stderr=PIPE, 99 ) 100 cpp_test.communicate(input=code_test) 101 if cpp_test.returncode == 0: 102 return False 103 # Double-check to see if -latomic actually can solve the problem. 104 # https://github.com/grpc/grpc/issues/22491 105 cpp_test = subprocess.Popen( 106 [cxx, "-x", "c++", "-std=c++17", "-", "-latomic"], 107 stdin=PIPE, 108 stdout=PIPE, 109 stderr=PIPE, 110 ) 111 cpp_test.communicate(input=code_test) 112 return cpp_test.returncode == 0 113 114 115class BuildExt(build_ext.build_ext): 116 """Custom build_ext command.""" 117 118 def get_ext_filename(self, ext_name): 119 # since python3.5, python extensions' shared libraries use a suffix that corresponds to the value 120 # of sysconfig.get_config_var('EXT_SUFFIX') and contains info about the architecture the library targets. 121 # E.g. on x64 linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so" 122 # When crosscompiling python wheels, we need to be able to override this suffix 123 # so that the resulting file name matches the target architecture and we end up with a well-formed 124 # wheel. 125 filename = build_ext.build_ext.get_ext_filename(self, ext_name) 126 orig_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") 127 new_ext_suffix = os.getenv("GRPC_PYTHON_OVERRIDE_EXT_SUFFIX") 128 if new_ext_suffix and filename.endswith(orig_ext_suffix): 129 filename = filename[: -len(orig_ext_suffix)] + new_ext_suffix 130 return filename 131 132 def build_extensions(self): 133 # This is to let UnixCompiler get either C or C++ compiler options depending on the source. 134 # Note that this doesn't work for MSVCCompiler and will be handled by _spawn_patch.py. 135 old_compile = self.compiler._compile 136 137 def new_compile(obj, src, ext, cc_args, extra_postargs, pp_opts): 138 if src.endswith(".c"): 139 extra_postargs = [ 140 arg for arg in extra_postargs if arg != "-std=c++17" 141 ] 142 elif src.endswith((".cc", ".cpp")): 143 extra_postargs = [ 144 arg for arg in extra_postargs if arg != "-std=c11" 145 ] 146 return old_compile(obj, src, ext, cc_args, extra_postargs, pp_opts) 147 148 self.compiler._compile = new_compile 149 150 build_ext.build_ext.build_extensions(self) 151 152 153# When building extensions for macOS on a system running macOS 10.14 or newer, 154# make sure they target macOS 10.14 or newer to use C++17 stdlib properly. 155# This overrides the default behavior of distutils, which targets the macOS 156# version Python was built on. You can further customize the target macOS 157# version by setting the MACOSX_DEPLOYMENT_TARGET environment variable before 158# running setup.py. 159if sys.platform == "darwin": 160 if "MACOSX_DEPLOYMENT_TARGET" not in os.environ: 161 target_ver = sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET") 162 if target_ver == "" or tuple(int(p) for p in target_ver.split(".")) < ( 163 10, 164 14, 165 ): 166 os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.14" 167 168# There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are 169# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. 170# We use these environment variables to thus get around that without locking 171# ourselves in w.r.t. the multitude of operating systems this ought to build on. 172# We can also use these variables as a way to inject environment-specific 173# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a 174# reasonable default. 175EXTRA_ENV_COMPILE_ARGS = os.environ.get("GRPC_PYTHON_CFLAGS", None) 176EXTRA_ENV_LINK_ARGS = os.environ.get("GRPC_PYTHON_LDFLAGS", None) 177if EXTRA_ENV_COMPILE_ARGS is None: 178 EXTRA_ENV_COMPILE_ARGS = "" 179 if "win32" in sys.platform: 180 # MSVC by defaults uses C++14 and C89 so both needs to be configured. 181 EXTRA_ENV_COMPILE_ARGS += " /std:c++17" 182 EXTRA_ENV_COMPILE_ARGS += " /std:c11" 183 # We need to statically link the C++ Runtime, only the C runtime is 184 # available dynamically 185 EXTRA_ENV_COMPILE_ARGS += " /MT" 186 elif "linux" in sys.platform: 187 # GCC by defaults uses C17 so only C++17 needs to be specified. 188 EXTRA_ENV_COMPILE_ARGS += " -std=c++17" 189 EXTRA_ENV_COMPILE_ARGS += " -fno-wrapv -frtti" 190 # Reduce the optimization level from O3 (in many cases) to O1 to 191 # workaround gcc misalignment bug with MOVAPS (internal b/329134877) 192 EXTRA_ENV_COMPILE_ARGS += " -O1" 193 elif "darwin" in sys.platform: 194 # AppleClang by defaults uses C17 so only C++17 needs to be specified. 195 EXTRA_ENV_COMPILE_ARGS += " -std=c++17" 196 EXTRA_ENV_COMPILE_ARGS += " -fno-wrapv -frtti" 197 EXTRA_ENV_COMPILE_ARGS += " -stdlib=libc++ -DHAVE_UNISTD_H" 198if EXTRA_ENV_LINK_ARGS is None: 199 EXTRA_ENV_LINK_ARGS = "" 200 # This is needed for protobuf/main.cc 201 if "win32" in sys.platform: 202 EXTRA_ENV_LINK_ARGS += " Shell32.lib" 203 # NOTE(rbellevi): Clang on Mac OS will make all static symbols (both 204 # variables and objects) global weak symbols. When a process loads the 205 # protobuf wheel's shared object library before loading *this* C extension, 206 # the runtime linker will prefer the protobuf module's version of symbols. 207 # This results in the process using a mixture of symbols from the protobuf 208 # wheel and this wheel, which may be using different versions of 209 # libprotobuf. In the case that they *are* using different versions of 210 # libprotobuf *and* there has been a change in data layout (or in other 211 # invariants) segfaults, data corruption, or "bad things" may happen. 212 # 213 # This flag ensures that on Mac, the only global symbol is the one loaded by 214 # the Python interpreter. The problematic global weak symbols become local 215 # weak symbols. This is not required on Linux since the compiler does not 216 # produce global weak symbols. This is not required on Windows as our ".pyd" 217 # file does not contain any symbols. 218 # 219 # Finally, the leading underscore here is part of the Mach-O ABI. Unlike 220 # more modern ABIs (ELF et al.), Mach-O prepends an underscore to the names 221 # of C functions. 222 if "darwin" in sys.platform: 223 EXTRA_ENV_LINK_ARGS += " -Wl,-exported_symbol,_{}".format( 224 _EXT_INIT_SYMBOL 225 ) 226 if "linux" in sys.platform or "darwin" in sys.platform: 227 EXTRA_ENV_LINK_ARGS += " -lpthread" 228 if check_linker_need_libatomic(): 229 EXTRA_ENV_LINK_ARGS += " -latomic" 230 231# Explicitly link Core Foundation framework for MacOS to ensure no symbol is 232# missing when compiled using package managers like Conda. 233if "darwin" in sys.platform: 234 EXTRA_ENV_LINK_ARGS += " -framework CoreFoundation" 235 236EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) 237EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) 238 239if BUILD_WITH_STATIC_LIBSTDCXX: 240 EXTRA_LINK_ARGS.append("-static-libstdc++") 241 242CC_FILES = [os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES] 243PROTO_FILES = [ 244 os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES 245] 246CC_INCLUDES = [ 247 os.path.normpath(include_dir) for include_dir in protoc_lib_deps.CC_INCLUDES 248] 249PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE) 250 251GRPC_PYTHON_TOOLS_PACKAGE = "grpc_tools" 252GRPC_PYTHON_PROTO_RESOURCES_NAME = "_proto" 253 254DEFINE_MACROS = () 255if "win32" in sys.platform: 256 DEFINE_MACROS += ( 257 ("WIN32_LEAN_AND_MEAN", 1), 258 # avoid https://github.com/abseil/abseil-cpp/issues/1425 259 ("NOMINMAX", 1), 260 ) 261 if "64bit" in platform.architecture()[0]: 262 DEFINE_MACROS += (("MS_WIN64", 1),) 263elif "linux" in sys.platform or "darwin" in sys.platform: 264 DEFINE_MACROS += (("HAVE_PTHREAD", 1),) 265 266 267def package_data(): 268 tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace(".", os.path.sep) 269 proto_resources_path = os.path.join( 270 tools_path, GRPC_PYTHON_PROTO_RESOURCES_NAME 271 ) 272 proto_files = [] 273 for proto_file in PROTO_FILES: 274 source = os.path.join(PROTO_INCLUDE, proto_file) 275 target = os.path.join(proto_resources_path, proto_file) 276 relative_target = os.path.join( 277 GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file 278 ) 279 try: 280 os.makedirs(os.path.dirname(target)) 281 except OSError as error: 282 if error.errno == errno.EEXIST: 283 pass 284 else: 285 raise 286 shutil.copy(source, target) 287 proto_files.append(relative_target) 288 return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files} 289 290 291def extension_modules(): 292 if BUILD_WITH_CYTHON: 293 plugin_sources = [os.path.join("grpc_tools", "_protoc_compiler.pyx")] 294 else: 295 plugin_sources = [os.path.join("grpc_tools", "_protoc_compiler.cpp")] 296 297 plugin_sources += [ 298 os.path.join("grpc_tools", "main.cc"), 299 os.path.join("grpc_root", "src", "compiler", "python_generator.cc"), 300 os.path.join("grpc_root", "src", "compiler", "proto_parser_helper.cc"), 301 ] + CC_FILES 302 303 plugin_ext = Extension( 304 name="grpc_tools._protoc_compiler", 305 sources=plugin_sources, 306 include_dirs=[ 307 ".", 308 "grpc_root", 309 os.path.join("grpc_root", "include"), 310 ] 311 + CC_INCLUDES, 312 define_macros=list(DEFINE_MACROS), 313 extra_compile_args=list(EXTRA_COMPILE_ARGS), 314 extra_link_args=list(EXTRA_LINK_ARGS), 315 ) 316 extensions = [plugin_ext] 317 if BUILD_WITH_CYTHON: 318 from Cython import Build 319 320 return Build.cythonize(extensions) 321 else: 322 return extensions 323 324 325setuptools.setup( 326 name="grpcio-tools", 327 version=grpc_version.VERSION, 328 description="Protobuf code generator for gRPC", 329 long_description_content_type="text/x-rst", 330 long_description=open(_README_PATH, "r").read(), 331 author="The gRPC Authors", 332 author_email="grpc-io@googlegroups.com", 333 url="https://grpc.io", 334 project_urls={ 335 "Source Code": "https://github.com/grpc/grpc/tree/master/tools/distrib/python/grpcio_tools", 336 "Bug Tracker": "https://github.com/grpc/grpc/issues", 337 }, 338 license="Apache License 2.0", 339 classifiers=CLASSIFIERS, 340 ext_modules=extension_modules(), 341 packages=setuptools.find_packages("."), 342 python_requires=f">={python_version.MIN_PYTHON_VERSION}", 343 install_requires=[ 344 "protobuf>=5.26.1,<6.0dev", 345 "grpcio>={version}".format(version=grpc_version.VERSION), 346 "setuptools", 347 ], 348 package_data=package_data(), 349 cmdclass={ 350 "build_ext": BuildExt, 351 }, 352) 353