1# Copyright 2015 gRPC authors. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Provides distutils command classes for the GRPC Python setup process.""" 15 16import distutils 17import glob 18import os 19import os.path 20import platform 21import re 22import shutil 23import subprocess 24import sys 25import traceback 26 27import setuptools 28from setuptools.command import build_ext 29from setuptools.command import build_py 30from setuptools.command import easy_install 31from setuptools.command import install 32from setuptools.command import test 33 34import support 35 36PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) 37GRPC_STEM = os.path.abspath(PYTHON_STEM + '../../../../') 38PROTO_STEM = os.path.join(GRPC_STEM, 'src', 'proto') 39PROTO_GEN_STEM = os.path.join(GRPC_STEM, 'src', 'python', 'gens') 40CYTHON_STEM = os.path.join(PYTHON_STEM, 'grpc', '_cython') 41 42CONF_PY_ADDENDUM = """ 43extensions.append('sphinx.ext.napoleon') 44napoleon_google_docstring = True 45napoleon_numpy_docstring = True 46napoleon_include_special_with_doc = True 47 48html_theme = 'sphinx_rtd_theme' 49copyright = "2016, The gRPC Authors" 50""" 51 52API_GLOSSARY = """ 53 54Glossary 55================ 56 57.. glossary:: 58 59 metadatum 60 A key-value pair included in the HTTP header. It is a 61 2-tuple where the first entry is the key and the 62 second is the value, i.e. (key, value). The metadata key is an ASCII str, 63 and must be a valid HTTP header name. The metadata value can be 64 either a valid HTTP ASCII str, or bytes. If bytes are provided, 65 the key must end with '-bin', i.e. 66 ``('binary-metadata-bin', b'\\x00\\xFF')`` 67 68 metadata 69 A sequence of metadatum. 70""" 71 72 73class CommandError(Exception): 74 """Simple exception class for GRPC custom commands.""" 75 76 77# TODO(atash): Remove this once PyPI has better Linux bdist support. See 78# https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported 79def _get_grpc_custom_bdist(decorated_basename, target_bdist_basename): 80 """Returns a string path to a bdist file for Linux to install. 81 82 If we can retrieve a pre-compiled bdist from online, uses it. Else, emits a 83 warning and builds from source. 84 """ 85 # TODO(atash): somehow the name that's returned from `wheel` is different 86 # between different versions of 'wheel' (but from a compatibility standpoint, 87 # the names are compatible); we should have some way of determining name 88 # compatibility in the same way `wheel` does to avoid having to rename all of 89 # the custom wheels that we build/upload to GCS. 90 91 # Break import style to ensure that setup.py has had a chance to install the 92 # relevant package. 93 from six.moves.urllib import request 94 decorated_path = decorated_basename + GRPC_CUSTOM_BDIST_EXT 95 try: 96 url = BINARIES_REPOSITORY + '/{target}'.format(target=decorated_path) 97 bdist_data = request.urlopen(url).read() 98 except IOError as error: 99 raise CommandError('{}\n\nCould not find the bdist {}: {}'.format( 100 traceback.format_exc(), decorated_path, error.message)) 101 # Our chosen local bdist path. 102 bdist_path = target_bdist_basename + GRPC_CUSTOM_BDIST_EXT 103 try: 104 with open(bdist_path, 'w') as bdist_file: 105 bdist_file.write(bdist_data) 106 except IOError as error: 107 raise CommandError('{}\n\nCould not write grpcio bdist: {}'.format( 108 traceback.format_exc(), error.message)) 109 return bdist_path 110 111 112class SphinxDocumentation(setuptools.Command): 113 """Command to generate documentation via sphinx.""" 114 115 description = 'generate sphinx documentation' 116 user_options = [] 117 118 def initialize_options(self): 119 pass 120 121 def finalize_options(self): 122 pass 123 124 def run(self): 125 # We import here to ensure that setup.py has had a chance to install the 126 # relevant package eggs first. 127 import sphinx 128 import sphinx.apidoc 129 metadata = self.distribution.metadata 130 src_dir = os.path.join(PYTHON_STEM, 'grpc') 131 sys.path.append(src_dir) 132 sphinx.apidoc.main([ 133 '', '--force', '--full', '-H', metadata.name, '-A', metadata.author, 134 '-V', metadata.version, '-R', metadata.version, '-o', 135 os.path.join('doc', 'src'), src_dir 136 ]) 137 conf_filepath = os.path.join('doc', 'src', 'conf.py') 138 with open(conf_filepath, 'a') as conf_file: 139 conf_file.write(CONF_PY_ADDENDUM) 140 glossary_filepath = os.path.join('doc', 'src', 'grpc.rst') 141 with open(glossary_filepath, 'a') as glossary_filepath: 142 glossary_filepath.write(API_GLOSSARY) 143 sphinx.main( 144 ['', os.path.join('doc', 'src'), 145 os.path.join('doc', 'build')]) 146 147 148class BuildProjectMetadata(setuptools.Command): 149 """Command to generate project metadata in a module.""" 150 151 description = 'build grpcio project metadata files' 152 user_options = [] 153 154 def initialize_options(self): 155 pass 156 157 def finalize_options(self): 158 pass 159 160 def run(self): 161 with open(os.path.join(PYTHON_STEM, 'grpc/_grpcio_metadata.py'), 162 'w') as module_file: 163 module_file.write('__version__ = """{}"""'.format( 164 self.distribution.get_version())) 165 166 167class BuildPy(build_py.build_py): 168 """Custom project build command.""" 169 170 def run(self): 171 self.run_command('build_project_metadata') 172 build_py.build_py.run(self) 173 174 175def _poison_extensions(extensions, message): 176 """Includes a file that will always fail to compile in all extensions.""" 177 poison_filename = os.path.join(PYTHON_STEM, 'poison.c') 178 with open(poison_filename, 'w') as poison: 179 poison.write('#error {}'.format(message)) 180 for extension in extensions: 181 extension.sources = [poison_filename] 182 183 184def check_and_update_cythonization(extensions): 185 """Replace .pyx files with their generated counterparts and return whether or 186 not cythonization still needs to occur.""" 187 for extension in extensions: 188 generated_pyx_sources = [] 189 other_sources = [] 190 for source in extension.sources: 191 base, file_ext = os.path.splitext(source) 192 if file_ext == '.pyx': 193 generated_pyx_source = next( 194 (base + gen_ext for gen_ext in ( 195 '.c', 196 '.cpp', 197 ) if os.path.isfile(base + gen_ext)), None) 198 if generated_pyx_source: 199 generated_pyx_sources.append(generated_pyx_source) 200 else: 201 sys.stderr.write('Cython-generated files are missing...\n') 202 return False 203 else: 204 other_sources.append(source) 205 extension.sources = generated_pyx_sources + other_sources 206 sys.stderr.write('Found cython-generated files...\n') 207 return True 208 209 210def try_cythonize(extensions, linetracing=False, mandatory=True): 211 """Attempt to cythonize the extensions. 212 213 Args: 214 extensions: A list of `distutils.extension.Extension`. 215 linetracing: A bool indicating whether or not to enable linetracing. 216 mandatory: Whether or not having Cython-generated files is mandatory. If it 217 is, extensions will be poisoned when they can't be fully generated. 218 """ 219 try: 220 # Break import style to ensure we have access to Cython post-setup_requires 221 import Cython.Build 222 except ImportError: 223 if mandatory: 224 sys.stderr.write( 225 "This package needs to generate C files with Cython but it cannot. " 226 "Poisoning extension sources to disallow extension commands...") 227 _poison_extensions( 228 extensions, 229 "Extensions have been poisoned due to missing Cython-generated code." 230 ) 231 return extensions 232 cython_compiler_directives = {} 233 if linetracing: 234 additional_define_macros = [('CYTHON_TRACE_NOGIL', '1')] 235 cython_compiler_directives['linetrace'] = True 236 return Cython.Build.cythonize( 237 extensions, 238 include_path=[ 239 include_dir 240 for extension in extensions 241 for include_dir in extension.include_dirs 242 ] + [CYTHON_STEM], 243 compiler_directives=cython_compiler_directives) 244 245 246class BuildExt(build_ext.build_ext): 247 """Custom build_ext command to enable compiler-specific flags.""" 248 249 C_OPTIONS = { 250 'unix': ('-pthread',), 251 'msvc': (), 252 } 253 LINK_OPTIONS = {} 254 255 def build_extensions(self): 256 if "darwin" in sys.platform: 257 config = os.environ.get('CONFIG', 'opt') 258 target_path = os.path.abspath( 259 os.path.join( 260 os.path.dirname(os.path.realpath(__file__)), '..', '..', 261 '..', 'libs', config)) 262 targets = [ 263 os.path.join(target_path, 'libboringssl.a'), 264 os.path.join(target_path, 'libares.a'), 265 os.path.join(target_path, 'libgpr.a'), 266 os.path.join(target_path, 'libgrpc.a') 267 ] 268 # Running make separately for Mac means we lose all 269 # Extension.define_macros configured in setup.py. Re-add the macro 270 # for gRPC Core's fork handlers. 271 # TODO(ericgribkoff) Decide what to do about the other missing core 272 # macros, including GRPC_ENABLE_FORK_SUPPORT, which defaults to 1 273 # on Linux but remains unset on Mac. 274 extra_defines = [ 275 'EXTRA_DEFINES="GRPC_POSIX_FORK_ALLOW_PTHREAD_ATFORK=1"' 276 ] 277 make_process = subprocess.Popen( 278 ['make'] + extra_defines + targets, 279 stdout=subprocess.PIPE, 280 stderr=subprocess.PIPE) 281 make_out, make_err = make_process.communicate() 282 if make_out and make_process.returncode != 0: 283 sys.stdout.write(str(make_out) + '\n') 284 if make_err: 285 sys.stderr.write(str(make_err) + '\n') 286 if make_process.returncode != 0: 287 raise Exception("make command failed!") 288 289 compiler = self.compiler.compiler_type 290 if compiler in BuildExt.C_OPTIONS: 291 for extension in self.extensions: 292 extension.extra_compile_args += list( 293 BuildExt.C_OPTIONS[compiler]) 294 if compiler in BuildExt.LINK_OPTIONS: 295 for extension in self.extensions: 296 extension.extra_link_args += list( 297 BuildExt.LINK_OPTIONS[compiler]) 298 if not check_and_update_cythonization(self.extensions): 299 self.extensions = try_cythonize(self.extensions) 300 try: 301 build_ext.build_ext.build_extensions(self) 302 except Exception as error: 303 formatted_exception = traceback.format_exc() 304 support.diagnose_build_ext_error(self, error, formatted_exception) 305 raise CommandError( 306 "Failed `build_ext` step:\n{}".format(formatted_exception)) 307 308 309class Gather(setuptools.Command): 310 """Command to gather project dependencies.""" 311 312 description = 'gather dependencies for grpcio' 313 user_options = [('test', 't', 314 'flag indicating to gather test dependencies'), 315 ('install', 'i', 316 'flag indicating to gather install dependencies')] 317 318 def initialize_options(self): 319 self.test = False 320 self.install = False 321 322 def finalize_options(self): 323 # distutils requires this override. 324 pass 325 326 def run(self): 327 if self.install and self.distribution.install_requires: 328 self.distribution.fetch_build_eggs( 329 self.distribution.install_requires) 330 if self.test and self.distribution.tests_require: 331 self.distribution.fetch_build_eggs(self.distribution.tests_require) 332