# Copyright 2015 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Provides distutils command classes for the GRPC Python setup process.""" from __future__ import print_function import distutils import glob import os import os.path import platform import re import shutil import subprocess import sys import traceback import setuptools from setuptools.command import build_ext from setuptools.command import build_py from setuptools.command import easy_install from setuptools.command import install from setuptools.command import test import support PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) GRPC_STEM = os.path.abspath(PYTHON_STEM + '../../../../') PROTO_STEM = os.path.join(GRPC_STEM, 'src', 'proto') PROTO_GEN_STEM = os.path.join(GRPC_STEM, 'src', 'python', 'gens') CYTHON_STEM = os.path.join(PYTHON_STEM, 'grpc', '_cython') class CommandError(Exception): """Simple exception class for GRPC custom commands.""" # TODO(atash): Remove this once PyPI has better Linux bdist support. See # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported def _get_grpc_custom_bdist(decorated_basename, target_bdist_basename): """Returns a string path to a bdist file for Linux to install. If we can retrieve a pre-compiled bdist from online, uses it. Else, emits a warning and builds from source. """ # TODO(atash): somehow the name that's returned from `wheel` is different # between different versions of 'wheel' (but from a compatibility standpoint, # the names are compatible); we should have some way of determining name # compatibility in the same way `wheel` does to avoid having to rename all of # the custom wheels that we build/upload to GCS. # Break import style to ensure that setup.py has had a chance to install the # relevant package. from six.moves.urllib import request decorated_path = decorated_basename + GRPC_CUSTOM_BDIST_EXT try: url = BINARIES_REPOSITORY + '/{target}'.format(target=decorated_path) bdist_data = request.urlopen(url).read() except IOError as error: raise CommandError('{}\n\nCould not find the bdist {}: {}'.format( traceback.format_exc(), decorated_path, error.message)) # Our chosen local bdist path. bdist_path = target_bdist_basename + GRPC_CUSTOM_BDIST_EXT try: with open(bdist_path, 'w') as bdist_file: bdist_file.write(bdist_data) except IOError as error: raise CommandError('{}\n\nCould not write grpcio bdist: {}'.format( traceback.format_exc(), error.message)) return bdist_path class SphinxDocumentation(setuptools.Command): """Command to generate documentation via sphinx.""" description = 'generate sphinx documentation' user_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(self): # We import here to ensure that setup.py has had a chance to install the # relevant package eggs first. import sphinx.cmd.build source_dir = os.path.join(GRPC_STEM, 'doc', 'python', 'sphinx') target_dir = os.path.join(GRPC_STEM, 'doc', 'build') exit_code = sphinx.cmd.build.build_main( ['-b', 'html', '-W', '--keep-going', source_dir, target_dir]) if exit_code != 0: raise CommandError( "Documentation generation has warnings or errors") class BuildProjectMetadata(setuptools.Command): """Command to generate project metadata in a module.""" description = 'build grpcio project metadata files' user_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(self): with open(os.path.join(PYTHON_STEM, 'grpc/_grpcio_metadata.py'), 'w') as module_file: module_file.write('__version__ = """{}"""'.format( self.distribution.get_version())) class BuildPy(build_py.build_py): """Custom project build command.""" def run(self): self.run_command('build_project_metadata') build_py.build_py.run(self) def _poison_extensions(extensions, message): """Includes a file that will always fail to compile in all extensions.""" poison_filename = os.path.join(PYTHON_STEM, 'poison.c') with open(poison_filename, 'w') as poison: poison.write('#error {}'.format(message)) for extension in extensions: extension.sources = [poison_filename] def check_and_update_cythonization(extensions): """Replace .pyx files with their generated counterparts and return whether or not cythonization still needs to occur.""" for extension in extensions: generated_pyx_sources = [] other_sources = [] for source in extension.sources: base, file_ext = os.path.splitext(source) if file_ext == '.pyx': generated_pyx_source = next((base + gen_ext for gen_ext in ( '.c', '.cpp', ) if os.path.isfile(base + gen_ext)), None) if generated_pyx_source: generated_pyx_sources.append(generated_pyx_source) else: sys.stderr.write('Cython-generated files are missing...\n') return False else: other_sources.append(source) extension.sources = generated_pyx_sources + other_sources sys.stderr.write('Found cython-generated files...\n') return True def try_cythonize(extensions, linetracing=False, mandatory=True): """Attempt to cythonize the extensions. Args: extensions: A list of `distutils.extension.Extension`. linetracing: A bool indicating whether or not to enable linetracing. mandatory: Whether or not having Cython-generated files is mandatory. If it is, extensions will be poisoned when they can't be fully generated. """ try: # Break import style to ensure we have access to Cython post-setup_requires import Cython.Build except ImportError: if mandatory: sys.stderr.write( "This package needs to generate C files with Cython but it cannot. " "Poisoning extension sources to disallow extension commands...") _poison_extensions( extensions, "Extensions have been poisoned due to missing Cython-generated code." ) return extensions cython_compiler_directives = {} if linetracing: additional_define_macros = [('CYTHON_TRACE_NOGIL', '1')] cython_compiler_directives['linetrace'] = True return Cython.Build.cythonize( extensions, include_path=[ include_dir for extension in extensions for include_dir in extension.include_dirs ] + [CYTHON_STEM], compiler_directives=cython_compiler_directives) class BuildExt(build_ext.build_ext): """Custom build_ext command to enable compiler-specific flags.""" C_OPTIONS = { 'unix': ('-pthread',), 'msvc': (), } LINK_OPTIONS = {} def build_extensions(self): def compiler_ok_with_extra_std(): """Test if default compiler is okay with specifying c++ version when invoked in C mode. GCC is okay with this, while clang is not. """ try: # TODO(lidiz) Remove the generated a.out for success tests. cc_test = subprocess.Popen(['cc', '-x', 'c', '-std=c++11', '-'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, cc_err = cc_test.communicate(input=b'int main(){return 0;}') return not 'invalid argument' in str(cc_err) except: sys.stderr.write('Non-fatal exception:' + traceback.format_exc() + '\n') return False # This special conditioning is here due to difference of compiler # behavior in gcc and clang. The clang doesn't take --stdc++11 # flags but gcc does. Since the setuptools of Python only support # all C or all C++ compilation, the mix of C and C++ will crash. # *By default*, macOS and FreBSD use clang and Linux use gcc # # If we are not using a permissive compiler that's OK with being # passed wrong std flags, swap out compile function by adding a filter # for it. if not compiler_ok_with_extra_std(): old_compile = self.compiler._compile def new_compile(obj, src, ext, cc_args, extra_postargs, pp_opts): if src[-2:] == '.c': extra_postargs = [ arg for arg in extra_postargs if not '-std=c++' in arg ] return old_compile(obj, src, ext, cc_args, extra_postargs, pp_opts) self.compiler._compile = new_compile compiler = self.compiler.compiler_type if compiler in BuildExt.C_OPTIONS: for extension in self.extensions: extension.extra_compile_args += list( BuildExt.C_OPTIONS[compiler]) if compiler in BuildExt.LINK_OPTIONS: for extension in self.extensions: extension.extra_link_args += list( BuildExt.LINK_OPTIONS[compiler]) if not check_and_update_cythonization(self.extensions): self.extensions = try_cythonize(self.extensions) try: build_ext.build_ext.build_extensions(self) except Exception as error: formatted_exception = traceback.format_exc() support.diagnose_build_ext_error(self, error, formatted_exception) raise CommandError( "Failed `build_ext` step:\n{}".format(formatted_exception)) class Gather(setuptools.Command): """Command to gather project dependencies.""" description = 'gather dependencies for grpcio' user_options = [ ('test', 't', 'flag indicating to gather test dependencies'), ('install', 'i', 'flag indicating to gather install dependencies') ] def initialize_options(self): self.test = False self.install = False def finalize_options(self): # distutils requires this override. pass def run(self): if self.install and self.distribution.install_requires: self.distribution.fetch_build_eggs( self.distribution.install_requires) if self.test and self.distribution.tests_require: self.distribution.fetch_build_eggs(self.distribution.tests_require) class Clean(setuptools.Command): """Command to clean build artifacts.""" description = 'Clean build artifacts.' user_options = [ ('all', 'a', 'a phony flag to allow our script to continue'), ] _FILE_PATTERNS = ( 'python_build', 'src/python/grpcio/__pycache__/', 'src/python/grpcio/grpc/_cython/cygrpc.cpp', 'src/python/grpcio/grpc/_cython/*.so', 'src/python/grpcio/grpcio.egg-info/', ) _CURRENT_DIRECTORY = os.path.normpath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../..")) def initialize_options(self): self.all = False def finalize_options(self): pass def run(self): for path_spec in self._FILE_PATTERNS: this_glob = os.path.normpath( os.path.join(Clean._CURRENT_DIRECTORY, path_spec)) abs_paths = glob.glob(this_glob) for path in abs_paths: if not str(path).startswith(Clean._CURRENT_DIRECTORY): raise ValueError( "Cowardly refusing to delete {}.".format(path)) print("Removing {}".format(os.path.relpath(path))) if os.path.isfile(path): os.remove(str(path)) else: shutil.rmtree(str(path))