1#!/usr/bin/env python 2# Copyright 2016 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Definition of targets to build artifacts.""" 16 17import os.path 18import random 19import string 20import sys 21 22sys.path.insert(0, os.path.abspath('..')) 23import python_utils.jobset as jobset 24 25 26def create_docker_jobspec(name, 27 dockerfile_dir, 28 shell_command, 29 environ={}, 30 flake_retries=0, 31 timeout_retries=0, 32 timeout_seconds=30 * 60, 33 docker_base_image=None, 34 extra_docker_args=None, 35 verbose_success=False): 36 """Creates jobspec for a task running under docker.""" 37 environ = environ.copy() 38 environ['RUN_COMMAND'] = shell_command 39 environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name 40 41 docker_args = [] 42 for k, v in environ.items(): 43 docker_args += ['-e', '%s=%s' % (k, v)] 44 docker_env = { 45 'DOCKERFILE_DIR': dockerfile_dir, 46 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', 47 'OUTPUT_DIR': 'artifacts' 48 } 49 50 if docker_base_image is not None: 51 docker_env['DOCKER_BASE_IMAGE'] = docker_base_image 52 if extra_docker_args is not None: 53 docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args 54 jobspec = jobset.JobSpec( 55 cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + 56 docker_args, 57 environ=docker_env, 58 shortname='build_artifact.%s' % (name), 59 timeout_seconds=timeout_seconds, 60 flake_retries=flake_retries, 61 timeout_retries=timeout_retries, 62 verbose_success=verbose_success) 63 return jobspec 64 65 66def create_jobspec(name, 67 cmdline, 68 environ={}, 69 shell=False, 70 flake_retries=0, 71 timeout_retries=0, 72 timeout_seconds=30 * 60, 73 use_workspace=False, 74 cpu_cost=1.0, 75 verbose_success=False): 76 """Creates jobspec.""" 77 environ = environ.copy() 78 if use_workspace: 79 environ['WORKSPACE_NAME'] = 'workspace_%s' % name 80 environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name) 81 cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh' 82 ] + cmdline 83 else: 84 environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name) 85 86 jobspec = jobset.JobSpec( 87 cmdline=cmdline, 88 environ=environ, 89 shortname='build_artifact.%s' % (name), 90 timeout_seconds=timeout_seconds, 91 flake_retries=flake_retries, 92 timeout_retries=timeout_retries, 93 shell=shell, 94 cpu_cost=cpu_cost, 95 verbose_success=verbose_success) 96 return jobspec 97 98 99_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' 100 101_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'} 102 103 104class PythonArtifact: 105 """Builds Python artifacts.""" 106 107 def __init__(self, platform, arch, py_version): 108 self.name = 'python_%s_%s_%s' % (platform, arch, py_version) 109 self.platform = platform 110 self.arch = arch 111 self.labels = ['artifact', 'python', platform, arch, py_version] 112 self.py_version = py_version 113 114 def pre_build_jobspecs(self): 115 return [] 116 117 def build_jobspec(self): 118 environ = {} 119 if self.platform == 'linux_extra': 120 # Raspberry Pi build 121 environ['PYTHON'] = '/usr/local/bin/python{}'.format( 122 self.py_version) 123 environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version) 124 # https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9 125 # A QEMU bug causes submodule update to hang, so we copy directly 126 environ['RELATIVE_COPY_PATH'] = '.' 127 extra_args = ' --entrypoint=/usr/bin/qemu-arm-static ' 128 return create_docker_jobspec( 129 self.name, 130 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), 131 'tools/run_tests/artifacts/build_artifact_python.sh', 132 environ=environ, 133 timeout_seconds=60 * 60 * 5, 134 docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch), 135 extra_docker_args=extra_args) 136 elif self.platform == 'linux': 137 if self.arch == 'x86': 138 environ['SETARCH_CMD'] = 'linux32' 139 # Inside the manylinux container, the python installations are located in 140 # special places... 141 environ['PYTHON'] = '/opt/python/{}/bin/python'.format( 142 self.py_version) 143 environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) 144 # Platform autodetection for the manylinux1 image breaks so we set the 145 # defines ourselves. 146 # TODO(atash) get better platform-detection support in core so we don't 147 # need to do this manually... 148 environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' 149 environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' 150 environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' 151 return create_docker_jobspec( 152 self.name, 153 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % 154 self.arch, 155 'tools/run_tests/artifacts/build_artifact_python.sh', 156 environ=environ, 157 timeout_seconds=60 * 60, 158 docker_base_image='quay.io/pypa/manylinux1_i686' 159 if self.arch == 'x86' else 'quay.io/pypa/manylinux1_x86_64') 160 elif self.platform == 'windows': 161 if 'Python27' in self.py_version or 'Python34' in self.py_version: 162 environ['EXT_COMPILER'] = 'mingw32' 163 else: 164 environ['EXT_COMPILER'] = 'msvc' 165 # For some reason, the batch script %random% always runs with the same 166 # seed. We create a random temp-dir here 167 dir = ''.join( 168 random.choice(string.ascii_uppercase) for _ in range(10)) 169 return create_jobspec( 170 self.name, [ 171 'tools\\run_tests\\artifacts\\build_artifact_python.bat', 172 self.py_version, '32' if self.arch == 'x86' else '64' 173 ], 174 environ=environ, 175 timeout_seconds=45 * 60, 176 use_workspace=True) 177 else: 178 environ['PYTHON'] = self.py_version 179 environ['SKIP_PIP_INSTALL'] = 'TRUE' 180 return create_jobspec( 181 self.name, 182 ['tools/run_tests/artifacts/build_artifact_python.sh'], 183 environ=environ, 184 timeout_seconds=60 * 60 * 2, 185 use_workspace=True) 186 187 def __str__(self): 188 return self.name 189 190 191class RubyArtifact: 192 """Builds ruby native gem.""" 193 194 def __init__(self, platform, arch): 195 self.name = 'ruby_native_gem_%s_%s' % (platform, arch) 196 self.platform = platform 197 self.arch = arch 198 self.labels = ['artifact', 'ruby', platform, arch] 199 200 def pre_build_jobspecs(self): 201 return [] 202 203 def build_jobspec(self): 204 # Ruby build uses docker internally and docker cannot be nested. 205 # We are using a custom workspace instead. 206 return create_jobspec( 207 self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'], 208 use_workspace=True, 209 timeout_seconds=45 * 60) 210 211 212class CSharpExtArtifact: 213 """Builds C# native extension library""" 214 215 def __init__(self, platform, arch, arch_abi=None): 216 self.name = 'csharp_ext_%s_%s' % (platform, arch) 217 self.platform = platform 218 self.arch = arch 219 self.arch_abi = arch_abi 220 self.labels = ['artifact', 'csharp', platform, arch] 221 if arch_abi: 222 self.name += '_%s' % arch_abi 223 self.labels.append(arch_abi) 224 225 def pre_build_jobspecs(self): 226 return [] 227 228 def build_jobspec(self): 229 if self.arch == 'android': 230 return create_docker_jobspec( 231 self.name, 232 'tools/dockerfile/grpc_artifact_android_ndk', 233 'tools/run_tests/artifacts/build_artifact_csharp_android.sh', 234 environ={ 235 'ANDROID_ABI': self.arch_abi 236 }) 237 elif self.arch == 'ios': 238 return create_jobspec( 239 self.name, 240 ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], 241 use_workspace=True) 242 elif self.platform == 'windows': 243 cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch 244 return create_jobspec( 245 self.name, [ 246 'tools\\run_tests\\artifacts\\build_artifact_csharp.bat', 247 cmake_arch_option 248 ], 249 use_workspace=True) 250 else: 251 environ = { 252 'CONFIG': 'opt', 253 'EMBED_OPENSSL': 'true', 254 'EMBED_ZLIB': 'true', 255 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', 256 'CXXFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', 257 'LDFLAGS': '' 258 } 259 if self.platform == 'linux': 260 return create_docker_jobspec( 261 self.name, 262 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, 263 'tools/run_tests/artifacts/build_artifact_csharp.sh', 264 environ=environ) 265 else: 266 archflag = _ARCH_FLAG_MAP[self.arch] 267 environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) 268 environ['CXXFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) 269 environ['LDFLAGS'] += ' %s' % archflag 270 return create_jobspec( 271 self.name, 272 ['tools/run_tests/artifacts/build_artifact_csharp.sh'], 273 environ=environ, 274 use_workspace=True) 275 276 def __str__(self): 277 return self.name 278 279 280class PHPArtifact: 281 """Builds PHP PECL package""" 282 283 def __init__(self, platform, arch): 284 self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) 285 self.platform = platform 286 self.arch = arch 287 self.labels = ['artifact', 'php', platform, arch] 288 289 def pre_build_jobspecs(self): 290 return [] 291 292 def build_jobspec(self): 293 return create_docker_jobspec( 294 self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( 295 self.arch), 'tools/run_tests/artifacts/build_artifact_php.sh') 296 297 298class ProtocArtifact: 299 """Builds protoc and protoc-plugin artifacts""" 300 301 def __init__(self, platform, arch): 302 self.name = 'protoc_%s_%s' % (platform, arch) 303 self.platform = platform 304 self.arch = arch 305 self.labels = ['artifact', 'protoc', platform, arch] 306 307 def pre_build_jobspecs(self): 308 return [] 309 310 def build_jobspec(self): 311 if self.platform != 'windows': 312 cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] 313 ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] 314 if self.platform != 'macos': 315 ldflags += ' -static-libgcc -static-libstdc++ -s' 316 environ = { 317 'CONFIG': 'opt', 318 'CXXFLAGS': cxxflags, 319 'LDFLAGS': ldflags, 320 'PROTOBUF_LDFLAGS_EXTRA': ldflags 321 } 322 if self.platform == 'linux': 323 return create_docker_jobspec( 324 self.name, 325 'tools/dockerfile/grpc_artifact_protoc', 326 'tools/run_tests/artifacts/build_artifact_protoc.sh', 327 environ=environ) 328 else: 329 environ[ 330 'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG 331 return create_jobspec( 332 self.name, 333 ['tools/run_tests/artifacts/build_artifact_protoc.sh'], 334 environ=environ, 335 timeout_seconds=60 * 60, 336 use_workspace=True) 337 else: 338 generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015' 339 return create_jobspec( 340 self.name, 341 ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], 342 environ={'generator': generator}, 343 use_workspace=True) 344 345 def __str__(self): 346 return self.name 347 348 349def targets(): 350 """Gets list of supported targets""" 351 return ([ 352 Cls(platform, arch) 353 for Cls in (CSharpExtArtifact, ProtocArtifact) 354 for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64') 355 ] + [ 356 CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), 357 CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), 358 CSharpExtArtifact('linux', 'android', arch_abi='x86'), 359 CSharpExtArtifact('macos', 'ios'), 360 PythonArtifact('linux', 'x86', 'cp27-cp27m'), 361 PythonArtifact('linux', 'x86', 'cp27-cp27mu'), 362 PythonArtifact('linux', 'x86', 'cp34-cp34m'), 363 PythonArtifact('linux', 'x86', 'cp35-cp35m'), 364 PythonArtifact('linux', 'x86', 'cp36-cp36m'), 365 PythonArtifact('linux', 'x86', 'cp37-cp37m'), 366 PythonArtifact('linux_extra', 'armv7', '2.7'), 367 PythonArtifact('linux_extra', 'armv7', '3.4'), 368 PythonArtifact('linux_extra', 'armv7', '3.5'), 369 PythonArtifact('linux_extra', 'armv7', '3.6'), 370 PythonArtifact('linux_extra', 'armv6', '2.7'), 371 PythonArtifact('linux_extra', 'armv6', '3.4'), 372 PythonArtifact('linux_extra', 'armv6', '3.5'), 373 PythonArtifact('linux_extra', 'armv6', '3.6'), 374 PythonArtifact('linux', 'x64', 'cp27-cp27m'), 375 PythonArtifact('linux', 'x64', 'cp27-cp27mu'), 376 PythonArtifact('linux', 'x64', 'cp34-cp34m'), 377 PythonArtifact('linux', 'x64', 'cp35-cp35m'), 378 PythonArtifact('linux', 'x64', 'cp36-cp36m'), 379 PythonArtifact('linux', 'x64', 'cp37-cp37m'), 380 PythonArtifact('macos', 'x64', 'python2.7'), 381 PythonArtifact('macos', 'x64', 'python3.4'), 382 PythonArtifact('macos', 'x64', 'python3.5'), 383 PythonArtifact('macos', 'x64', 'python3.6'), 384 PythonArtifact('macos', 'x64', 'python3.7'), 385 PythonArtifact('windows', 'x86', 'Python27_32bits'), 386 PythonArtifact('windows', 'x86', 'Python34_32bits'), 387 PythonArtifact('windows', 'x86', 'Python35_32bits'), 388 PythonArtifact('windows', 'x86', 'Python36_32bits'), 389 PythonArtifact('windows', 'x86', 'Python37_32bits'), 390 PythonArtifact('windows', 'x64', 'Python27'), 391 PythonArtifact('windows', 'x64', 'Python34'), 392 PythonArtifact('windows', 'x64', 'Python35'), 393 PythonArtifact('windows', 'x64', 'Python36'), 394 PythonArtifact('windows', 'x64', 'Python37'), 395 RubyArtifact('linux', 'x64'), 396 RubyArtifact('macos', 'x64'), 397 PHPArtifact('linux', 'x64') 398 ]) 399