1#!/usr/bin/env python 2# Copyright 2016 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Definition of targets to build artifacts.""" 16 17import os.path 18import random 19import string 20import sys 21 22sys.path.insert(0, os.path.abspath('..')) 23import python_utils.jobset as jobset 24 25 26def create_docker_jobspec(name, 27 dockerfile_dir, 28 shell_command, 29 environ={}, 30 flake_retries=0, 31 timeout_retries=0, 32 timeout_seconds=30 * 60, 33 docker_base_image=None, 34 extra_docker_args=None, 35 verbose_success=False): 36 """Creates jobspec for a task running under docker.""" 37 environ = environ.copy() 38 environ['RUN_COMMAND'] = shell_command 39 environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name 40 41 docker_args = [] 42 for k, v in environ.items(): 43 docker_args += ['-e', '%s=%s' % (k, v)] 44 docker_env = { 45 'DOCKERFILE_DIR': dockerfile_dir, 46 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', 47 'OUTPUT_DIR': 'artifacts' 48 } 49 50 if docker_base_image is not None: 51 docker_env['DOCKER_BASE_IMAGE'] = docker_base_image 52 if extra_docker_args is not None: 53 docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args 54 jobspec = jobset.JobSpec( 55 cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + 56 docker_args, 57 environ=docker_env, 58 shortname='build_artifact.%s' % (name), 59 timeout_seconds=timeout_seconds, 60 flake_retries=flake_retries, 61 timeout_retries=timeout_retries, 62 verbose_success=verbose_success) 63 return jobspec 64 65 66def create_jobspec(name, 67 cmdline, 68 environ={}, 69 shell=False, 70 flake_retries=0, 71 timeout_retries=0, 72 timeout_seconds=30 * 60, 73 use_workspace=False, 74 cpu_cost=1.0, 75 verbose_success=False): 76 """Creates jobspec.""" 77 environ = environ.copy() 78 if use_workspace: 79 environ['WORKSPACE_NAME'] = 'workspace_%s' % name 80 environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name) 81 cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh' 82 ] + cmdline 83 else: 84 environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name) 85 86 jobspec = jobset.JobSpec(cmdline=cmdline, 87 environ=environ, 88 shortname='build_artifact.%s' % (name), 89 timeout_seconds=timeout_seconds, 90 flake_retries=flake_retries, 91 timeout_retries=timeout_retries, 92 shell=shell, 93 cpu_cost=cpu_cost, 94 verbose_success=verbose_success) 95 return jobspec 96 97 98_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' 99 100_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'} 101 102 103class PythonArtifact: 104 """Builds Python artifacts.""" 105 106 def __init__(self, platform, arch, py_version): 107 self.name = 'python_%s_%s_%s' % (platform, arch, py_version) 108 self.platform = platform 109 self.arch = arch 110 self.labels = ['artifact', 'python', platform, arch, py_version] 111 self.py_version = py_version 112 if 'manylinux' in platform: 113 self.labels.append('linux') 114 115 def pre_build_jobspecs(self): 116 return [] 117 118 def build_jobspec(self): 119 environ = {} 120 if self.platform == 'linux_extra': 121 # Raspberry Pi build 122 environ['PYTHON'] = '/usr/local/bin/python{}'.format( 123 self.py_version) 124 environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version) 125 # https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9 126 # A QEMU bug causes submodule update to hang, so we copy directly 127 environ['RELATIVE_COPY_PATH'] = '.' 128 # Parallel builds are counterproductive in emulated environment 129 environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'] = '1' 130 extra_args = ' --entrypoint=/usr/bin/qemu-arm-static ' 131 return create_docker_jobspec( 132 self.name, 133 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), 134 'tools/run_tests/artifacts/build_artifact_python.sh', 135 environ=environ, 136 timeout_seconds=60 * 60 * 5, 137 docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch), 138 extra_docker_args=extra_args) 139 elif 'manylinux' in self.platform: 140 if self.arch == 'x86': 141 environ['SETARCH_CMD'] = 'linux32' 142 # Inside the manylinux container, the python installations are located in 143 # special places... 144 environ['PYTHON'] = '/opt/python/{}/bin/python'.format( 145 self.py_version) 146 environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) 147 environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' 148 environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' 149 return create_docker_jobspec( 150 self.name, 151 # NOTE(rbellevi): Do *not* update this without also ensuring the 152 # base_docker_image attribute is accurate. 153 'tools/dockerfile/grpc_artifact_python_%s_%s' % 154 (self.platform, self.arch), 155 'tools/run_tests/artifacts/build_artifact_python.sh', 156 environ=environ, 157 timeout_seconds=60 * 60) 158 elif self.platform == 'windows': 159 if 'Python27' in self.py_version: 160 environ['EXT_COMPILER'] = 'mingw32' 161 else: 162 environ['EXT_COMPILER'] = 'msvc' 163 # For some reason, the batch script %random% always runs with the same 164 # seed. We create a random temp-dir here 165 dir = ''.join( 166 random.choice(string.ascii_uppercase) for _ in range(10)) 167 return create_jobspec(self.name, [ 168 'tools\\run_tests\\artifacts\\build_artifact_python.bat', 169 self.py_version, '32' if self.arch == 'x86' else '64' 170 ], 171 environ=environ, 172 timeout_seconds=45 * 60, 173 use_workspace=True) 174 else: 175 environ['PYTHON'] = self.py_version 176 environ['SKIP_PIP_INSTALL'] = 'TRUE' 177 return create_jobspec( 178 self.name, 179 ['tools/run_tests/artifacts/build_artifact_python.sh'], 180 environ=environ, 181 timeout_seconds=60 * 60 * 2, 182 use_workspace=True) 183 184 def __str__(self): 185 return self.name 186 187 188class RubyArtifact: 189 """Builds ruby native gem.""" 190 191 def __init__(self, platform, arch): 192 self.name = 'ruby_native_gem_%s_%s' % (platform, arch) 193 self.platform = platform 194 self.arch = arch 195 self.labels = ['artifact', 'ruby', platform, arch] 196 197 def pre_build_jobspecs(self): 198 return [] 199 200 def build_jobspec(self): 201 # Ruby build uses docker internally and docker cannot be nested. 202 # We are using a custom workspace instead. 203 return create_jobspec( 204 self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'], 205 use_workspace=True, 206 timeout_seconds=45 * 60) 207 208 209class CSharpExtArtifact: 210 """Builds C# native extension library""" 211 212 def __init__(self, platform, arch, arch_abi=None): 213 self.name = 'csharp_ext_%s_%s' % (platform, arch) 214 self.platform = platform 215 self.arch = arch 216 self.arch_abi = arch_abi 217 self.labels = ['artifact', 'csharp', platform, arch] 218 if arch_abi: 219 self.name += '_%s' % arch_abi 220 self.labels.append(arch_abi) 221 222 def pre_build_jobspecs(self): 223 return [] 224 225 def build_jobspec(self): 226 if self.arch == 'android': 227 return create_docker_jobspec( 228 self.name, 229 'tools/dockerfile/grpc_artifact_android_ndk', 230 'tools/run_tests/artifacts/build_artifact_csharp_android.sh', 231 environ={'ANDROID_ABI': self.arch_abi}) 232 elif self.arch == 'ios': 233 return create_jobspec( 234 self.name, 235 ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], 236 use_workspace=True) 237 elif self.platform == 'windows': 238 return create_jobspec(self.name, [ 239 'tools\\run_tests\\artifacts\\build_artifact_csharp.bat', 240 self.arch 241 ], 242 use_workspace=True) 243 else: 244 if self.platform == 'linux': 245 cmake_arch_option = '' # x64 is the default architecture 246 if self.arch == 'x86': 247 # TODO(jtattermusch): more work needed to enable 248 # boringssl assembly optimizations for 32-bit linux. 249 # Problem: currently we are building the artifact under 250 # 32-bit docker image, but CMAKE_SYSTEM_PROCESSOR is still 251 # set to x86_64, so the resulting boringssl binary 252 # would have undefined symbols. 253 cmake_arch_option = '-DOPENSSL_NO_ASM=ON' 254 return create_docker_jobspec( 255 self.name, 256 'tools/dockerfile/grpc_artifact_centos6_{}'.format( 257 self.arch), 258 'tools/run_tests/artifacts/build_artifact_csharp.sh', 259 environ={'CMAKE_ARCH_OPTION': cmake_arch_option}) 260 else: 261 cmake_arch_option = '' # x64 is the default architecture 262 if self.arch == 'x86': 263 cmake_arch_option = '-DCMAKE_OSX_ARCHITECTURES=i386' 264 return create_jobspec( 265 self.name, 266 ['tools/run_tests/artifacts/build_artifact_csharp.sh'], 267 environ={'CMAKE_ARCH_OPTION': cmake_arch_option}, 268 use_workspace=True) 269 270 def __str__(self): 271 return self.name 272 273 274class PHPArtifact: 275 """Builds PHP PECL package""" 276 277 def __init__(self, platform, arch): 278 self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) 279 self.platform = platform 280 self.arch = arch 281 self.labels = ['artifact', 'php', platform, arch] 282 283 def pre_build_jobspecs(self): 284 return [] 285 286 def build_jobspec(self): 287 return create_docker_jobspec( 288 self.name, 289 'tools/dockerfile/test/php73_zts_stretch_{}'.format(self.arch), 290 'tools/run_tests/artifacts/build_artifact_php.sh') 291 292 293class ProtocArtifact: 294 """Builds protoc and protoc-plugin artifacts""" 295 296 def __init__(self, platform, arch): 297 self.name = 'protoc_%s_%s' % (platform, arch) 298 self.platform = platform 299 self.arch = arch 300 self.labels = ['artifact', 'protoc', platform, arch] 301 302 def pre_build_jobspecs(self): 303 return [] 304 305 def build_jobspec(self): 306 if self.platform != 'windows': 307 cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] 308 ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] 309 if self.platform != 'macos': 310 ldflags += ' -static-libgcc -static-libstdc++ -s' 311 environ = { 312 'CONFIG': 'opt', 313 'CXXFLAGS': cxxflags, 314 'LDFLAGS': ldflags, 315 'PROTOBUF_LDFLAGS_EXTRA': ldflags 316 } 317 if self.platform == 'linux': 318 return create_docker_jobspec( 319 self.name, 320 'tools/dockerfile/grpc_artifact_centos6_{}'.format( 321 self.arch), 322 'tools/run_tests/artifacts/build_artifact_protoc.sh', 323 environ=environ) 324 else: 325 environ[ 326 'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG 327 return create_jobspec( 328 self.name, 329 ['tools/run_tests/artifacts/build_artifact_protoc.sh'], 330 environ=environ, 331 timeout_seconds=60 * 60, 332 use_workspace=True) 333 else: 334 generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015' 335 return create_jobspec( 336 self.name, 337 ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], 338 environ={'generator': generator}, 339 use_workspace=True) 340 341 def __str__(self): 342 return self.name 343 344 345def targets(): 346 """Gets list of supported targets""" 347 return ([ 348 Cls(platform, arch) for Cls in (CSharpExtArtifact, ProtocArtifact) 349 for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64') 350 ] + [ 351 CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), 352 CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), 353 CSharpExtArtifact('linux', 'android', arch_abi='x86'), 354 CSharpExtArtifact('macos', 'ios'), 355 PythonArtifact('manylinux2014', 'x64', 'cp35-cp35m'), 356 PythonArtifact('manylinux2014', 'x64', 'cp36-cp36m'), 357 PythonArtifact('manylinux2014', 'x64', 'cp37-cp37m'), 358 PythonArtifact('manylinux2014', 'x64', 'cp38-cp38'), 359 PythonArtifact('manylinux2014', 'x86', 'cp35-cp35m'), 360 PythonArtifact('manylinux2014', 'x86', 'cp36-cp36m'), 361 PythonArtifact('manylinux2014', 'x86', 'cp37-cp37m'), 362 PythonArtifact('manylinux2014', 'x86', 'cp38-cp38'), 363 PythonArtifact('manylinux2010', 'x64', 'cp27-cp27m'), 364 PythonArtifact('manylinux2010', 'x64', 'cp27-cp27mu'), 365 PythonArtifact('manylinux2010', 'x64', 'cp35-cp35m'), 366 PythonArtifact('manylinux2010', 'x64', 'cp36-cp36m'), 367 PythonArtifact('manylinux2010', 'x64', 'cp37-cp37m'), 368 PythonArtifact('manylinux2010', 'x64', 'cp38-cp38'), 369 PythonArtifact('manylinux2010', 'x86', 'cp27-cp27m'), 370 PythonArtifact('manylinux2010', 'x86', 'cp27-cp27mu'), 371 PythonArtifact('manylinux2010', 'x86', 'cp35-cp35m'), 372 PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'), 373 PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'), 374 PythonArtifact('manylinux2010', 'x86', 'cp38-cp38'), 375 PythonArtifact('linux_extra', 'armv7', '2.7'), 376 PythonArtifact('linux_extra', 'armv7', '3.5'), 377 PythonArtifact('linux_extra', 'armv7', '3.6'), 378 PythonArtifact('linux_extra', 'armv6', '2.7'), 379 PythonArtifact('linux_extra', 'armv6', '3.5'), 380 PythonArtifact('linux_extra', 'armv6', '3.6'), 381 PythonArtifact('macos', 'x64', 'python2.7'), 382 PythonArtifact('macos', 'x64', 'python3.5'), 383 PythonArtifact('macos', 'x64', 'python3.6'), 384 PythonArtifact('macos', 'x64', 'python3.7'), 385 PythonArtifact('macos', 'x64', 'python3.8'), 386 PythonArtifact('windows', 'x86', 'Python27_32bit'), 387 PythonArtifact('windows', 'x86', 'Python35_32bit'), 388 PythonArtifact('windows', 'x86', 'Python36_32bit'), 389 PythonArtifact('windows', 'x86', 'Python37_32bit'), 390 PythonArtifact('windows', 'x86', 'Python38_32bit'), 391 PythonArtifact('windows', 'x64', 'Python27'), 392 PythonArtifact('windows', 'x64', 'Python35'), 393 PythonArtifact('windows', 'x64', 'Python36'), 394 PythonArtifact('windows', 'x64', 'Python37'), 395 PythonArtifact('windows', 'x64', 'Python38'), 396 RubyArtifact('linux', 'x64'), 397 RubyArtifact('macos', 'x64'), 398 PHPArtifact('linux', 'x64') 399 ]) 400