1#!/usr/bin/env python 2# Copyright 2016 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Definition of targets to build artifacts.""" 16 17import os.path 18import random 19import string 20import sys 21 22sys.path.insert(0, os.path.abspath('..')) 23import python_utils.jobset as jobset 24 25 26def create_docker_jobspec(name, 27 dockerfile_dir, 28 shell_command, 29 environ={}, 30 flake_retries=0, 31 timeout_retries=0, 32 timeout_seconds=30 * 60, 33 extra_docker_args=None, 34 verbose_success=False): 35 """Creates jobspec for a task running under docker.""" 36 environ = environ.copy() 37 environ['RUN_COMMAND'] = shell_command 38 environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name 39 40 docker_args = [] 41 for k, v in environ.items(): 42 docker_args += ['-e', '%s=%s' % (k, v)] 43 docker_env = { 44 'DOCKERFILE_DIR': dockerfile_dir, 45 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', 46 'OUTPUT_DIR': 'artifacts' 47 } 48 if extra_docker_args is not None: 49 docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args 50 jobspec = jobset.JobSpec( 51 cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + 52 docker_args, 53 environ=docker_env, 54 shortname='build_artifact.%s' % (name), 55 timeout_seconds=timeout_seconds, 56 flake_retries=flake_retries, 57 timeout_retries=timeout_retries, 58 verbose_success=verbose_success) 59 return jobspec 60 61 62def create_jobspec(name, 63 cmdline, 64 environ={}, 65 shell=False, 66 flake_retries=0, 67 timeout_retries=0, 68 timeout_seconds=30 * 60, 69 use_workspace=False, 70 cpu_cost=1.0, 71 verbose_success=False): 72 """Creates jobspec.""" 73 environ = environ.copy() 74 if use_workspace: 75 environ['WORKSPACE_NAME'] = 'workspace_%s' % name 76 environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name) 77 cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh' 78 ] + cmdline 79 else: 80 environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name) 81 82 jobspec = jobset.JobSpec(cmdline=cmdline, 83 environ=environ, 84 shortname='build_artifact.%s' % (name), 85 timeout_seconds=timeout_seconds, 86 flake_retries=flake_retries, 87 timeout_retries=timeout_retries, 88 shell=shell, 89 cpu_cost=cpu_cost, 90 verbose_success=verbose_success) 91 return jobspec 92 93 94_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.10' 95 96_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'} 97 98 99class PythonArtifact: 100 """Builds Python artifacts.""" 101 102 def __init__(self, platform, arch, py_version): 103 self.name = 'python_%s_%s_%s' % (platform, arch, py_version) 104 self.platform = platform 105 self.arch = arch 106 self.labels = ['artifact', 'python', platform, arch, py_version] 107 self.py_version = py_version 108 if 'manylinux' in platform: 109 self.labels.append('linux') 110 111 def pre_build_jobspecs(self): 112 return [] 113 114 def build_jobspec(self): 115 environ = {} 116 if self.platform == 'linux_extra': 117 # Crosscompilation build for armv7 (e.g. Raspberry Pi) 118 environ['PYTHON'] = '/opt/python/{}/bin/python3'.format( 119 self.py_version) 120 environ['PIP'] = '/opt/python/{}/bin/pip3'.format(self.py_version) 121 environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE' 122 environ['GRPC_SKIP_TWINE_CHECK'] = 'TRUE' 123 # when crosscompiling, we need to force statically linking libstdc++ 124 # otherwise libstdc++ symbols would be too new and the resulting 125 # wheel wouldn't pass the auditwheel check. 126 # This is needed because C core won't build with GCC 4.8 that's 127 # included in the default dockcross toolchain and we needed 128 # to opt into using a slighly newer version of GCC. 129 environ['GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX'] = 'TRUE' 130 131 return create_docker_jobspec( 132 self.name, 133 'tools/dockerfile/grpc_artifact_python_linux_{}'.format( 134 self.arch), 135 'tools/run_tests/artifacts/build_artifact_python.sh', 136 environ=environ, 137 timeout_seconds=60 * 60) 138 elif 'manylinux' in self.platform: 139 if self.arch == 'x86': 140 environ['SETARCH_CMD'] = 'linux32' 141 # Inside the manylinux container, the python installations are located in 142 # special places... 143 environ['PYTHON'] = '/opt/python/{}/bin/python'.format( 144 self.py_version) 145 environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) 146 environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE' 147 if self.arch == 'aarch64': 148 environ['GRPC_SKIP_TWINE_CHECK'] = 'TRUE' 149 # when crosscompiling, we need to force statically linking libstdc++ 150 # otherwise libstdc++ symbols would be too new and the resulting 151 # wheel wouldn't pass the auditwheel check. 152 # This is needed because C core won't build with GCC 4.8 that's 153 # included in the default dockcross toolchain and we needed 154 # to opt into using a slighly newer version of GCC. 155 environ['GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX'] = 'TRUE' 156 157 else: 158 # only run auditwheel if we're not crosscompiling 159 environ['GRPC_RUN_AUDITWHEEL_REPAIR'] = 'TRUE' 160 # only build the packages that depend on grpcio-tools 161 # if we're not crosscompiling. 162 # - they require protoc to run on current architecture 163 # - they only have sdist packages anyway, so it's useless to build them again 164 environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' 165 return create_docker_jobspec( 166 self.name, 167 'tools/dockerfile/grpc_artifact_python_%s_%s' % 168 (self.platform, self.arch), 169 'tools/run_tests/artifacts/build_artifact_python.sh', 170 environ=environ, 171 timeout_seconds=60 * 60 * 2) 172 elif self.platform == 'windows': 173 if 'Python27' in self.py_version: 174 environ['EXT_COMPILER'] = 'mingw32' 175 else: 176 environ['EXT_COMPILER'] = 'msvc' 177 # For some reason, the batch script %random% always runs with the same 178 # seed. We create a random temp-dir here 179 dir = ''.join( 180 random.choice(string.ascii_uppercase) for _ in range(10)) 181 return create_jobspec(self.name, [ 182 'tools\\run_tests\\artifacts\\build_artifact_python.bat', 183 self.py_version, '32' if self.arch == 'x86' else '64' 184 ], 185 environ=environ, 186 timeout_seconds=45 * 60, 187 use_workspace=True) 188 else: 189 environ['PYTHON'] = self.py_version 190 environ['SKIP_PIP_INSTALL'] = 'TRUE' 191 return create_jobspec( 192 self.name, 193 ['tools/run_tests/artifacts/build_artifact_python.sh'], 194 environ=environ, 195 timeout_seconds=60 * 60 * 2, 196 use_workspace=True) 197 198 def __str__(self): 199 return self.name 200 201 202class RubyArtifact: 203 """Builds ruby native gem.""" 204 205 def __init__(self, platform, arch): 206 self.name = 'ruby_native_gem_%s_%s' % (platform, arch) 207 self.platform = platform 208 self.arch = arch 209 self.labels = ['artifact', 'ruby', platform, arch] 210 211 def pre_build_jobspecs(self): 212 return [] 213 214 def build_jobspec(self): 215 # Ruby build uses docker internally and docker cannot be nested. 216 # We are using a custom workspace instead. 217 return create_jobspec( 218 self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'], 219 use_workspace=True, 220 timeout_seconds=60 * 60) 221 222 223class CSharpExtArtifact: 224 """Builds C# native extension library""" 225 226 def __init__(self, platform, arch, arch_abi=None): 227 self.name = 'csharp_ext_%s_%s' % (platform, arch) 228 self.platform = platform 229 self.arch = arch 230 self.arch_abi = arch_abi 231 self.labels = ['artifact', 'csharp', platform, arch] 232 if arch_abi: 233 self.name += '_%s' % arch_abi 234 self.labels.append(arch_abi) 235 236 def pre_build_jobspecs(self): 237 return [] 238 239 def build_jobspec(self): 240 if self.arch == 'android': 241 return create_docker_jobspec( 242 self.name, 243 'tools/dockerfile/grpc_artifact_android_ndk', 244 'tools/run_tests/artifacts/build_artifact_csharp_android.sh', 245 environ={'ANDROID_ABI': self.arch_abi}) 246 elif self.arch == 'ios': 247 return create_jobspec( 248 self.name, 249 ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], 250 timeout_seconds=60 * 60, 251 use_workspace=True) 252 elif self.platform == 'windows': 253 return create_jobspec(self.name, [ 254 'tools\\run_tests\\artifacts\\build_artifact_csharp.bat', 255 self.arch 256 ], 257 timeout_seconds=45 * 60, 258 use_workspace=True) 259 else: 260 if self.platform == 'linux': 261 dockerfile_dir = 'tools/dockerfile/grpc_artifact_centos6_{}'.format( 262 self.arch) 263 if self.arch == 'aarch64': 264 # for aarch64, use a dockcross manylinux image that will 265 # give us both ready to use crosscompiler and sufficient backward compatibility 266 dockerfile_dir = 'tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64' 267 return create_docker_jobspec( 268 self.name, dockerfile_dir, 269 'tools/run_tests/artifacts/build_artifact_csharp.sh') 270 else: 271 return create_jobspec( 272 self.name, 273 ['tools/run_tests/artifacts/build_artifact_csharp.sh'], 274 timeout_seconds=45 * 60, 275 use_workspace=True) 276 277 def __str__(self): 278 return self.name 279 280 281class PHPArtifact: 282 """Builds PHP PECL package""" 283 284 def __init__(self, platform, arch): 285 self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) 286 self.platform = platform 287 self.arch = arch 288 self.labels = ['artifact', 'php', platform, arch] 289 290 def pre_build_jobspecs(self): 291 return [] 292 293 def build_jobspec(self): 294 return create_docker_jobspec( 295 self.name, 296 'tools/dockerfile/test/php73_zts_stretch_{}'.format(self.arch), 297 'tools/run_tests/artifacts/build_artifact_php.sh') 298 299 300class ProtocArtifact: 301 """Builds protoc and protoc-plugin artifacts""" 302 303 def __init__(self, platform, arch): 304 self.name = 'protoc_%s_%s' % (platform, arch) 305 self.platform = platform 306 self.arch = arch 307 self.labels = ['artifact', 'protoc', platform, arch] 308 309 def pre_build_jobspecs(self): 310 return [] 311 312 def build_jobspec(self): 313 if self.platform != 'windows': 314 environ = {'CXXFLAGS': '', 'LDFLAGS': ''} 315 if self.platform == 'linux': 316 dockerfile_dir = 'tools/dockerfile/grpc_artifact_centos6_{}'.format( 317 self.arch) 318 if self.arch == 'aarch64': 319 # for aarch64, use a dockcross manylinux image that will 320 # give us both ready to use crosscompiler and sufficient backward compatibility 321 dockerfile_dir = 'tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64' 322 environ['LDFLAGS'] += ' -static-libgcc -static-libstdc++ -s' 323 return create_docker_jobspec( 324 self.name, 325 dockerfile_dir, 326 'tools/run_tests/artifacts/build_artifact_protoc.sh', 327 environ=environ) 328 else: 329 environ[ 330 'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG 331 return create_jobspec( 332 self.name, 333 ['tools/run_tests/artifacts/build_artifact_protoc.sh'], 334 environ=environ, 335 timeout_seconds=60 * 60, 336 use_workspace=True) 337 else: 338 generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015' 339 return create_jobspec( 340 self.name, 341 ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], 342 environ={'generator': generator}, 343 use_workspace=True) 344 345 def __str__(self): 346 return self.name 347 348 349def targets(): 350 """Gets list of supported targets""" 351 return [ 352 ProtocArtifact('linux', 'x64'), 353 ProtocArtifact('linux', 'x86'), 354 ProtocArtifact('linux', 'aarch64'), 355 ProtocArtifact('macos', 'x64'), 356 ProtocArtifact('windows', 'x64'), 357 ProtocArtifact('windows', 'x86'), 358 CSharpExtArtifact('linux', 'x64'), 359 CSharpExtArtifact('linux', 'aarch64'), 360 CSharpExtArtifact('macos', 'x64'), 361 CSharpExtArtifact('windows', 'x64'), 362 CSharpExtArtifact('windows', 'x86'), 363 CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), 364 CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), 365 CSharpExtArtifact('linux', 'android', arch_abi='x86'), 366 CSharpExtArtifact('macos', 'ios'), 367 PythonArtifact('manylinux2014', 'x64', 'cp35-cp35m'), 368 PythonArtifact('manylinux2014', 'x64', 'cp36-cp36m'), 369 PythonArtifact('manylinux2014', 'x64', 'cp37-cp37m'), 370 PythonArtifact('manylinux2014', 'x64', 'cp38-cp38'), 371 PythonArtifact('manylinux2014', 'x64', 'cp39-cp39'), 372 PythonArtifact('manylinux2014', 'x86', 'cp35-cp35m'), 373 PythonArtifact('manylinux2014', 'x86', 'cp36-cp36m'), 374 PythonArtifact('manylinux2014', 'x86', 'cp37-cp37m'), 375 PythonArtifact('manylinux2014', 'x86', 'cp38-cp38'), 376 PythonArtifact('manylinux2014', 'x86', 'cp39-cp39'), 377 PythonArtifact('manylinux2010', 'x64', 'cp27-cp27m'), 378 PythonArtifact('manylinux2010', 'x64', 'cp27-cp27mu'), 379 PythonArtifact('manylinux2010', 'x64', 'cp35-cp35m'), 380 PythonArtifact('manylinux2010', 'x64', 'cp36-cp36m'), 381 PythonArtifact('manylinux2010', 'x64', 'cp37-cp37m'), 382 PythonArtifact('manylinux2010', 'x64', 'cp38-cp38'), 383 PythonArtifact('manylinux2010', 'x64', 'cp39-cp39'), 384 PythonArtifact('manylinux2010', 'x86', 'cp27-cp27m'), 385 PythonArtifact('manylinux2010', 'x86', 'cp27-cp27mu'), 386 PythonArtifact('manylinux2010', 'x86', 'cp35-cp35m'), 387 PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'), 388 PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'), 389 PythonArtifact('manylinux2010', 'x86', 'cp38-cp38'), 390 PythonArtifact('manylinux2010', 'x86', 'cp39-cp39'), 391 PythonArtifact('manylinux2014', 'aarch64', 'cp36-cp36m'), 392 PythonArtifact('manylinux2014', 'aarch64', 'cp37-cp37m'), 393 PythonArtifact('manylinux2014', 'aarch64', 'cp38-cp38'), 394 PythonArtifact('manylinux2014', 'aarch64', 'cp39-cp39'), 395 PythonArtifact('linux_extra', 'armv7', 'cp36-cp36m'), 396 PythonArtifact('linux_extra', 'armv7', 'cp37-cp37m'), 397 PythonArtifact('linux_extra', 'armv7', 'cp38-cp38'), 398 PythonArtifact('linux_extra', 'armv7', 'cp39-cp39'), 399 PythonArtifact('macos', 'x64', 'python2.7'), 400 PythonArtifact('macos', 'x64', 'python3.5'), 401 PythonArtifact('macos', 'x64', 'python3.6'), 402 PythonArtifact('macos', 'x64', 'python3.7'), 403 PythonArtifact('macos', 'x64', 'python3.8'), 404 PythonArtifact('macos', 'x64', 'python3.9'), 405 PythonArtifact('windows', 'x86', 'Python27_32bit'), 406 PythonArtifact('windows', 'x86', 'Python35_32bit'), 407 PythonArtifact('windows', 'x86', 'Python36_32bit'), 408 PythonArtifact('windows', 'x86', 'Python37_32bit'), 409 PythonArtifact('windows', 'x86', 'Python38_32bit'), 410 PythonArtifact('windows', 'x86', 'Python39_32bit'), 411 PythonArtifact('windows', 'x64', 'Python27'), 412 PythonArtifact('windows', 'x64', 'Python35'), 413 PythonArtifact('windows', 'x64', 'Python36'), 414 PythonArtifact('windows', 'x64', 'Python37'), 415 PythonArtifact('windows', 'x64', 'Python38'), 416 PythonArtifact('windows', 'x64', 'Python39'), 417 RubyArtifact('linux', 'x64'), 418 RubyArtifact('macos', 'x64'), 419 PHPArtifact('linux', 'x64') 420 ] 421