1#!/usr/bin/env python3 2# Copyright 2016 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Definition of targets to build artifacts.""" 16 17import os.path 18import random 19import string 20import sys 21 22sys.path.insert(0, os.path.abspath("..")) 23import python_utils.jobset as jobset 24 25_LATEST_MANYLINUX = "manylinux2014" 26 27 28def create_docker_jobspec( 29 name, 30 dockerfile_dir, 31 shell_command, 32 environ={}, 33 flake_retries=0, 34 timeout_retries=0, 35 timeout_seconds=30 * 60, 36 extra_docker_args=None, 37 verbose_success=False, 38): 39 """Creates jobspec for a task running under docker.""" 40 environ = environ.copy() 41 environ["ARTIFACTS_OUT"] = "artifacts/%s" % name 42 43 docker_args = [] 44 for k, v in list(environ.items()): 45 docker_args += ["-e", "%s=%s" % (k, v)] 46 docker_env = { 47 "DOCKERFILE_DIR": dockerfile_dir, 48 "DOCKER_RUN_SCRIPT": "tools/run_tests/dockerize/docker_run.sh", 49 "DOCKER_RUN_SCRIPT_COMMAND": shell_command, 50 "OUTPUT_DIR": "artifacts", 51 } 52 if extra_docker_args is not None: 53 docker_env["EXTRA_DOCKER_ARGS"] = extra_docker_args 54 jobspec = jobset.JobSpec( 55 cmdline=["tools/run_tests/dockerize/build_and_run_docker.sh"] 56 + docker_args, 57 environ=docker_env, 58 shortname="build_artifact.%s" % (name), 59 timeout_seconds=timeout_seconds, 60 flake_retries=flake_retries, 61 timeout_retries=timeout_retries, 62 verbose_success=verbose_success, 63 ) 64 return jobspec 65 66 67def create_jobspec( 68 name, 69 cmdline, 70 environ={}, 71 shell=False, 72 flake_retries=0, 73 timeout_retries=0, 74 timeout_seconds=30 * 60, 75 use_workspace=False, 76 cpu_cost=1.0, 77 verbose_success=False, 78): 79 """Creates jobspec.""" 80 environ = environ.copy() 81 if use_workspace: 82 environ["WORKSPACE_NAME"] = "workspace_%s" % name 83 environ["ARTIFACTS_OUT"] = os.path.join("..", "artifacts", name) 84 cmdline = [ 85 "bash", 86 "tools/run_tests/artifacts/run_in_workspace.sh", 87 ] + cmdline 88 else: 89 environ["ARTIFACTS_OUT"] = os.path.join("artifacts", name) 90 91 jobspec = jobset.JobSpec( 92 cmdline=cmdline, 93 environ=environ, 94 shortname="build_artifact.%s" % (name), 95 timeout_seconds=timeout_seconds, 96 flake_retries=flake_retries, 97 timeout_retries=timeout_retries, 98 shell=shell, 99 cpu_cost=cpu_cost, 100 verbose_success=verbose_success, 101 ) 102 return jobspec 103 104 105_MACOS_COMPAT_FLAG = "-mmacosx-version-min=10.10" 106 107_ARCH_FLAG_MAP = {"x86": "-m32", "x64": "-m64"} 108 109 110class PythonArtifact: 111 """Builds Python artifacts.""" 112 113 def __init__(self, platform, arch, py_version, presubmit=False): 114 self.name = "python_%s_%s_%s" % (platform, arch, py_version) 115 self.platform = platform 116 self.arch = arch 117 self.labels = ["artifact", "python", platform, arch, py_version] 118 if presubmit: 119 self.labels.append("presubmit") 120 self.py_version = py_version 121 if platform == _LATEST_MANYLINUX: 122 self.labels.append("latest-manylinux") 123 if "manylinux" in platform: 124 self.labels.append("linux") 125 if "linux_extra" in platform: 126 # linux_extra wheels used to be built by a separate kokoro job. 127 # Their build is now much faster, so they can be included 128 # in the regular artifact build. 129 self.labels.append("linux") 130 if "musllinux" in platform: 131 self.labels.append("linux") 132 133 def pre_build_jobspecs(self): 134 return [] 135 136 def build_jobspec(self, inner_jobs=None): 137 environ = {} 138 if inner_jobs is not None: 139 # set number of parallel jobs when building native extension 140 # building the native extension is the most time-consuming part of the build 141 environ["GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS"] = str(inner_jobs) 142 143 if self.platform == "macos": 144 environ["ARCHFLAGS"] = "-arch arm64 -arch x86_64" 145 environ["GRPC_BUILD_MAC"] = "true" 146 147 if self.platform == "linux_extra": 148 # Crosscompilation build for armv7 (e.g. Raspberry Pi) 149 environ["PYTHON"] = "/opt/python/{}/bin/python3".format( 150 self.py_version 151 ) 152 environ["PIP"] = "/opt/python/{}/bin/pip3".format(self.py_version) 153 environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" 154 environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" 155 environ["LDFLAGS"] = "-s" 156 return create_docker_jobspec( 157 self.name, 158 "tools/dockerfile/grpc_artifact_python_linux_{}".format( 159 self.arch 160 ), 161 "tools/run_tests/artifacts/build_artifact_python.sh", 162 environ=environ, 163 timeout_seconds=60 * 60 * 2, 164 ) 165 elif "manylinux" in self.platform: 166 if self.arch == "x86": 167 environ["SETARCH_CMD"] = "linux32" 168 environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" 169 # Inside the manylinux container, the python installations are located in 170 # special places... 171 environ["PYTHON"] = "/opt/python/{}/bin/python".format( 172 self.py_version 173 ) 174 environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) 175 environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" 176 if self.arch == "aarch64": 177 environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" 178 # As we won't strip the binary with auditwheel (see below), strip 179 # it at link time. 180 environ["LDFLAGS"] = "-s" 181 else: 182 # only run auditwheel if we're not crosscompiling 183 environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" 184 # only build the packages that depend on grpcio-tools 185 # if we're not crosscompiling. 186 # - they require protoc to run on current architecture 187 # - they only have sdist packages anyway, so it's useless to build them again 188 environ["GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS"] = "TRUE" 189 return create_docker_jobspec( 190 self.name, 191 "tools/dockerfile/grpc_artifact_python_%s_%s" 192 % (self.platform, self.arch), 193 "tools/run_tests/artifacts/build_artifact_python.sh", 194 environ=environ, 195 timeout_seconds=60 * 60 * 2, 196 ) 197 elif "musllinux" in self.platform: 198 environ["PYTHON"] = "/opt/python/{}/bin/python".format( 199 self.py_version 200 ) 201 environ["PIP"] = "/opt/python/{}/bin/pip".format(self.py_version) 202 environ["GRPC_SKIP_PIP_CYTHON_UPGRADE"] = "TRUE" 203 environ["GRPC_RUN_AUDITWHEEL_REPAIR"] = "TRUE" 204 environ["GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX"] = "TRUE" 205 206 if self.arch == "x86": 207 environ["GRPC_SKIP_TWINE_CHECK"] = "TRUE" 208 209 return create_docker_jobspec( 210 self.name, 211 "tools/dockerfile/grpc_artifact_python_%s_%s" 212 % (self.platform, self.arch), 213 "tools/run_tests/artifacts/build_artifact_python.sh", 214 environ=environ, 215 timeout_seconds=60 * 60 * 2, 216 ) 217 elif self.platform == "windows": 218 environ["EXT_COMPILER"] = "msvc" 219 # For some reason, the batch script %random% always runs with the same 220 # seed. We create a random temp-dir here 221 dir = "".join( 222 random.choice(string.ascii_uppercase) for _ in range(10) 223 ) 224 return create_jobspec( 225 self.name, 226 [ 227 "tools\\run_tests\\artifacts\\build_artifact_python.bat", 228 self.py_version, 229 "32" if self.arch == "x86" else "64", 230 ], 231 environ=environ, 232 timeout_seconds=60 * 60 * 2, 233 use_workspace=True, 234 ) 235 else: 236 environ["PYTHON"] = self.py_version 237 environ["SKIP_PIP_INSTALL"] = "TRUE" 238 return create_jobspec( 239 self.name, 240 ["tools/run_tests/artifacts/build_artifact_python.sh"], 241 environ=environ, 242 timeout_seconds=60 * 60 * 2, 243 use_workspace=True, 244 ) 245 246 def __str__(self): 247 return self.name 248 249 250class RubyArtifact: 251 """Builds ruby native gem.""" 252 253 def __init__(self, platform, gem_platform, presubmit=False): 254 self.name = "ruby_native_gem_%s_%s" % (platform, gem_platform) 255 self.platform = platform 256 self.gem_platform = gem_platform 257 self.labels = ["artifact", "ruby", platform, gem_platform] 258 if presubmit: 259 self.labels.append("presubmit") 260 261 def pre_build_jobspecs(self): 262 return [] 263 264 def build_jobspec(self, inner_jobs=None): 265 environ = {} 266 if inner_jobs is not None: 267 # set number of parallel jobs when building native extension 268 environ["GRPC_RUBY_BUILD_PROCS"] = str(inner_jobs) 269 # Ruby build uses docker internally and docker cannot be nested. 270 # We are using a custom workspace instead. 271 return create_jobspec( 272 self.name, 273 [ 274 "tools/run_tests/artifacts/build_artifact_ruby.sh", 275 self.gem_platform, 276 ], 277 use_workspace=True, 278 timeout_seconds=180 * 60, 279 environ=environ, 280 ) 281 282 283class PHPArtifact: 284 """Builds PHP PECL package""" 285 286 def __init__(self, platform, arch, presubmit=False): 287 self.name = "php_pecl_package_{0}_{1}".format(platform, arch) 288 self.platform = platform 289 self.arch = arch 290 self.labels = ["artifact", "php", platform, arch] 291 if presubmit: 292 self.labels.append("presubmit") 293 294 def pre_build_jobspecs(self): 295 return [] 296 297 def build_jobspec(self, inner_jobs=None): 298 del inner_jobs # arg unused as PHP artifact build is basically just packing an archive 299 if self.platform == "linux": 300 return create_docker_jobspec( 301 self.name, 302 "tools/dockerfile/test/php8_zts_debian12_{}".format(self.arch), 303 "tools/run_tests/artifacts/build_artifact_php.sh", 304 ) 305 else: 306 return create_jobspec( 307 self.name, 308 ["tools/run_tests/artifacts/build_artifact_php.sh"], 309 use_workspace=True, 310 ) 311 312 313class ProtocArtifact: 314 """Builds protoc and protoc-plugin artifacts""" 315 316 def __init__(self, platform, arch, presubmit=False): 317 self.name = "protoc_%s_%s" % (platform, arch) 318 self.platform = platform 319 self.arch = arch 320 self.labels = ["artifact", "protoc", platform, arch] 321 if presubmit: 322 self.labels.append("presubmit") 323 324 def pre_build_jobspecs(self): 325 return [] 326 327 def build_jobspec(self, inner_jobs=None): 328 environ = {} 329 if inner_jobs is not None: 330 # set number of parallel jobs when building protoc 331 environ["GRPC_PROTOC_BUILD_COMPILER_JOBS"] = str(inner_jobs) 332 333 if self.platform != "windows": 334 environ["CXXFLAGS"] = "" 335 environ["LDFLAGS"] = "" 336 if self.platform == "linux": 337 dockerfile_dir = ( 338 "tools/dockerfile/grpc_artifact_centos6_{}".format( 339 self.arch 340 ) 341 ) 342 if self.arch == "aarch64": 343 # for aarch64, use a dockcross manylinux image that will 344 # give us both ready to use crosscompiler and sufficient backward compatibility 345 dockerfile_dir = ( 346 "tools/dockerfile/grpc_artifact_protoc_aarch64" 347 ) 348 environ["LDFLAGS"] += " -static-libgcc -static-libstdc++ -s" 349 return create_docker_jobspec( 350 self.name, 351 dockerfile_dir, 352 "tools/run_tests/artifacts/build_artifact_protoc.sh", 353 environ=environ, 354 ) 355 else: 356 environ["CXXFLAGS"] += ( 357 " -std=c++17 -stdlib=libc++ %s" % _MACOS_COMPAT_FLAG 358 ) 359 return create_jobspec( 360 self.name, 361 ["tools/run_tests/artifacts/build_artifact_protoc.sh"], 362 environ=environ, 363 timeout_seconds=60 * 60, 364 use_workspace=True, 365 ) 366 else: 367 vs_tools_architecture = ( 368 self.arch 369 ) # architecture selector passed to vcvarsall.bat 370 environ["ARCHITECTURE"] = vs_tools_architecture 371 return create_jobspec( 372 self.name, 373 ["tools\\run_tests\\artifacts\\build_artifact_protoc.bat"], 374 environ=environ, 375 use_workspace=True, 376 ) 377 378 def __str__(self): 379 return self.name 380 381 382def _reorder_targets_for_build_speed(targets): 383 """Reorder targets to achieve optimal build speed""" 384 # ruby artifact build builds multiple artifacts at once, so make sure 385 # we start building ruby artifacts first, so that they don't end up 386 # being a long tail once everything else finishes. 387 return list( 388 sorted( 389 targets, 390 key=lambda target: 0 if target.name.startswith("ruby_") else 1, 391 ) 392 ) 393 394 395def targets(): 396 """Gets list of supported targets""" 397 return _reorder_targets_for_build_speed( 398 [ 399 ProtocArtifact("linux", "x64", presubmit=True), 400 ProtocArtifact("linux", "x86", presubmit=True), 401 ProtocArtifact("linux", "aarch64", presubmit=True), 402 ProtocArtifact("macos", "x64", presubmit=True), 403 ProtocArtifact("windows", "x64", presubmit=True), 404 ProtocArtifact("windows", "x86", presubmit=True), 405 PythonArtifact("manylinux2014", "x64", "cp38-cp38", presubmit=True), 406 PythonArtifact("manylinux2014", "x64", "cp39-cp39", presubmit=True), 407 PythonArtifact("manylinux2014", "x64", "cp310-cp310"), 408 PythonArtifact("manylinux2014", "x64", "cp311-cp311"), 409 PythonArtifact("manylinux2014", "x64", "cp312-cp312"), 410 PythonArtifact( 411 "manylinux2014", "x64", "cp313-cp313", presubmit=True 412 ), 413 PythonArtifact("manylinux2014", "x86", "cp38-cp38", presubmit=True), 414 PythonArtifact("manylinux2014", "x86", "cp39-cp39", presubmit=True), 415 PythonArtifact("manylinux2014", "x86", "cp310-cp310"), 416 PythonArtifact("manylinux2014", "x86", "cp311-cp311"), 417 PythonArtifact("manylinux2014", "x86", "cp312-cp312"), 418 PythonArtifact( 419 "manylinux2014", "x86", "cp313-cp313", presubmit=True 420 ), 421 PythonArtifact( 422 "manylinux2014", "aarch64", "cp38-cp38", presubmit=True 423 ), 424 PythonArtifact("manylinux2014", "aarch64", "cp39-cp39"), 425 PythonArtifact("manylinux2014", "aarch64", "cp310-cp310"), 426 PythonArtifact("manylinux2014", "aarch64", "cp311-cp311"), 427 PythonArtifact("manylinux2014", "aarch64", "cp312-cp312"), 428 PythonArtifact( 429 "manylinux2014", "aarch64", "cp313-cp313", presubmit=True 430 ), 431 PythonArtifact("linux_extra", "armv7", "cp38-cp38", presubmit=True), 432 PythonArtifact("linux_extra", "armv7", "cp39-cp39"), 433 PythonArtifact("linux_extra", "armv7", "cp310-cp310"), 434 PythonArtifact("linux_extra", "armv7", "cp311-cp311"), 435 PythonArtifact("linux_extra", "armv7", "cp312-cp312"), 436 PythonArtifact( 437 "linux_extra", "armv7", "cp313-cp313", presubmit=True 438 ), 439 PythonArtifact("musllinux_1_1", "x64", "cp38-cp38", presubmit=True), 440 PythonArtifact("musllinux_1_1", "x64", "cp39-cp39"), 441 PythonArtifact("musllinux_1_1", "x64", "cp310-cp310"), 442 PythonArtifact("musllinux_1_1", "x64", "cp311-cp311"), 443 PythonArtifact("musllinux_1_1", "x64", "cp312-cp312"), 444 PythonArtifact( 445 "musllinux_1_1", "x64", "cp313-cp313", presubmit=True 446 ), 447 PythonArtifact("musllinux_1_1", "x86", "cp38-cp38", presubmit=True), 448 PythonArtifact("musllinux_1_1", "x86", "cp39-cp39"), 449 PythonArtifact("musllinux_1_1", "x86", "cp310-cp310"), 450 PythonArtifact("musllinux_1_1", "x86", "cp311-cp311"), 451 PythonArtifact("musllinux_1_1", "x86", "cp312-cp312"), 452 PythonArtifact( 453 "musllinux_1_1", "x86", "cp313-cp313", presubmit=True 454 ), 455 PythonArtifact("macos", "x64", "python3.8", presubmit=True), 456 PythonArtifact("macos", "x64", "python3.9"), 457 PythonArtifact("macos", "x64", "python3.10"), 458 PythonArtifact("macos", "x64", "python3.11"), 459 PythonArtifact("macos", "x64", "python3.12"), 460 PythonArtifact("macos", "x64", "python3.13", presubmit=True), 461 PythonArtifact("windows", "x86", "Python38_32bit", presubmit=True), 462 PythonArtifact("windows", "x86", "Python39_32bit"), 463 PythonArtifact("windows", "x86", "Python310_32bit"), 464 PythonArtifact("windows", "x86", "Python311_32bit"), 465 PythonArtifact("windows", "x86", "Python312_32bit"), 466 PythonArtifact("windows", "x86", "Python313_32bit", presubmit=True), 467 PythonArtifact("windows", "x64", "Python38", presubmit=True), 468 PythonArtifact("windows", "x64", "Python39"), 469 PythonArtifact("windows", "x64", "Python310"), 470 PythonArtifact("windows", "x64", "Python311"), 471 PythonArtifact("windows", "x64", "Python312"), 472 PythonArtifact("windows", "x64", "Python313", presubmit=True), 473 RubyArtifact("linux", "x86-mingw32", presubmit=True), 474 RubyArtifact("linux", "x64-mingw32"), 475 RubyArtifact("linux", "x64-mingw-ucrt", presubmit=True), 476 RubyArtifact("linux", "x86_64-linux", presubmit=True), 477 RubyArtifact("linux", "x86-linux"), 478 RubyArtifact("linux", "aarch64-linux", presubmit=True), 479 RubyArtifact("linux", "x86_64-darwin", presubmit=True), 480 RubyArtifact("linux", "arm64-darwin", presubmit=True), 481 PHPArtifact("linux", "x64", presubmit=True), 482 PHPArtifact("macos", "x64", presubmit=True), 483 ] 484 ) 485