1# Copyright 2022 The Bazel Authors. All rights reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""This file contains macros to be called during WORKSPACE evaluation. 16 17For historic reasons, pip_repositories() is defined in //python:pip.bzl. 18""" 19 20load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive") 21load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") 22load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") 23load("//python/private:coverage_deps.bzl", "coverage_dep") 24load( 25 "//python/private:toolchains_repo.bzl", 26 "multi_toolchain_aliases", 27 "toolchain_aliases", 28 "toolchains_repo", 29) 30load("//python/private:which.bzl", "which_with_fail") 31load( 32 ":versions.bzl", 33 "DEFAULT_RELEASE_BASE_URL", 34 "MINOR_MAPPING", 35 "PLATFORMS", 36 "TOOL_VERSIONS", 37 "get_release_info", 38) 39 40def http_archive(**kwargs): 41 maybe(_http_archive, **kwargs) 42 43def py_repositories(): 44 """Runtime dependencies that users must install. 45 46 This function should be loaded and called in the user's WORKSPACE. 47 With bzlmod enabled, this function is not needed since MODULE.bazel handles transitive deps. 48 """ 49 http_archive( 50 name = "bazel_skylib", 51 sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", 52 urls = [ 53 "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", 54 "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", 55 ], 56 ) 57 58######## 59# Remaining content of the file is only used to support toolchains. 60######## 61 62STANDALONE_INTERPRETER_FILENAME = "STANDALONE_INTERPRETER" 63 64def get_interpreter_dirname(rctx, python_interpreter_target): 65 """Get a python interpreter target dirname. 66 67 Args: 68 rctx (repository_ctx): The repository rule's context object. 69 python_interpreter_target (Target): A target representing a python interpreter. 70 71 Returns: 72 str: The Python interpreter directory. 73 """ 74 75 return rctx.path(Label("{}//:WORKSPACE".format(str(python_interpreter_target).split("//")[0]))).dirname 76 77def is_standalone_interpreter(rctx, python_interpreter_target): 78 """Query a python interpreter target for whether or not it's a rules_rust provided toolchain 79 80 Args: 81 rctx (repository_ctx): The repository rule's context object. 82 python_interpreter_target (Target): A target representing a python interpreter. 83 84 Returns: 85 bool: Whether or not the target is from a rules_python generated toolchain. 86 """ 87 88 # Only update the location when using a hermetic toolchain. 89 if not python_interpreter_target: 90 return False 91 92 # This is a rules_python provided toolchain. 93 return rctx.execute([ 94 "ls", 95 "{}/{}".format( 96 get_interpreter_dirname(rctx, python_interpreter_target), 97 STANDALONE_INTERPRETER_FILENAME, 98 ), 99 ]).return_code == 0 100 101def _python_repository_impl(rctx): 102 if rctx.attr.distutils and rctx.attr.distutils_content: 103 fail("Only one of (distutils, distutils_content) should be set.") 104 if bool(rctx.attr.url) == bool(rctx.attr.urls): 105 fail("Exactly one of (url, urls) must be set.") 106 107 platform = rctx.attr.platform 108 python_version = rctx.attr.python_version 109 python_short_version = python_version.rpartition(".")[0] 110 release_filename = rctx.attr.release_filename 111 urls = rctx.attr.urls or [rctx.attr.url] 112 113 if release_filename.endswith(".zst"): 114 rctx.download( 115 url = urls, 116 sha256 = rctx.attr.sha256, 117 output = release_filename, 118 ) 119 unzstd = rctx.which("unzstd") 120 if not unzstd: 121 url = rctx.attr.zstd_url.format(version = rctx.attr.zstd_version) 122 rctx.download_and_extract( 123 url = url, 124 sha256 = rctx.attr.zstd_sha256, 125 ) 126 working_directory = "zstd-{version}".format(version = rctx.attr.zstd_version) 127 128 make_result = rctx.execute( 129 [which_with_fail("make", rctx), "--jobs=4"], 130 timeout = 600, 131 quiet = True, 132 working_directory = working_directory, 133 ) 134 if make_result.return_code: 135 fail_msg = ( 136 "Failed to compile 'zstd' from source for use in Python interpreter extraction. " + 137 "'make' error message: {}".format(make_result.stderr) 138 ) 139 fail(fail_msg) 140 zstd = "{working_directory}/zstd".format(working_directory = working_directory) 141 unzstd = "./unzstd" 142 rctx.symlink(zstd, unzstd) 143 144 exec_result = rctx.execute([ 145 which_with_fail("tar", rctx), 146 "--extract", 147 "--strip-components=2", 148 "--use-compress-program={unzstd}".format(unzstd = unzstd), 149 "--file={}".format(release_filename), 150 ]) 151 if exec_result.return_code: 152 fail_msg = ( 153 "Failed to extract Python interpreter from '{}'. ".format(release_filename) + 154 "'tar' error message: {}".format(exec_result.stderr) 155 ) 156 fail(fail_msg) 157 else: 158 rctx.download_and_extract( 159 url = urls, 160 sha256 = rctx.attr.sha256, 161 stripPrefix = rctx.attr.strip_prefix, 162 ) 163 164 patches = rctx.attr.patches 165 if patches: 166 for patch in patches: 167 # Should take the strip as an attr, but this is fine for the moment 168 rctx.patch(patch, strip = 1) 169 170 # Write distutils.cfg to the Python installation. 171 if "windows" in rctx.os.name: 172 distutils_path = "Lib/distutils/distutils.cfg" 173 else: 174 distutils_path = "lib/python{}/distutils/distutils.cfg".format(python_short_version) 175 if rctx.attr.distutils: 176 rctx.file(distutils_path, rctx.read(rctx.attr.distutils)) 177 elif rctx.attr.distutils_content: 178 rctx.file(distutils_path, rctx.attr.distutils_content) 179 180 # Make the Python installation read-only. 181 if not rctx.attr.ignore_root_user_error: 182 if "windows" not in rctx.os.name: 183 lib_dir = "lib" if "windows" not in platform else "Lib" 184 185 exec_result = rctx.execute([which_with_fail("chmod", rctx), "-R", "ugo-w", lib_dir]) 186 if exec_result.return_code != 0: 187 fail_msg = "Failed to make interpreter installation read-only. 'chmod' error msg: {}".format( 188 exec_result.stderr, 189 ) 190 fail(fail_msg) 191 exec_result = rctx.execute([which_with_fail("touch", rctx), "{}/.test".format(lib_dir)]) 192 if exec_result.return_code == 0: 193 exec_result = rctx.execute([which_with_fail("id", rctx), "-u"]) 194 if exec_result.return_code != 0: 195 fail("Could not determine current user ID. 'id -u' error msg: {}".format( 196 exec_result.stderr, 197 )) 198 uid = int(exec_result.stdout.strip()) 199 if uid == 0: 200 fail("The current user is root, please run as non-root when using the hermetic Python interpreter. See https://github.com/bazelbuild/rules_python/pull/713.") 201 else: 202 fail("The current user has CAP_DAC_OVERRIDE set, please drop this capability when using the hermetic Python interpreter. See https://github.com/bazelbuild/rules_python/pull/713.") 203 204 python_bin = "python.exe" if ("windows" in platform) else "bin/python3" 205 206 glob_include = [] 207 glob_exclude = [ 208 "**/* *", # Bazel does not support spaces in file names. 209 # Unused shared libraries. `python` executable and the `:libpython` target 210 # depend on `libpython{python_version}.so.1.0`. 211 "lib/libpython{python_version}.so".format(python_version = python_short_version), 212 # static libraries 213 "lib/**/*.a", 214 # tests for the standard libraries. 215 "lib/python{python_version}/**/test/**".format(python_version = python_short_version), 216 "lib/python{python_version}/**/tests/**".format(python_version = python_short_version), 217 ] 218 219 if rctx.attr.ignore_root_user_error: 220 glob_exclude += [ 221 # These pycache files are created on first use of the associated python files. 222 # Exclude them from the glob because otherwise between the first time and second time a python toolchain is used," 223 # the definition of this filegroup will change, and depending rules will get invalidated." 224 # See https://github.com/bazelbuild/rules_python/issues/1008 for unconditionally adding these to toolchains so we can stop ignoring them." 225 "**/__pycache__/*.pyc", 226 "**/__pycache__/*.pyc.*", # During pyc creation, temp files named *.pyc.NNN are created 227 "**/__pycache__/*.pyo", 228 ] 229 230 if "windows" in platform: 231 glob_include += [ 232 "*.exe", 233 "*.dll", 234 "bin/**", 235 "DLLs/**", 236 "extensions/**", 237 "include/**", 238 "Lib/**", 239 "libs/**", 240 "Scripts/**", 241 "share/**", 242 ] 243 else: 244 glob_include += [ 245 "bin/**", 246 "extensions/**", 247 "include/**", 248 "lib/**", 249 "libs/**", 250 "share/**", 251 ] 252 253 if rctx.attr.coverage_tool: 254 if "windows" in rctx.os.name: 255 coverage_tool = None 256 else: 257 coverage_tool = '"{}"'.format(rctx.attr.coverage_tool) 258 259 coverage_attr_text = """\ 260 coverage_tool = select({{ 261 ":coverage_enabled": {coverage_tool}, 262 "//conditions:default": None 263 }}), 264""".format(coverage_tool = coverage_tool) 265 else: 266 coverage_attr_text = " # coverage_tool attribute not supported by this Bazel version" 267 268 build_content = """\ 269# Generated by python/repositories.bzl 270 271load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") 272load("@rules_python//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain") 273 274package(default_visibility = ["//visibility:public"]) 275 276filegroup( 277 name = "files", 278 srcs = glob( 279 include = {glob_include}, 280 # Platform-agnostic filegroup can't match on all patterns. 281 allow_empty = True, 282 exclude = {glob_exclude}, 283 ), 284) 285 286cc_import( 287 name = "interface", 288 interface_library = "libs/python{python_version_nodot}.lib", 289 system_provided = True, 290) 291 292filegroup( 293 name = "includes", 294 srcs = glob(["include/**/*.h"]), 295) 296 297cc_library( 298 name = "python_headers", 299 deps = select({{ 300 "@bazel_tools//src/conditions:windows": [":interface"], 301 "//conditions:default": None, 302 }}), 303 hdrs = [":includes"], 304 includes = [ 305 "include", 306 "include/python{python_version}", 307 "include/python{python_version}m", 308 ], 309) 310 311cc_library( 312 name = "libpython", 313 hdrs = [":includes"], 314 srcs = select({{ 315 "@platforms//os:windows": ["python3.dll", "libs/python{python_version_nodot}.lib"], 316 "@platforms//os:macos": ["lib/libpython{python_version}.dylib"], 317 "@platforms//os:linux": ["lib/libpython{python_version}.so", "lib/libpython{python_version}.so.1.0"], 318 }}), 319) 320 321exports_files(["python", "{python_path}"]) 322 323# Used to only download coverage toolchain when the coverage is collected by 324# bazel. 325config_setting( 326 name = "coverage_enabled", 327 values = {{"collect_code_coverage": "true"}}, 328 visibility = ["//visibility:private"], 329) 330 331py_runtime( 332 name = "py3_runtime", 333 files = [":files"], 334{coverage_attr} 335 interpreter = "{python_path}", 336 python_version = "PY3", 337) 338 339py_runtime_pair( 340 name = "python_runtimes", 341 py2_runtime = None, 342 py3_runtime = ":py3_runtime", 343) 344 345py_cc_toolchain( 346 name = "py_cc_toolchain", 347 headers = ":python_headers", 348 python_version = "{python_version}", 349) 350""".format( 351 glob_exclude = repr(glob_exclude), 352 glob_include = repr(glob_include), 353 python_path = python_bin, 354 python_version = python_short_version, 355 python_version_nodot = python_short_version.replace(".", ""), 356 coverage_attr = coverage_attr_text, 357 ) 358 rctx.delete("python") 359 rctx.symlink(python_bin, "python") 360 rctx.file(STANDALONE_INTERPRETER_FILENAME, "# File intentionally left blank. Indicates that this is an interpreter repo created by rules_python.") 361 rctx.file("BUILD.bazel", build_content) 362 363 attrs = { 364 "coverage_tool": rctx.attr.coverage_tool, 365 "distutils": rctx.attr.distutils, 366 "distutils_content": rctx.attr.distutils_content, 367 "ignore_root_user_error": rctx.attr.ignore_root_user_error, 368 "name": rctx.attr.name, 369 "patches": rctx.attr.patches, 370 "platform": platform, 371 "python_version": python_version, 372 "release_filename": release_filename, 373 "sha256": rctx.attr.sha256, 374 "strip_prefix": rctx.attr.strip_prefix, 375 } 376 377 if rctx.attr.url: 378 attrs["url"] = rctx.attr.url 379 else: 380 attrs["urls"] = urls 381 382 return attrs 383 384python_repository = repository_rule( 385 _python_repository_impl, 386 doc = "Fetches the external tools needed for the Python toolchain.", 387 attrs = { 388 "coverage_tool": attr.string( 389 # Mirrors the definition at 390 # https://github.com/bazelbuild/bazel/blob/master/src/main/starlark/builtins_bzl/common/python/py_runtime_rule.bzl 391 doc = """ 392This is a target to use for collecting code coverage information from `py_binary` 393and `py_test` targets. 394 395If set, the target must either produce a single file or be an executable target. 396The path to the single file, or the executable if the target is executable, 397determines the entry point for the python coverage tool. The target and its 398runfiles will be added to the runfiles when coverage is enabled. 399 400The entry point for the tool must be loadable by a Python interpreter (e.g. a 401`.py` or `.pyc` file). It must accept the command line arguments 402of coverage.py (https://coverage.readthedocs.io), at least including 403the `run` and `lcov` subcommands. 404 405The target is accepted as a string by the python_repository and evaluated within 406the context of the toolchain repository. 407 408For more information see the official bazel docs 409(https://bazel.build/reference/be/python#py_runtime.coverage_tool). 410""", 411 ), 412 "distutils": attr.label( 413 allow_single_file = True, 414 doc = "A distutils.cfg file to be included in the Python installation. " + 415 "Either distutils or distutils_content can be specified, but not both.", 416 mandatory = False, 417 ), 418 "distutils_content": attr.string( 419 doc = "A distutils.cfg file content to be included in the Python installation. " + 420 "Either distutils or distutils_content can be specified, but not both.", 421 mandatory = False, 422 ), 423 "ignore_root_user_error": attr.bool( 424 default = False, 425 doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", 426 mandatory = False, 427 ), 428 "patches": attr.label_list( 429 doc = "A list of patch files to apply to the unpacked interpreter", 430 mandatory = False, 431 ), 432 "platform": attr.string( 433 doc = "The platform name for the Python interpreter tarball.", 434 mandatory = True, 435 values = PLATFORMS.keys(), 436 ), 437 "python_version": attr.string( 438 doc = "The Python version.", 439 mandatory = True, 440 ), 441 "release_filename": attr.string( 442 doc = "The filename of the interpreter to be downloaded", 443 mandatory = True, 444 ), 445 "sha256": attr.string( 446 doc = "The SHA256 integrity hash for the Python interpreter tarball.", 447 mandatory = True, 448 ), 449 "strip_prefix": attr.string( 450 doc = "A directory prefix to strip from the extracted files.", 451 ), 452 "url": attr.string( 453 doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", 454 ), 455 "urls": attr.string_list( 456 doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", 457 ), 458 "zstd_sha256": attr.string( 459 default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0", 460 ), 461 "zstd_url": attr.string( 462 default = "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz", 463 ), 464 "zstd_version": attr.string( 465 default = "1.5.2", 466 ), 467 }, 468) 469 470# Wrapper macro around everything above, this is the primary API. 471def python_register_toolchains( 472 name, 473 python_version, 474 distutils = None, 475 distutils_content = None, 476 register_toolchains = True, 477 register_coverage_tool = False, 478 set_python_version_constraint = False, 479 tool_versions = TOOL_VERSIONS, 480 **kwargs): 481 """Convenience macro for users which does typical setup. 482 483 - Create a repository for each built-in platform like "python_linux_amd64" - 484 this repository is lazily fetched when Python is needed for that platform. 485 - Create a repository exposing toolchains for each platform like 486 "python_platforms". 487 - Register a toolchain pointing at each platform. 488 Users can avoid this macro and do these steps themselves, if they want more 489 control. 490 Args: 491 name: base name for all created repos, like "python38". 492 python_version: the Python version. 493 distutils: see the distutils attribute in the python_repository repository rule. 494 distutils_content: see the distutils_content attribute in the python_repository repository rule. 495 register_toolchains: Whether or not to register the downloaded toolchains. 496 register_coverage_tool: Whether or not to register the downloaded coverage tool to the toolchains. 497 NOTE: Coverage support using the toolchain is only supported in Bazel 6 and higher. 498 499 set_python_version_constraint: When set to true, target_compatible_with for the toolchains will include a version constraint. 500 tool_versions: a dict containing a mapping of version with SHASUM and platform info. If not supplied, the defaults 501 in python/versions.bzl will be used. 502 **kwargs: passed to each python_repositories call. 503 """ 504 505 if BZLMOD_ENABLED: 506 # you cannot used native.register_toolchains when using bzlmod. 507 register_toolchains = False 508 509 base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL) 510 511 if python_version in MINOR_MAPPING: 512 python_version = MINOR_MAPPING[python_version] 513 514 toolchain_repo_name = "{name}_toolchains".format(name = name) 515 516 # When using unreleased Bazel versions, the version is an empty string 517 if native.bazel_version: 518 bazel_major = int(native.bazel_version.split(".")[0]) 519 if bazel_major < 6: 520 if register_coverage_tool: 521 # buildifier: disable=print 522 print(( 523 "WARNING: ignoring register_coverage_tool=True when " + 524 "registering @{name}: Bazel 6+ required, got {version}" 525 ).format( 526 name = name, 527 version = native.bazel_version, 528 )) 529 register_coverage_tool = False 530 531 for platform in PLATFORMS.keys(): 532 sha256 = tool_versions[python_version]["sha256"].get(platform, None) 533 if not sha256: 534 continue 535 536 (release_filename, urls, strip_prefix, patches) = get_release_info(platform, python_version, base_url, tool_versions) 537 538 # allow passing in a tool version 539 coverage_tool = None 540 coverage_tool = tool_versions[python_version].get("coverage_tool", {}).get(platform, None) 541 if register_coverage_tool and coverage_tool == None: 542 coverage_tool = coverage_dep( 543 name = "{name}_{platform}_coverage".format( 544 name = name, 545 platform = platform, 546 ), 547 python_version = python_version, 548 platform = platform, 549 visibility = ["@{name}_{platform}//:__subpackages__".format( 550 name = name, 551 platform = platform, 552 )], 553 ) 554 555 python_repository( 556 name = "{name}_{platform}".format( 557 name = name, 558 platform = platform, 559 ), 560 sha256 = sha256, 561 patches = patches, 562 platform = platform, 563 python_version = python_version, 564 release_filename = release_filename, 565 urls = urls, 566 distutils = distutils, 567 distutils_content = distutils_content, 568 strip_prefix = strip_prefix, 569 coverage_tool = coverage_tool, 570 **kwargs 571 ) 572 if register_toolchains: 573 native.register_toolchains("@{toolchain_repo_name}//:{platform}_toolchain".format( 574 toolchain_repo_name = toolchain_repo_name, 575 platform = platform, 576 )) 577 578 toolchain_aliases( 579 name = name, 580 python_version = python_version, 581 user_repository_name = name, 582 ) 583 584 # in bzlmod we write out our own toolchain repos 585 if BZLMOD_ENABLED: 586 return 587 588 toolchains_repo( 589 name = toolchain_repo_name, 590 python_version = python_version, 591 set_python_version_constraint = set_python_version_constraint, 592 user_repository_name = name, 593 ) 594 595def python_register_multi_toolchains( 596 name, 597 python_versions, 598 default_version = None, 599 **kwargs): 600 """Convenience macro for registering multiple Python toolchains. 601 602 Args: 603 name: base name for each name in python_register_toolchains call. 604 python_versions: the Python version. 605 default_version: the default Python version. If not set, the first version in 606 python_versions is used. 607 **kwargs: passed to each python_register_toolchains call. 608 """ 609 if len(python_versions) == 0: 610 fail("python_versions must not be empty") 611 612 if not default_version: 613 default_version = python_versions.pop(0) 614 for python_version in python_versions: 615 if python_version == default_version: 616 # We register the default version lastly so that it's not picked first when --platforms 617 # is set with a constraint during toolchain resolution. This is due to the fact that 618 # Bazel will match the unconstrained toolchain if we register it before the constrained 619 # ones. 620 continue 621 python_register_toolchains( 622 name = name + "_" + python_version.replace(".", "_"), 623 python_version = python_version, 624 set_python_version_constraint = True, 625 **kwargs 626 ) 627 python_register_toolchains( 628 name = name + "_" + default_version.replace(".", "_"), 629 python_version = default_version, 630 set_python_version_constraint = False, 631 **kwargs 632 ) 633 634 multi_toolchain_aliases( 635 name = name, 636 python_versions = { 637 python_version: name + "_" + python_version.replace(".", "_") 638 for python_version in (python_versions + [default_version]) 639 }, 640 ) 641