1# Copyright 2022 The Bazel Authors. All rights reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Common functionality between test/binary executables.""" 15 16load("@bazel_skylib//lib:dicts.bzl", "dicts") 17load("@bazel_skylib//lib:structs.bzl", "structs") 18load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") 19load("@rules_cc//cc/common:cc_common.bzl", "cc_common") 20load( 21 ":attributes.bzl", 22 "AGNOSTIC_EXECUTABLE_ATTRS", 23 "COMMON_ATTRS", 24 "PY_SRCS_ATTRS", 25 "PrecompileAttr", 26 "PycCollectionAttr", 27 "REQUIRED_EXEC_GROUPS", 28 "SRCS_VERSION_ALL_VALUES", 29 "create_srcs_attr", 30 "create_srcs_version_attr", 31) 32load(":builders.bzl", "builders") 33load(":cc_helper.bzl", "cc_helper") 34load( 35 ":common.bzl", 36 "collect_imports", 37 "collect_runfiles", 38 "create_instrumented_files_info", 39 "create_output_group_info", 40 "create_py_info", 41 "csv", 42 "filter_to_py_srcs", 43 "target_platform_has_any_constraint", 44 "union_attrs", 45) 46load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") 47load(":py_executable_info.bzl", "PyExecutableInfo") 48load(":py_info.bzl", "PyInfo") 49load(":py_internal.bzl", "py_internal") 50load(":py_runtime_info.bzl", "PyRuntimeInfo") 51load(":reexports.bzl", "BuiltinPyInfo", "BuiltinPyRuntimeInfo") 52load( 53 ":semantics.bzl", 54 "ALLOWED_MAIN_EXTENSIONS", 55 "BUILD_DATA_SYMLINK_PATH", 56 "IS_BAZEL", 57 "PY_RUNTIME_ATTR_NAME", 58) 59load( 60 ":toolchain_types.bzl", 61 "EXEC_TOOLS_TOOLCHAIN_TYPE", 62 TOOLCHAIN_TYPE = "TARGET_TOOLCHAIN_TYPE", 63) 64 65_py_builtins = py_internal 66 67# Bazel 5.4 doesn't have config_common.toolchain_type 68_CC_TOOLCHAINS = [config_common.toolchain_type( 69 "@bazel_tools//tools/cpp:toolchain_type", 70 mandatory = False, 71)] if hasattr(config_common, "toolchain_type") else [] 72 73# Non-Google-specific attributes for executables 74# These attributes are for rules that accept Python sources. 75EXECUTABLE_ATTRS = union_attrs( 76 COMMON_ATTRS, 77 AGNOSTIC_EXECUTABLE_ATTRS, 78 PY_SRCS_ATTRS, 79 { 80 # TODO(b/203567235): In the Java impl, any file is allowed. While marked 81 # label, it is more treated as a string, and doesn't have to refer to 82 # anything that exists because it gets treated as suffix-search string 83 # over `srcs`. 84 "main": attr.label( 85 allow_single_file = True, 86 doc = """\ 87Optional; the name of the source file that is the main entry point of the 88application. This file must also be listed in `srcs`. If left unspecified, 89`name`, with `.py` appended, is used instead. If `name` does not match any 90filename in `srcs`, `main` must be specified. 91""", 92 ), 93 "pyc_collection": attr.string( 94 default = PycCollectionAttr.INHERIT, 95 values = sorted(PycCollectionAttr.__members__.values()), 96 doc = """ 97Determines whether pyc files from dependencies should be manually included. 98 99Valid values are: 100* `inherit`: Inherit the value from {flag}`--precompile`. 101* `include_pyc`: Add implicitly generated pyc files from dependencies. i.e. 102 pyc files for targets that specify {attr}`precompile="inherit"`. 103* `disabled`: Don't add implicitly generated pyc files. Note that 104 pyc files may still come from dependencies that enable precompiling at the 105 target level. 106""", 107 ), 108 # TODO(b/203567235): In Google, this attribute is deprecated, and can 109 # only effectively be PY3. Externally, with Bazel, this attribute has 110 # a separate story. 111 "python_version": attr.string( 112 # TODO(b/203567235): In the Java impl, the default comes from 113 # --python_version. Not clear what the Starlark equivalent is. 114 default = "PY3", 115 # NOTE: Some tests care about the order of these values. 116 values = ["PY2", "PY3"], 117 doc = "Defunct, unused, does nothing.", 118 ), 119 "_bootstrap_impl_flag": attr.label( 120 default = "//python/config_settings:bootstrap_impl", 121 providers = [BuildSettingInfo], 122 ), 123 "_windows_constraints": attr.label_list( 124 default = [ 125 "@platforms//os:windows", 126 ], 127 ), 128 }, 129 create_srcs_version_attr(values = SRCS_VERSION_ALL_VALUES), 130 create_srcs_attr(mandatory = True), 131 allow_none = True, 132) 133 134def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = []): 135 """Base rule implementation for a Python executable. 136 137 Google and Bazel call this common base and apply customizations using the 138 semantics object. 139 140 Args: 141 ctx: The rule ctx 142 semantics: BinarySemantics struct; see create_binary_semantics_struct() 143 is_test: bool, True if the rule is a test rule (has `test=True`), 144 False if not (has `executable=True`) 145 inherited_environment: List of str; additional environment variable 146 names that should be inherited from the runtime environment when the 147 executable is run. 148 Returns: 149 DefaultInfo provider for the executable 150 """ 151 _validate_executable(ctx) 152 153 main_py = determine_main(ctx) 154 direct_sources = filter_to_py_srcs(ctx.files.srcs) 155 precompile_result = semantics.maybe_precompile(ctx, direct_sources) 156 157 required_py_files = precompile_result.keep_srcs 158 required_pyc_files = [] 159 implicit_pyc_files = [] 160 implicit_pyc_source_files = direct_sources 161 162 if ctx.attr.precompile == PrecompileAttr.ENABLED: 163 required_pyc_files.extend(precompile_result.pyc_files) 164 else: 165 implicit_pyc_files.extend(precompile_result.pyc_files) 166 167 # Sourceless precompiled builds omit the main py file from outputs, so 168 # main has to be pointed to the precompiled main instead. 169 if (main_py not in precompile_result.keep_srcs and 170 PycCollectionAttr.is_pyc_collection_enabled(ctx)): 171 main_py = precompile_result.py_to_pyc_map[main_py] 172 173 executable = _declare_executable_file(ctx) 174 default_outputs = builders.DepsetBuilder() 175 default_outputs.add(executable) 176 default_outputs.add(precompile_result.keep_srcs) 177 default_outputs.add(required_pyc_files) 178 179 imports = collect_imports(ctx, semantics) 180 181 runtime_details = _get_runtime_details(ctx, semantics) 182 if ctx.configuration.coverage_enabled: 183 extra_deps = semantics.get_coverage_deps(ctx, runtime_details) 184 else: 185 extra_deps = [] 186 187 # The debugger dependency should be prevented by select() config elsewhere, 188 # but just to be safe, also guard against adding it to the output here. 189 if not _is_tool_config(ctx): 190 extra_deps.extend(semantics.get_debugger_deps(ctx, runtime_details)) 191 192 cc_details = semantics.get_cc_details_for_binary(ctx, extra_deps = extra_deps) 193 native_deps_details = _get_native_deps_details( 194 ctx, 195 semantics = semantics, 196 cc_details = cc_details, 197 is_test = is_test, 198 ) 199 runfiles_details = _get_base_runfiles_for_binary( 200 ctx, 201 executable = executable, 202 extra_deps = extra_deps, 203 required_py_files = required_py_files, 204 required_pyc_files = required_pyc_files, 205 implicit_pyc_files = implicit_pyc_files, 206 implicit_pyc_source_files = implicit_pyc_source_files, 207 extra_common_runfiles = [ 208 runtime_details.runfiles, 209 cc_details.extra_runfiles, 210 native_deps_details.runfiles, 211 semantics.get_extra_common_runfiles_for_binary(ctx), 212 ], 213 semantics = semantics, 214 ) 215 exec_result = semantics.create_executable( 216 ctx, 217 executable = executable, 218 main_py = main_py, 219 imports = imports, 220 is_test = is_test, 221 runtime_details = runtime_details, 222 cc_details = cc_details, 223 native_deps_details = native_deps_details, 224 runfiles_details = runfiles_details, 225 ) 226 default_outputs.add(exec_result.extra_files_to_build) 227 228 extra_exec_runfiles = exec_result.extra_runfiles.merge( 229 ctx.runfiles(transitive_files = exec_result.extra_files_to_build), 230 ) 231 232 # Copy any existing fields in case of company patches. 233 runfiles_details = struct(**( 234 structs.to_dict(runfiles_details) | dict( 235 default_runfiles = runfiles_details.default_runfiles.merge(extra_exec_runfiles), 236 data_runfiles = runfiles_details.data_runfiles.merge(extra_exec_runfiles), 237 ) 238 )) 239 240 return _create_providers( 241 ctx = ctx, 242 executable = executable, 243 runfiles_details = runfiles_details, 244 main_py = main_py, 245 imports = imports, 246 required_py_files = required_py_files, 247 required_pyc_files = required_pyc_files, 248 implicit_pyc_files = implicit_pyc_files, 249 implicit_pyc_source_files = implicit_pyc_source_files, 250 default_outputs = default_outputs.build(), 251 runtime_details = runtime_details, 252 cc_info = cc_details.cc_info_for_propagating, 253 inherited_environment = inherited_environment, 254 semantics = semantics, 255 output_groups = exec_result.output_groups, 256 ) 257 258def _get_build_info(ctx, cc_toolchain): 259 build_info_files = py_internal.cc_toolchain_build_info_files(cc_toolchain) 260 if cc_helper.is_stamping_enabled(ctx): 261 # Makes the target depend on BUILD_INFO_KEY, which helps to discover stamped targets 262 # See b/326620485 for more details. 263 ctx.version_file # buildifier: disable=no-effect 264 return build_info_files.non_redacted_build_info_files.to_list() 265 else: 266 return build_info_files.redacted_build_info_files.to_list() 267 268def _validate_executable(ctx): 269 if ctx.attr.python_version != "PY3": 270 fail("It is not allowed to use Python 2") 271 272def _declare_executable_file(ctx): 273 if target_platform_has_any_constraint(ctx, ctx.attr._windows_constraints): 274 executable = ctx.actions.declare_file(ctx.label.name + ".exe") 275 else: 276 executable = ctx.actions.declare_file(ctx.label.name) 277 278 return executable 279 280def _get_runtime_details(ctx, semantics): 281 """Gets various information about the Python runtime to use. 282 283 While most information comes from the toolchain, various legacy and 284 compatibility behaviors require computing some other information. 285 286 Args: 287 ctx: Rule ctx 288 semantics: A `BinarySemantics` struct; see `create_binary_semantics_struct` 289 290 Returns: 291 A struct; see inline-field comments of the return value for details. 292 """ 293 294 # Bazel has --python_path. This flag has a computed default of "python" when 295 # its actual default is null (see 296 # BazelPythonConfiguration.java#getPythonPath). This flag is only used if 297 # toolchains are not enabled and `--python_top` isn't set. Note that Google 298 # used to have a variant of this named --python_binary, but it has since 299 # been removed. 300 # 301 # TOOD(bazelbuild/bazel#7901): Remove this once --python_path flag is removed. 302 303 if IS_BAZEL: 304 flag_interpreter_path = ctx.fragments.bazel_py.python_path 305 toolchain_runtime, effective_runtime = _maybe_get_runtime_from_ctx(ctx) 306 if not effective_runtime: 307 # Clear these just in case 308 toolchain_runtime = None 309 effective_runtime = None 310 311 else: # Google code path 312 flag_interpreter_path = None 313 toolchain_runtime, effective_runtime = _maybe_get_runtime_from_ctx(ctx) 314 if not effective_runtime: 315 fail("Unable to find Python runtime") 316 317 if effective_runtime: 318 direct = [] # List of files 319 transitive = [] # List of depsets 320 if effective_runtime.interpreter: 321 direct.append(effective_runtime.interpreter) 322 transitive.append(effective_runtime.files) 323 324 if ctx.configuration.coverage_enabled: 325 if effective_runtime.coverage_tool: 326 direct.append(effective_runtime.coverage_tool) 327 if effective_runtime.coverage_files: 328 transitive.append(effective_runtime.coverage_files) 329 runtime_files = depset(direct = direct, transitive = transitive) 330 else: 331 runtime_files = depset() 332 333 executable_interpreter_path = semantics.get_interpreter_path( 334 ctx, 335 runtime = effective_runtime, 336 flag_interpreter_path = flag_interpreter_path, 337 ) 338 339 return struct( 340 # Optional PyRuntimeInfo: The runtime found from toolchain resolution. 341 # This may be None because, within Google, toolchain resolution isn't 342 # yet enabled. 343 toolchain_runtime = toolchain_runtime, 344 # Optional PyRuntimeInfo: The runtime that should be used. When 345 # toolchain resolution is enabled, this is the same as 346 # `toolchain_resolution`. Otherwise, this probably came from the 347 # `_python_top` attribute that the Google implementation still uses. 348 # This is separate from `toolchain_runtime` because toolchain_runtime 349 # is propagated as a provider, while non-toolchain runtimes are not. 350 effective_runtime = effective_runtime, 351 # str; Path to the Python interpreter to use for running the executable 352 # itself (not the bootstrap script). Either an absolute path (which 353 # means it is platform-specific), or a runfiles-relative path (which 354 # means the interpreter should be within `runtime_files`) 355 executable_interpreter_path = executable_interpreter_path, 356 # runfiles: Additional runfiles specific to the runtime that should 357 # be included. For in-build runtimes, this shold include the interpreter 358 # and any supporting files. 359 runfiles = ctx.runfiles(transitive_files = runtime_files), 360 ) 361 362def _maybe_get_runtime_from_ctx(ctx): 363 """Finds the PyRuntimeInfo from the toolchain or attribute, if available. 364 365 Returns: 366 2-tuple of toolchain_runtime, effective_runtime 367 """ 368 if ctx.fragments.py.use_toolchains: 369 toolchain = ctx.toolchains[TOOLCHAIN_TYPE] 370 371 if not hasattr(toolchain, "py3_runtime"): 372 fail("Python toolchain field 'py3_runtime' is missing") 373 if not toolchain.py3_runtime: 374 fail("Python toolchain missing py3_runtime") 375 py3_runtime = toolchain.py3_runtime 376 377 # Hack around the fact that the autodetecting Python toolchain, which is 378 # automatically registered, does not yet support Windows. In this case, 379 # we want to return null so that _get_interpreter_path falls back on 380 # --python_path. See tools/python/toolchain.bzl. 381 # TODO(#7844): Remove this hack when the autodetecting toolchain has a 382 # Windows implementation. 383 if py3_runtime.interpreter_path == "/_magic_pyruntime_sentinel_do_not_use": 384 return None, None 385 386 if py3_runtime.python_version != "PY3": 387 fail("Python toolchain py3_runtime must be python_version=PY3, got {}".format( 388 py3_runtime.python_version, 389 )) 390 toolchain_runtime = toolchain.py3_runtime 391 effective_runtime = toolchain_runtime 392 else: 393 toolchain_runtime = None 394 attr_target = getattr(ctx.attr, PY_RUNTIME_ATTR_NAME) 395 396 # In Bazel, --python_top is null by default. 397 if attr_target and PyRuntimeInfo in attr_target: 398 effective_runtime = attr_target[PyRuntimeInfo] 399 else: 400 return None, None 401 402 return toolchain_runtime, effective_runtime 403 404def _get_base_runfiles_for_binary( 405 ctx, 406 *, 407 executable, 408 extra_deps, 409 required_py_files, 410 required_pyc_files, 411 implicit_pyc_files, 412 implicit_pyc_source_files, 413 extra_common_runfiles, 414 semantics): 415 """Returns the set of runfiles necessary prior to executable creation. 416 417 NOTE: The term "common runfiles" refers to the runfiles that are common to 418 runfiles_without_exe, default_runfiles, and data_runfiles. 419 420 Args: 421 ctx: The rule ctx. 422 executable: The main executable output. 423 extra_deps: List of Targets; additional targets whose runfiles 424 will be added to the common runfiles. 425 required_py_files: `depset[File]` the direct, `.py` sources for the 426 target that **must** be included by downstream targets. This should 427 only be Python source files. It should not include pyc files. 428 required_pyc_files: `depset[File]` the direct `.pyc` files this target 429 produces. 430 implicit_pyc_files: `depset[File]` pyc files that are only used if pyc 431 collection is enabled. 432 implicit_pyc_source_files: `depset[File]` source files for implicit pyc 433 files that are used when the implicit pyc files are not. 434 extra_common_runfiles: List of runfiles; additional runfiles that 435 will be added to the common runfiles. 436 semantics: A `BinarySemantics` struct; see `create_binary_semantics_struct`. 437 438 Returns: 439 struct with attributes: 440 * default_runfiles: The default runfiles 441 * data_runfiles: The data runfiles 442 * runfiles_without_exe: The default runfiles, but without the executable 443 or files specific to the original program/executable. 444 * build_data_file: A file with build stamp information if stamping is enabled, otherwise 445 None. 446 """ 447 common_runfiles = builders.RunfilesBuilder() 448 common_runfiles.files.add(required_py_files) 449 common_runfiles.files.add(required_pyc_files) 450 pyc_collection_enabled = PycCollectionAttr.is_pyc_collection_enabled(ctx) 451 if pyc_collection_enabled: 452 common_runfiles.files.add(implicit_pyc_files) 453 else: 454 common_runfiles.files.add(implicit_pyc_source_files) 455 456 for dep in (ctx.attr.deps + extra_deps): 457 if not (PyInfo in dep or (BuiltinPyInfo != None and BuiltinPyInfo in dep)): 458 continue 459 info = dep[PyInfo] if PyInfo in dep else dep[BuiltinPyInfo] 460 common_runfiles.files.add(info.transitive_sources) 461 462 # Everything past this won't work with BuiltinPyInfo 463 if not hasattr(info, "transitive_pyc_files"): 464 continue 465 466 common_runfiles.files.add(info.transitive_pyc_files) 467 if pyc_collection_enabled: 468 common_runfiles.files.add(info.transitive_implicit_pyc_files) 469 else: 470 common_runfiles.files.add(info.transitive_implicit_pyc_source_files) 471 472 common_runfiles.runfiles.append(collect_runfiles(ctx)) 473 if extra_deps: 474 common_runfiles.add_targets(extra_deps) 475 common_runfiles.add(extra_common_runfiles) 476 477 common_runfiles = common_runfiles.build(ctx) 478 479 if semantics.should_create_init_files(ctx): 480 common_runfiles = _py_builtins.merge_runfiles_with_generated_inits_empty_files_supplier( 481 ctx = ctx, 482 runfiles = common_runfiles, 483 ) 484 485 # Don't include build_data.txt in the non-exe runfiles. The build data 486 # may contain program-specific content (e.g. target name). 487 runfiles_with_exe = common_runfiles.merge(ctx.runfiles([executable])) 488 489 # Don't include build_data.txt in data runfiles. This allows binaries to 490 # contain other binaries while still using the same fixed location symlink 491 # for the build_data.txt file. Really, the fixed location symlink should be 492 # removed and another way found to locate the underlying build data file. 493 data_runfiles = runfiles_with_exe 494 495 if is_stamping_enabled(ctx, semantics) and semantics.should_include_build_data(ctx): 496 build_data_file, build_data_runfiles = _create_runfiles_with_build_data( 497 ctx, 498 semantics.get_central_uncachable_version_file(ctx), 499 semantics.get_extra_write_build_data_env(ctx), 500 ) 501 default_runfiles = runfiles_with_exe.merge(build_data_runfiles) 502 else: 503 build_data_file = None 504 default_runfiles = runfiles_with_exe 505 506 return struct( 507 runfiles_without_exe = common_runfiles, 508 default_runfiles = default_runfiles, 509 build_data_file = build_data_file, 510 data_runfiles = data_runfiles, 511 ) 512 513def _create_runfiles_with_build_data( 514 ctx, 515 central_uncachable_version_file, 516 extra_write_build_data_env): 517 build_data_file = _write_build_data( 518 ctx, 519 central_uncachable_version_file, 520 extra_write_build_data_env, 521 ) 522 build_data_runfiles = ctx.runfiles(symlinks = { 523 BUILD_DATA_SYMLINK_PATH: build_data_file, 524 }) 525 return build_data_file, build_data_runfiles 526 527def _write_build_data(ctx, central_uncachable_version_file, extra_write_build_data_env): 528 # TODO: Remove this logic when a central file is always available 529 if not central_uncachable_version_file: 530 version_file = ctx.actions.declare_file(ctx.label.name + "-uncachable_version_file.txt") 531 _py_builtins.copy_without_caching( 532 ctx = ctx, 533 read_from = ctx.version_file, 534 write_to = version_file, 535 ) 536 else: 537 version_file = central_uncachable_version_file 538 539 direct_inputs = [ctx.info_file, version_file] 540 541 # A "constant metadata" file is basically a special file that doesn't 542 # support change detection logic and reports that it is unchanged. i.e., it 543 # behaves like ctx.version_file and is ignored when computing "what inputs 544 # changed" (see https://bazel.build/docs/user-manual#workspace-status). 545 # 546 # We do this so that consumers of the final build data file don't have 547 # to transitively rebuild everything -- the `uncachable_version_file` file 548 # isn't cachable, which causes the build data action to always re-run. 549 # 550 # While this technically means a binary could have stale build info, 551 # it ends up not mattering in practice because the volatile information 552 # doesn't meaningfully effect other outputs. 553 # 554 # This is also done for performance and Make It work reasons: 555 # * Passing the transitive dependencies into the action requires passing 556 # the runfiles, but actions don't directly accept runfiles. While 557 # flattening the depsets can be deferred, accessing the 558 # `runfiles.empty_filenames` attribute will will invoke the empty 559 # file supplier a second time, which is too much of a memory and CPU 560 # performance hit. 561 # * Some targets specify a directory in `data`, which is unsound, but 562 # mostly works. Google's RBE, unfortunately, rejects it. 563 # * A binary's transitive closure may be so large that it exceeds 564 # Google RBE limits for action inputs. 565 build_data = _py_builtins.declare_constant_metadata_file( 566 ctx = ctx, 567 name = ctx.label.name + ".build_data.txt", 568 root = ctx.bin_dir, 569 ) 570 571 ctx.actions.run( 572 executable = ctx.executable._build_data_gen, 573 env = dicts.add({ 574 # NOTE: ctx.info_file is undocumented; see 575 # https://github.com/bazelbuild/bazel/issues/9363 576 "INFO_FILE": ctx.info_file.path, 577 "OUTPUT": build_data.path, 578 "PLATFORM": cc_helper.find_cpp_toolchain(ctx).toolchain_id, 579 "TARGET": str(ctx.label), 580 "VERSION_FILE": version_file.path, 581 }, extra_write_build_data_env), 582 inputs = depset( 583 direct = direct_inputs, 584 ), 585 outputs = [build_data], 586 mnemonic = "PyWriteBuildData", 587 progress_message = "Generating %{label} build_data.txt", 588 ) 589 return build_data 590 591def _get_native_deps_details(ctx, *, semantics, cc_details, is_test): 592 if not semantics.should_build_native_deps_dso(ctx): 593 return struct(dso = None, runfiles = ctx.runfiles()) 594 595 cc_info = cc_details.cc_info_for_self_link 596 597 if not cc_info.linking_context.linker_inputs: 598 return struct(dso = None, runfiles = ctx.runfiles()) 599 600 dso = ctx.actions.declare_file(semantics.get_native_deps_dso_name(ctx)) 601 share_native_deps = py_internal.share_native_deps(ctx) 602 cc_feature_config = cc_details.feature_config 603 if share_native_deps: 604 linked_lib = _create_shared_native_deps_dso( 605 ctx, 606 cc_info = cc_info, 607 is_test = is_test, 608 requested_features = cc_feature_config.requested_features, 609 feature_configuration = cc_feature_config.feature_configuration, 610 cc_toolchain = cc_details.cc_toolchain, 611 ) 612 ctx.actions.symlink( 613 output = dso, 614 target_file = linked_lib, 615 progress_message = "Symlinking shared native deps for %{label}", 616 ) 617 else: 618 linked_lib = dso 619 620 # The regular cc_common.link API can't be used because several 621 # args are private-use only; see # private comments 622 py_internal.link( 623 name = ctx.label.name, 624 actions = ctx.actions, 625 linking_contexts = [cc_info.linking_context], 626 output_type = "dynamic_library", 627 never_link = True, # private 628 native_deps = True, # private 629 feature_configuration = cc_feature_config.feature_configuration, 630 cc_toolchain = cc_details.cc_toolchain, 631 test_only_target = is_test, # private 632 stamp = 1 if is_stamping_enabled(ctx, semantics) else 0, 633 main_output = linked_lib, # private 634 use_shareable_artifact_factory = True, # private 635 # NOTE: Only flags not captured by cc_info.linking_context need to 636 # be manually passed 637 user_link_flags = semantics.get_native_deps_user_link_flags(ctx), 638 ) 639 return struct( 640 dso = dso, 641 runfiles = ctx.runfiles(files = [dso]), 642 ) 643 644def _create_shared_native_deps_dso( 645 ctx, 646 *, 647 cc_info, 648 is_test, 649 feature_configuration, 650 requested_features, 651 cc_toolchain): 652 linkstamps = py_internal.linking_context_linkstamps(cc_info.linking_context) 653 654 partially_disabled_thin_lto = ( 655 cc_common.is_enabled( 656 feature_name = "thin_lto_linkstatic_tests_use_shared_nonlto_backends", 657 feature_configuration = feature_configuration, 658 ) and not cc_common.is_enabled( 659 feature_name = "thin_lto_all_linkstatic_use_shared_nonlto_backends", 660 feature_configuration = feature_configuration, 661 ) 662 ) 663 dso_hash = _get_shared_native_deps_hash( 664 linker_inputs = cc_helper.get_static_mode_params_for_dynamic_library_libraries( 665 depset([ 666 lib 667 for linker_input in cc_info.linking_context.linker_inputs.to_list() 668 for lib in linker_input.libraries 669 ]), 670 ), 671 link_opts = [ 672 flag 673 for input in cc_info.linking_context.linker_inputs.to_list() 674 for flag in input.user_link_flags 675 ], 676 linkstamps = [ 677 py_internal.linkstamp_file(linkstamp) 678 for linkstamp in linkstamps.to_list() 679 ], 680 build_info_artifacts = _get_build_info(ctx, cc_toolchain) if linkstamps else [], 681 features = requested_features, 682 is_test_target_partially_disabled_thin_lto = is_test and partially_disabled_thin_lto, 683 ) 684 return py_internal.declare_shareable_artifact(ctx, "_nativedeps/%x.so" % dso_hash) 685 686# This is a minimal version of NativeDepsHelper.getSharedNativeDepsPath, see 687# com.google.devtools.build.lib.rules.nativedeps.NativeDepsHelper#getSharedNativeDepsPath 688# The basic idea is to take all the inputs that affect linking and encode (via 689# hashing) them into the filename. 690# TODO(b/234232820): The settings that affect linking must be kept in sync with the actual 691# C++ link action. For more information, see the large descriptive comment on 692# NativeDepsHelper#getSharedNativeDepsPath. 693def _get_shared_native_deps_hash( 694 *, 695 linker_inputs, 696 link_opts, 697 linkstamps, 698 build_info_artifacts, 699 features, 700 is_test_target_partially_disabled_thin_lto): 701 # NOTE: We use short_path because the build configuration root in which 702 # files are always created already captures the configuration-specific 703 # parts, so no need to include them manually. 704 parts = [] 705 for artifact in linker_inputs: 706 parts.append(artifact.short_path) 707 parts.append(str(len(link_opts))) 708 parts.extend(link_opts) 709 for artifact in linkstamps: 710 parts.append(artifact.short_path) 711 for artifact in build_info_artifacts: 712 parts.append(artifact.short_path) 713 parts.extend(sorted(features)) 714 715 # Sharing of native dependencies may cause an {@link 716 # ActionConflictException} when ThinLTO is disabled for test and test-only 717 # targets that are statically linked, but enabled for other statically 718 # linked targets. This happens in case the artifacts for the shared native 719 # dependency are output by {@link Action}s owned by the non-test and test 720 # targets both. To fix this, we allow creation of multiple artifacts for the 721 # shared native library - one shared among the test and test-only targets 722 # where ThinLTO is disabled, and the other shared among other targets where 723 # ThinLTO is enabled. See b/138118275 724 parts.append("1" if is_test_target_partially_disabled_thin_lto else "0") 725 726 return hash("".join(parts)) 727 728def determine_main(ctx): 729 """Determine the main entry point .py source file. 730 731 Args: 732 ctx: The rule ctx. 733 734 Returns: 735 Artifact; the main file. If one can't be found, an error is raised. 736 """ 737 if ctx.attr.main: 738 proposed_main = ctx.attr.main.label.name 739 if not proposed_main.endswith(tuple(ALLOWED_MAIN_EXTENSIONS)): 740 fail("main must end in '.py'") 741 else: 742 if ctx.label.name.endswith(".py"): 743 fail("name must not end in '.py'") 744 proposed_main = ctx.label.name + ".py" 745 746 main_files = [src for src in ctx.files.srcs if _path_endswith(src.short_path, proposed_main)] 747 if not main_files: 748 if ctx.attr.main: 749 fail("could not find '{}' as specified by 'main' attribute".format(proposed_main)) 750 else: 751 fail(("corresponding default '{}' does not appear in srcs. Add " + 752 "it or override default file name with a 'main' attribute").format( 753 proposed_main, 754 )) 755 756 elif len(main_files) > 1: 757 if ctx.attr.main: 758 fail(("file name '{}' specified by 'main' attributes matches multiple files. " + 759 "Matches: {}").format( 760 proposed_main, 761 csv([f.short_path for f in main_files]), 762 )) 763 else: 764 fail(("default main file '{}' matches multiple files in srcs. Perhaps specify " + 765 "an explicit file with 'main' attribute? Matches were: {}").format( 766 proposed_main, 767 csv([f.short_path for f in main_files]), 768 )) 769 return main_files[0] 770 771def _path_endswith(path, endswith): 772 # Use slash to anchor each path to prevent e.g. 773 # "ab/c.py".endswith("b/c.py") from incorrectly matching. 774 return ("/" + path).endswith("/" + endswith) 775 776def is_stamping_enabled(ctx, semantics): 777 """Tells if stamping is enabled or not. 778 779 Args: 780 ctx: The rule ctx 781 semantics: a semantics struct (see create_semantics_struct). 782 Returns: 783 bool; True if stamping is enabled, False if not. 784 """ 785 if _is_tool_config(ctx): 786 return False 787 788 stamp = ctx.attr.stamp 789 if stamp == 1: 790 return True 791 elif stamp == 0: 792 return False 793 elif stamp == -1: 794 return semantics.get_stamp_flag(ctx) 795 else: 796 fail("Unsupported `stamp` value: {}".format(stamp)) 797 798def _is_tool_config(ctx): 799 # NOTE: The is_tool_configuration() function is only usable by builtins. 800 # See https://github.com/bazelbuild/bazel/issues/14444 for the FR for 801 # a more public API. Until that's available, py_internal to the rescue. 802 return py_internal.is_tool_configuration(ctx) 803 804def _create_providers( 805 *, 806 ctx, 807 executable, 808 main_py, 809 required_py_files, 810 required_pyc_files, 811 implicit_pyc_files, 812 implicit_pyc_source_files, 813 default_outputs, 814 runfiles_details, 815 imports, 816 cc_info, 817 inherited_environment, 818 runtime_details, 819 output_groups, 820 semantics): 821 """Creates the providers an executable should return. 822 823 Args: 824 ctx: The rule ctx. 825 executable: File; the target's executable file. 826 main_py: File; the main .py entry point. 827 required_py_files: `depset[File]` the direct, `.py` sources for the 828 target that **must** be included by downstream targets. This should 829 only be Python source files. It should not include pyc files. 830 required_pyc_files: `depset[File]` the direct `.pyc` files this target 831 produces. 832 implicit_pyc_files: `depset[File]` pyc files that are only used if pyc 833 collection is enabled. 834 implicit_pyc_source_files: `depset[File]` source files for implicit pyc 835 files that are used when the implicit pyc files are not. 836 default_outputs: depset of Files; the files for DefaultInfo.files 837 runfiles_details: runfiles that will become the default and data runfiles. 838 imports: depset of strings; the import paths to propagate 839 cc_info: optional CcInfo; Linking information to propagate as 840 PyCcLinkParamsInfo. Note that only the linking information 841 is propagated, not the whole CcInfo. 842 inherited_environment: list of strings; Environment variable names 843 that should be inherited from the environment the executuble 844 is run within. 845 runtime_details: struct of runtime information; see _get_runtime_details() 846 output_groups: dict[str, depset[File]]; used to create OutputGroupInfo 847 semantics: BinarySemantics struct; see create_binary_semantics() 848 849 Returns: 850 A list of modern providers. 851 """ 852 providers = [ 853 DefaultInfo( 854 executable = executable, 855 files = default_outputs, 856 default_runfiles = _py_builtins.make_runfiles_respect_legacy_external_runfiles( 857 ctx, 858 runfiles_details.default_runfiles, 859 ), 860 data_runfiles = _py_builtins.make_runfiles_respect_legacy_external_runfiles( 861 ctx, 862 runfiles_details.data_runfiles, 863 ), 864 ), 865 create_instrumented_files_info(ctx), 866 _create_run_environment_info(ctx, inherited_environment), 867 PyExecutableInfo( 868 main = main_py, 869 runfiles_without_exe = runfiles_details.runfiles_without_exe, 870 build_data_file = runfiles_details.build_data_file, 871 interpreter_path = runtime_details.executable_interpreter_path, 872 ), 873 ] 874 875 # TODO(b/265840007): Make this non-conditional once Google enables 876 # --incompatible_use_python_toolchains. 877 if runtime_details.toolchain_runtime: 878 py_runtime_info = runtime_details.toolchain_runtime 879 providers.append(py_runtime_info) 880 881 # Re-add the builtin PyRuntimeInfo for compatibility to make 882 # transitioning easier, but only if it isn't already added because 883 # returning the same provider type multiple times is an error. 884 # NOTE: The PyRuntimeInfo from the toolchain could be a rules_python 885 # PyRuntimeInfo or a builtin PyRuntimeInfo -- a user could have used the 886 # builtin py_runtime rule or defined their own. We can't directly detect 887 # the type of the provider object, but the rules_python PyRuntimeInfo 888 # object has an extra attribute that the builtin one doesn't. 889 if hasattr(py_runtime_info, "interpreter_version_info") and BuiltinPyRuntimeInfo != None: 890 providers.append(BuiltinPyRuntimeInfo( 891 interpreter_path = py_runtime_info.interpreter_path, 892 interpreter = py_runtime_info.interpreter, 893 files = py_runtime_info.files, 894 coverage_tool = py_runtime_info.coverage_tool, 895 coverage_files = py_runtime_info.coverage_files, 896 python_version = py_runtime_info.python_version, 897 stub_shebang = py_runtime_info.stub_shebang, 898 bootstrap_template = py_runtime_info.bootstrap_template, 899 )) 900 901 # TODO(b/163083591): Remove the PyCcLinkParamsInfo once binaries-in-deps 902 # are cleaned up. 903 if cc_info: 904 providers.append( 905 PyCcLinkParamsInfo(cc_info = cc_info), 906 ) 907 908 py_info, deps_transitive_sources, builtin_py_info = create_py_info( 909 ctx, 910 required_py_files = required_py_files, 911 required_pyc_files = required_pyc_files, 912 implicit_pyc_files = implicit_pyc_files, 913 implicit_pyc_source_files = implicit_pyc_source_files, 914 imports = imports, 915 ) 916 917 # TODO(b/253059598): Remove support for extra actions; https://github.com/bazelbuild/bazel/issues/16455 918 listeners_enabled = _py_builtins.are_action_listeners_enabled(ctx) 919 if listeners_enabled: 920 _py_builtins.add_py_extra_pseudo_action( 921 ctx = ctx, 922 dependency_transitive_python_sources = deps_transitive_sources, 923 ) 924 925 providers.append(py_info) 926 if builtin_py_info: 927 providers.append(builtin_py_info) 928 providers.append(create_output_group_info(py_info.transitive_sources, output_groups)) 929 930 extra_providers = semantics.get_extra_providers( 931 ctx, 932 main_py = main_py, 933 runtime_details = runtime_details, 934 ) 935 providers.extend(extra_providers) 936 return providers 937 938def _create_run_environment_info(ctx, inherited_environment): 939 expanded_env = {} 940 for key, value in ctx.attr.env.items(): 941 expanded_env[key] = _py_builtins.expand_location_and_make_variables( 942 ctx = ctx, 943 attribute_name = "env[{}]".format(key), 944 expression = value, 945 targets = ctx.attr.data, 946 ) 947 return RunEnvironmentInfo( 948 environment = expanded_env, 949 inherited_environment = inherited_environment, 950 ) 951 952def create_base_executable_rule(*, attrs, fragments = [], **kwargs): 953 """Create a function for defining for Python binary/test targets. 954 955 Args: 956 attrs: Rule attributes 957 fragments: List of str; extra config fragments that are required. 958 **kwargs: Additional args to pass onto `rule()` 959 960 Returns: 961 A rule function 962 """ 963 if "py" not in fragments: 964 # The list might be frozen, so use concatentation 965 fragments = fragments + ["py"] 966 kwargs.setdefault("provides", []).append(PyExecutableInfo) 967 kwargs["exec_groups"] = REQUIRED_EXEC_GROUPS | (kwargs.get("exec_groups") or {}) 968 return rule( 969 # TODO: add ability to remove attrs, i.e. for imports attr 970 attrs = dicts.add(EXECUTABLE_ATTRS, attrs), 971 toolchains = [ 972 TOOLCHAIN_TYPE, 973 config_common.toolchain_type(EXEC_TOOLS_TOOLCHAIN_TYPE, mandatory = False), 974 ] + _CC_TOOLCHAINS, 975 fragments = fragments, 976 **kwargs 977 ) 978 979def cc_configure_features( 980 ctx, 981 *, 982 cc_toolchain, 983 extra_features, 984 linking_mode = "static_linking_mode"): 985 """Configure C++ features for Python purposes. 986 987 Args: 988 ctx: Rule ctx 989 cc_toolchain: The CcToolchain the target is using. 990 extra_features: list of strings; additional features to request be 991 enabled. 992 linking_mode: str; either "static_linking_mode" or 993 "dynamic_linking_mode". Specifies the linking mode feature for 994 C++ linking. 995 996 Returns: 997 struct of the feature configuration and all requested features. 998 """ 999 requested_features = [linking_mode] 1000 requested_features.extend(extra_features) 1001 requested_features.extend(ctx.features) 1002 if "legacy_whole_archive" not in ctx.disabled_features: 1003 requested_features.append("legacy_whole_archive") 1004 feature_configuration = cc_common.configure_features( 1005 ctx = ctx, 1006 cc_toolchain = cc_toolchain, 1007 requested_features = requested_features, 1008 unsupported_features = ctx.disabled_features, 1009 ) 1010 return struct( 1011 feature_configuration = feature_configuration, 1012 requested_features = requested_features, 1013 ) 1014 1015only_exposed_for_google_internal_reason = struct( 1016 create_runfiles_with_build_data = _create_runfiles_with_build_data, 1017) 1018