1# Copyright 2023 The Bazel Authors. All rights reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"pip module extension for use with bzlmod" 16 17load("@bazel_features//:features.bzl", "bazel_features") 18load("@pythons_hub//:interpreters.bzl", "INTERPRETER_LABELS") 19load("//python/private:auth.bzl", "AUTH_ATTRS") 20load("//python/private:normalize_name.bzl", "normalize_name") 21load("//python/private:repo_utils.bzl", "repo_utils") 22load("//python/private:semver.bzl", "semver") 23load("//python/private:version_label.bzl", "version_label") 24load(":attrs.bzl", "use_isolated") 25load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS") 26load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") 27load(":parse_requirements.bzl", "parse_requirements") 28load(":parse_whl_name.bzl", "parse_whl_name") 29load(":pip_repository_attrs.bzl", "ATTRS") 30load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") 31load(":simpleapi_download.bzl", "simpleapi_download") 32load(":whl_config_setting.bzl", "whl_config_setting") 33load(":whl_library.bzl", "whl_library") 34load(":whl_repo_name.bzl", "pypi_repo_name", "whl_repo_name") 35 36def _major_minor_version(version): 37 version = semver(version) 38 return "{}.{}".format(version.major, version.minor) 39 40def _whl_mods_impl(whl_mods_dict): 41 """Implementation of the pip.whl_mods tag class. 42 43 This creates the JSON files used to modify the creation of different wheels. 44""" 45 for hub_name, whl_maps in whl_mods_dict.items(): 46 whl_mods = {} 47 48 # create a struct that we can pass to the _whl_mods_repo rule 49 # to create the different JSON files. 50 for whl_name, mods in whl_maps.items(): 51 whl_mods[whl_name] = json.encode(struct( 52 additive_build_content = mods.build_content, 53 copy_files = mods.copy_files, 54 copy_executables = mods.copy_executables, 55 data = mods.data, 56 data_exclude_glob = mods.data_exclude_glob, 57 srcs_exclude_glob = mods.srcs_exclude_glob, 58 )) 59 60 _whl_mods_repo( 61 name = hub_name, 62 whl_mods = whl_mods, 63 ) 64 65def _create_whl_repos( 66 module_ctx, 67 *, 68 pip_attr, 69 whl_overrides, 70 simpleapi_cache, 71 evaluate_markers = evaluate_markers, 72 available_interpreters = INTERPRETER_LABELS, 73 simpleapi_download = simpleapi_download): 74 """create all of the whl repositories 75 76 Args: 77 module_ctx: {type}`module_ctx`. 78 pip_attr: {type}`struct` - the struct that comes from the tag class iteration. 79 whl_overrides: {type}`dict[str, struct]` - per-wheel overrides. 80 simpleapi_cache: {type}`dict` - an opaque dictionary used for caching the results from calling 81 SimpleAPI evaluating all of the tag class invocations {bzl:obj}`pip.parse`. 82 evaluate_markers: the function to use to evaluate markers. 83 simpleapi_download: Used for testing overrides 84 available_interpreters: {type}`dict[str, Label]` The dictionary of available 85 interpreters that have been registered using the `python` bzlmod extension. 86 The keys are in the form `python_{snake_case_version}_host`. This is to be 87 used during the `repository_rule` and must be always compatible with the host. 88 89 Returns a {type}`struct` with the following attributes: 90 whl_map: {type}`dict[str, list[struct]]` the output is keyed by the 91 normalized package name and the values are the instances of the 92 {bzl:obj}`whl_config_setting` return values. 93 exposed_packages: {type}`dict[str, Any]` this is just a way to 94 represent a set of string values. 95 whl_libraries: {type}`dict[str, dict[str, Any]]` the keys are the 96 aparent repository names for the hub repo and the values are the 97 arguments that will be passed to {bzl:obj}`whl_library` repository 98 rule. 99 is_reproducible: {type}`bool` set to True if does not make calls to the 100 internet to evaluate the requirements files. 101 """ 102 logger = repo_utils.logger(module_ctx, "pypi:create_whl_repos") 103 python_interpreter_target = pip_attr.python_interpreter_target 104 is_reproducible = True 105 106 # containers to aggregate outputs from this function 107 whl_map = {} 108 exposed_packages = {} 109 extra_aliases = { 110 whl_name: {alias: True for alias in aliases} 111 for whl_name, aliases in pip_attr.extra_hub_aliases.items() 112 } 113 whl_libraries = {} 114 115 # if we do not have the python_interpreter set in the attributes 116 # we programmatically find it. 117 hub_name = pip_attr.hub_name 118 if python_interpreter_target == None and not pip_attr.python_interpreter: 119 python_name = "python_{}_host".format( 120 pip_attr.python_version.replace(".", "_"), 121 ) 122 if python_name not in available_interpreters: 123 fail(( 124 "Unable to find interpreter for pip hub '{hub_name}' for " + 125 "python_version={version}: Make sure a corresponding " + 126 '`python.toolchain(python_version="{version}")` call exists.' + 127 "Expected to find {python_name} among registered versions:\n {labels}" 128 ).format( 129 hub_name = hub_name, 130 version = pip_attr.python_version, 131 python_name = python_name, 132 labels = " \n".join(available_interpreters), 133 )) 134 python_interpreter_target = available_interpreters[python_name] 135 136 pip_name = "{}_{}".format( 137 hub_name, 138 version_label(pip_attr.python_version), 139 ) 140 major_minor = _major_minor_version(pip_attr.python_version) 141 142 whl_modifications = {} 143 if pip_attr.whl_modifications != None: 144 for mod, whl_name in pip_attr.whl_modifications.items(): 145 whl_modifications[normalize_name(whl_name)] = mod 146 147 if pip_attr.experimental_requirement_cycles: 148 requirement_cycles = { 149 name: [normalize_name(whl_name) for whl_name in whls] 150 for name, whls in pip_attr.experimental_requirement_cycles.items() 151 } 152 153 whl_group_mapping = { 154 whl_name: group_name 155 for group_name, group_whls in requirement_cycles.items() 156 for whl_name in group_whls 157 } 158 else: 159 whl_group_mapping = {} 160 requirement_cycles = {} 161 162 # Create a new wheel library for each of the different whls 163 164 get_index_urls = None 165 if pip_attr.experimental_index_url: 166 get_index_urls = lambda ctx, distributions: simpleapi_download( 167 ctx, 168 attr = struct( 169 index_url = pip_attr.experimental_index_url, 170 extra_index_urls = pip_attr.experimental_extra_index_urls or [], 171 index_url_overrides = pip_attr.experimental_index_url_overrides or {}, 172 sources = distributions, 173 envsubst = pip_attr.envsubst, 174 # Auth related info 175 netrc = pip_attr.netrc, 176 auth_patterns = pip_attr.auth_patterns, 177 ), 178 cache = simpleapi_cache, 179 parallel_download = pip_attr.parallel_download, 180 ) 181 182 requirements_by_platform = parse_requirements( 183 module_ctx, 184 requirements_by_platform = requirements_files_by_platform( 185 requirements_by_platform = pip_attr.requirements_by_platform, 186 requirements_linux = pip_attr.requirements_linux, 187 requirements_lock = pip_attr.requirements_lock, 188 requirements_osx = pip_attr.requirements_darwin, 189 requirements_windows = pip_attr.requirements_windows, 190 extra_pip_args = pip_attr.extra_pip_args, 191 python_version = major_minor, 192 logger = logger, 193 ), 194 extra_pip_args = pip_attr.extra_pip_args, 195 get_index_urls = get_index_urls, 196 # NOTE @aignas 2024-08-02: , we will execute any interpreter that we find either 197 # in the PATH or if specified as a label. We will configure the env 198 # markers when evaluating the requirement lines based on the output 199 # from the `requirements_files_by_platform` which should have something 200 # similar to: 201 # { 202 # "//:requirements.txt": ["cp311_linux_x86_64", ...] 203 # } 204 # 205 # We know the target python versions that we need to evaluate the 206 # markers for and thus we don't need to use multiple python interpreter 207 # instances to perform this manipulation. This function should be executed 208 # only once by the underlying code to minimize the overhead needed to 209 # spin up a Python interpreter. 210 evaluate_markers = lambda module_ctx, requirements: evaluate_markers( 211 module_ctx, 212 requirements = requirements, 213 python_interpreter = pip_attr.python_interpreter, 214 python_interpreter_target = python_interpreter_target, 215 srcs = pip_attr._evaluate_markers_srcs, 216 logger = logger, 217 ), 218 logger = logger, 219 ) 220 221 for whl_name, requirements in requirements_by_platform.items(): 222 whl_name = normalize_name(whl_name) 223 224 group_name = whl_group_mapping.get(whl_name) 225 group_deps = requirement_cycles.get(group_name, []) 226 227 # Construct args separately so that the lock file can be smaller and does not include unused 228 # attrs. 229 whl_library_args = dict( 230 repo = pip_name, 231 dep_template = "@{}//{{name}}:{{target}}".format(hub_name), 232 ) 233 maybe_args = dict( 234 # The following values are safe to omit if they have false like values 235 annotation = whl_modifications.get(whl_name), 236 download_only = pip_attr.download_only, 237 enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs, 238 environment = pip_attr.environment, 239 envsubst = pip_attr.envsubst, 240 experimental_target_platforms = pip_attr.experimental_target_platforms, 241 group_deps = group_deps, 242 group_name = group_name, 243 pip_data_exclude = pip_attr.pip_data_exclude, 244 python_interpreter = pip_attr.python_interpreter, 245 python_interpreter_target = python_interpreter_target, 246 whl_patches = { 247 p: json.encode(args) 248 for p, args in whl_overrides.get(whl_name, {}).items() 249 }, 250 ) 251 whl_library_args.update({k: v for k, v in maybe_args.items() if v}) 252 maybe_args_with_default = dict( 253 # The following values have defaults next to them 254 isolated = (use_isolated(module_ctx, pip_attr), True), 255 quiet = (pip_attr.quiet, True), 256 timeout = (pip_attr.timeout, 600), 257 ) 258 whl_library_args.update({ 259 k: v 260 for k, (v, default) in maybe_args_with_default.items() 261 if v != default 262 }) 263 264 is_exposed = False 265 if get_index_urls: 266 # TODO @aignas 2024-05-26: move to a separate function 267 found_something = False 268 for requirement in requirements: 269 is_exposed = is_exposed or requirement.is_exposed 270 dists = requirement.whls 271 if not pip_attr.download_only and requirement.sdist: 272 dists = dists + [requirement.sdist] 273 274 for distribution in dists: 275 found_something = True 276 is_reproducible = False 277 278 args = dict(whl_library_args) 279 if pip_attr.netrc: 280 args["netrc"] = pip_attr.netrc 281 if pip_attr.auth_patterns: 282 args["auth_patterns"] = pip_attr.auth_patterns 283 284 if not distribution.filename.endswith(".whl"): 285 # pip is not used to download wheels and the python 286 # `whl_library` helpers are only extracting things, however 287 # for sdists, they will be built by `pip`, so we still 288 # need to pass the extra args there. 289 args["extra_pip_args"] = requirement.extra_pip_args 290 291 # This is no-op because pip is not used to download the wheel. 292 args.pop("download_only", None) 293 294 repo_name = whl_repo_name(pip_name, distribution.filename, distribution.sha256) 295 args["requirement"] = requirement.srcs.requirement 296 args["urls"] = [distribution.url] 297 args["sha256"] = distribution.sha256 298 args["filename"] = distribution.filename 299 args["experimental_target_platforms"] = requirement.target_platforms 300 301 # Pure python wheels or sdists may need to have a platform here 302 target_platforms = None 303 if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"): 304 if len(requirements) > 1: 305 target_platforms = requirement.target_platforms 306 307 whl_libraries[repo_name] = args 308 309 whl_map.setdefault(whl_name, {})[whl_config_setting( 310 version = major_minor, 311 filename = distribution.filename, 312 target_platforms = target_platforms, 313 )] = repo_name 314 315 if found_something: 316 if is_exposed: 317 exposed_packages[whl_name] = None 318 continue 319 320 is_exposed = False 321 for requirement in requirements: 322 is_exposed = is_exposed or requirement.is_exposed 323 if get_index_urls: 324 logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line)) 325 326 args = dict(whl_library_args) # make a copy 327 args["requirement"] = requirement.requirement_line 328 if requirement.extra_pip_args: 329 args["extra_pip_args"] = requirement.extra_pip_args 330 331 if pip_attr.download_only: 332 args.setdefault("experimental_target_platforms", requirement.target_platforms) 333 334 target_platforms = requirement.target_platforms if len(requirements) > 1 else [] 335 repo_name = pypi_repo_name( 336 pip_name, 337 whl_name, 338 *target_platforms 339 ) 340 whl_libraries[repo_name] = args 341 whl_map.setdefault(whl_name, {})[whl_config_setting( 342 version = major_minor, 343 target_platforms = target_platforms or None, 344 )] = repo_name 345 346 if is_exposed: 347 exposed_packages[whl_name] = None 348 349 return struct( 350 is_reproducible = is_reproducible, 351 whl_map = whl_map, 352 exposed_packages = exposed_packages, 353 extra_aliases = extra_aliases, 354 whl_libraries = whl_libraries, 355 ) 356 357def parse_modules(module_ctx, _fail = fail, **kwargs): 358 """Implementation of parsing the tag classes for the extension and return a struct for registering repositories. 359 360 Args: 361 module_ctx: {type}`module_ctx` module context. 362 _fail: {type}`function` the failure function, mainly for testing. 363 **kwargs: Extra arguments passed to the layers below. 364 365 Returns: 366 A struct with the following attributes: 367 """ 368 whl_mods = {} 369 for mod in module_ctx.modules: 370 for whl_mod in mod.tags.whl_mods: 371 if whl_mod.whl_name in whl_mods.get(whl_mod.hub_name, {}): 372 # We cannot have the same wheel name in the same hub, as we 373 # will create the same JSON file name. 374 _fail("""\ 375Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format( 376 whl_mod.whl_name, 377 whl_mod.hub_name, 378 )) 379 return None 380 381 build_content = whl_mod.additive_build_content 382 if whl_mod.additive_build_content_file != None and whl_mod.additive_build_content != "": 383 _fail("""\ 384You cannot use both the additive_build_content and additive_build_content_file arguments at the same time. 385""") 386 return None 387 elif whl_mod.additive_build_content_file != None: 388 build_content = module_ctx.read(whl_mod.additive_build_content_file) 389 390 whl_mods.setdefault(whl_mod.hub_name, {})[whl_mod.whl_name] = struct( 391 build_content = build_content, 392 copy_files = whl_mod.copy_files, 393 copy_executables = whl_mod.copy_executables, 394 data = whl_mod.data, 395 data_exclude_glob = whl_mod.data_exclude_glob, 396 srcs_exclude_glob = whl_mod.srcs_exclude_glob, 397 ) 398 399 _overriden_whl_set = {} 400 whl_overrides = {} 401 for module in module_ctx.modules: 402 for attr in module.tags.override: 403 if not module.is_root: 404 fail("overrides are only supported in root modules") 405 406 if not attr.file.endswith(".whl"): 407 fail("Only whl overrides are supported at this time") 408 409 whl_name = normalize_name(parse_whl_name(attr.file).distribution) 410 411 if attr.file in _overriden_whl_set: 412 fail("Duplicate module overrides for '{}'".format(attr.file)) 413 _overriden_whl_set[attr.file] = None 414 415 for patch in attr.patches: 416 if whl_name not in whl_overrides: 417 whl_overrides[whl_name] = {} 418 419 if patch not in whl_overrides[whl_name]: 420 whl_overrides[whl_name][patch] = struct( 421 patch_strip = attr.patch_strip, 422 whls = [], 423 ) 424 425 whl_overrides[whl_name][patch].whls.append(attr.file) 426 427 # Used to track all the different pip hubs and the spoke pip Python 428 # versions. 429 pip_hub_map = {} 430 simpleapi_cache = {} 431 432 # Keeps track of all the hub's whl repos across the different versions. 433 # dict[hub, dict[whl, dict[version, str pip]]] 434 # Where hub, whl, and pip are the repo names 435 hub_whl_map = {} 436 hub_group_map = {} 437 exposed_packages = {} 438 extra_aliases = {} 439 whl_libraries = {} 440 441 is_reproducible = True 442 443 for mod in module_ctx.modules: 444 for pip_attr in mod.tags.parse: 445 hub_name = pip_attr.hub_name 446 if hub_name not in pip_hub_map: 447 pip_hub_map[pip_attr.hub_name] = struct( 448 module_name = mod.name, 449 python_versions = [pip_attr.python_version], 450 ) 451 elif pip_hub_map[hub_name].module_name != mod.name: 452 # We cannot have two hubs with the same name in different 453 # modules. 454 fail(( 455 "Duplicate cross-module pip hub named '{hub}': pip hub " + 456 "names must be unique across modules. First defined " + 457 "by module '{first_module}', second attempted by " + 458 "module '{second_module}'" 459 ).format( 460 hub = hub_name, 461 first_module = pip_hub_map[hub_name].module_name, 462 second_module = mod.name, 463 )) 464 465 elif pip_attr.python_version in pip_hub_map[hub_name].python_versions: 466 fail(( 467 "Duplicate pip python version '{version}' for hub " + 468 "'{hub}' in module '{module}': the Python versions " + 469 "used for a hub must be unique" 470 ).format( 471 hub = hub_name, 472 module = mod.name, 473 version = pip_attr.python_version, 474 )) 475 else: 476 pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) 477 478 out = _create_whl_repos( 479 module_ctx, 480 pip_attr = pip_attr, 481 simpleapi_cache = simpleapi_cache, 482 whl_overrides = whl_overrides, 483 **kwargs 484 ) 485 hub_whl_map.setdefault(hub_name, {}) 486 for key, settings in out.whl_map.items(): 487 for setting, repo in settings.items(): 488 hub_whl_map[hub_name].setdefault(key, {}).setdefault(repo, []).append(setting) 489 extra_aliases.setdefault(hub_name, {}) 490 for whl_name, aliases in out.extra_aliases.items(): 491 extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases) 492 exposed_packages.setdefault(hub_name, {}).update(out.exposed_packages) 493 whl_libraries.update(out.whl_libraries) 494 is_reproducible = is_reproducible and out.is_reproducible 495 496 # TODO @aignas 2024-04-05: how do we support different requirement 497 # cycles for different abis/oses? For now we will need the users to 498 # assume the same groups across all versions/platforms until we start 499 # using an alternative cycle resolution strategy. 500 hub_group_map[hub_name] = pip_attr.experimental_requirement_cycles 501 502 return struct( 503 # We sort so that the lock-file remains the same no matter the order of how the 504 # args are manipulated in the code going before. 505 whl_mods = dict(sorted(whl_mods.items())), 506 hub_whl_map = { 507 hub_name: { 508 whl_name: dict(settings) 509 for whl_name, settings in sorted(whl_map.items()) 510 } 511 for hub_name, whl_map in sorted(hub_whl_map.items()) 512 }, 513 hub_group_map = { 514 hub_name: { 515 key: sorted(values) 516 for key, values in sorted(group_map.items()) 517 } 518 for hub_name, group_map in sorted(hub_group_map.items()) 519 }, 520 exposed_packages = { 521 k: sorted(v) 522 for k, v in sorted(exposed_packages.items()) 523 }, 524 extra_aliases = { 525 hub_name: { 526 whl_name: sorted(aliases) 527 for whl_name, aliases in extra_whl_aliases.items() 528 } 529 for hub_name, extra_whl_aliases in extra_aliases.items() 530 }, 531 whl_libraries = { 532 k: dict(sorted(args.items())) 533 for k, args in sorted(whl_libraries.items()) 534 }, 535 is_reproducible = is_reproducible, 536 ) 537 538def _pip_impl(module_ctx): 539 """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories. 540 541 This implementation iterates through all of the `pip.parse` calls and creates 542 different pip hub repositories based on the "hub_name". Each of the 543 pip calls create spoke repos that uses a specific Python interpreter. 544 545 In a MODULES.bazel file we have: 546 547 pip.parse( 548 hub_name = "pip", 549 python_version = 3.9, 550 requirements_lock = "//:requirements_lock_3_9.txt", 551 requirements_windows = "//:requirements_windows_3_9.txt", 552 ) 553 pip.parse( 554 hub_name = "pip", 555 python_version = 3.10, 556 requirements_lock = "//:requirements_lock_3_10.txt", 557 requirements_windows = "//:requirements_windows_3_10.txt", 558 ) 559 560 For instance, we have a hub with the name of "pip". 561 A repository named the following is created. It is actually called last when 562 all of the pip spokes are collected. 563 564 - @@rules_python~override~pip~pip 565 566 As shown in the example code above we have the following. 567 Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip". 568 These definitions create two different pip spoke repositories that are 569 related to the hub "pip". 570 One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically 571 determines the Python version and the interpreter. 572 Both of these pip spokes contain requirements files that includes websocket 573 and its dependencies. 574 575 We also need repositories for the wheels that the different pip spokes contain. 576 For each Python version a different wheel repository is created. In our example 577 each pip spoke had a requirements file that contained websockets. We 578 then create two different wheel repositories that are named the following. 579 580 - @@rules_python~override~pip~pip_39_websockets 581 - @@rules_python~override~pip~pip_310_websockets 582 583 And if the wheel has any other dependencies subsequent wheels are created in the same fashion. 584 585 The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to 586 a spoke repository depending on the Python version. 587 588 Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple 589 hubs pointing to various different pip spokes. 590 591 Some other business rules notes. A hub can only have one spoke per Python version. We cannot 592 have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second 593 we cannot have the same hub name used in sub-modules. The hub name has to be globally 594 unique. 595 596 This implementation also handles the creation of whl_modification JSON files that are used 597 during the creation of wheel libraries. These JSON files used via the annotations argument 598 when calling wheel_installer.py. 599 600 Args: 601 module_ctx: module contents 602 """ 603 604 mods = parse_modules(module_ctx) 605 606 # Build all of the wheel modifications if the tag class is called. 607 _whl_mods_impl(mods.whl_mods) 608 609 for name, args in mods.whl_libraries.items(): 610 whl_library(name = name, **args) 611 612 for hub_name, whl_map in mods.hub_whl_map.items(): 613 hub_repository( 614 name = hub_name, 615 repo_name = hub_name, 616 extra_hub_aliases = mods.extra_aliases.get(hub_name, {}), 617 whl_map = { 618 key: whl_config_settings_to_json(values) 619 for key, values in whl_map.items() 620 }, 621 packages = mods.exposed_packages.get(hub_name, []), 622 groups = mods.hub_group_map.get(hub_name), 623 ) 624 625 if bazel_features.external_deps.extension_metadata_has_reproducible: 626 # If we are not using the `experimental_index_url feature, the extension is fully 627 # deterministic and we don't need to create a lock entry for it. 628 # 629 # In order to be able to dogfood the `experimental_index_url` feature before it gets 630 # stabilized, we have created the `_pip_non_reproducible` function, that will result 631 # in extra entries in the lock file. 632 return module_ctx.extension_metadata(reproducible = mods.is_reproducible) 633 else: 634 return None 635 636def _pip_parse_ext_attrs(**kwargs): 637 """Get the attributes for the pip extension. 638 639 Args: 640 **kwargs: A kwarg for setting defaults for the specific attributes. The 641 key is expected to be the same as the attribute key. 642 643 Returns: 644 A dict of attributes. 645 """ 646 attrs = dict({ 647 "experimental_extra_index_urls": attr.string_list( 648 doc = """\ 649The extra index URLs to use for downloading wheels using bazel downloader. 650Each value is going to be subject to `envsubst` substitutions if necessary. 651 652The indexes must support Simple API as described here: 653https://packaging.python.org/en/latest/specifications/simple-repository-api/ 654 655This is equivalent to `--extra-index-urls` `pip` option. 656""", 657 default = [], 658 ), 659 "experimental_index_url": attr.string( 660 default = kwargs.get("experimental_index_url", ""), 661 doc = """\ 662The index URL to use for downloading wheels using bazel downloader. This value is going 663to be subject to `envsubst` substitutions if necessary. 664 665The indexes must support Simple API as described here: 666https://packaging.python.org/en/latest/specifications/simple-repository-api/ 667 668In the future this could be defaulted to `https://pypi.org` when this feature becomes 669stable. 670 671This is equivalent to `--index-url` `pip` option. 672 673:::{versionchanged} 0.37.0 674If {attr}`download_only` is set, then `sdist` archives will be discarded and `pip.parse` will 675operate in wheel-only mode. 676::: 677""", 678 ), 679 "experimental_index_url_overrides": attr.string_dict( 680 doc = """\ 681The index URL overrides for each package to use for downloading wheels using 682bazel downloader. This value is going to be subject to `envsubst` substitutions 683if necessary. 684 685The key is the package name (will be normalized before usage) and the value is the 686index URL. 687 688This design pattern has been chosen in order to be fully deterministic about which 689packages come from which source. We want to avoid issues similar to what happened in 690https://pytorch.org/blog/compromised-nightly-dependency/. 691 692The indexes must support Simple API as described here: 693https://packaging.python.org/en/latest/specifications/simple-repository-api/ 694""", 695 ), 696 "hub_name": attr.string( 697 mandatory = True, 698 doc = """ 699The name of the repo pip dependencies will be accessible from. 700 701This name must be unique between modules; unless your module is guaranteed to 702always be the root module, it's highly recommended to include your module name 703in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can 704be used for shorter local names within your module. 705 706Within a module, the same `hub_name` can be specified to group different Python 707versions of pip dependencies under one repository name. This allows using a 708Python version-agnostic name when referring to pip dependencies; the 709correct version will be automatically selected. 710 711Typically, a module will only have a single hub of pip dependencies, but this 712is not required. Each hub is a separate resolution of pip dependencies. This 713means if different programs need different versions of some library, separate 714hubs can be created, and each program can use its respective hub's targets. 715Targets from different hubs should not be used together. 716""", 717 ), 718 "parallel_download": attr.bool( 719 doc = """\ 720The flag allows to make use of parallel downloading feature in bazel 7.1 and above 721when the bazel downloader is used. This is by default enabled as it improves the 722performance by a lot, but in case the queries to the simple API are very expensive 723or when debugging authentication issues one may want to disable this feature. 724 725NOTE, This will download (potentially duplicate) data for multiple packages if 726there is more than one index available, but in general this should be negligible 727because the simple API calls are very cheap and the user should not notice any 728extra overhead. 729 730If we are in synchronous mode, then we will use the first result that we 731find in case extra indexes are specified. 732""", 733 default = True, 734 ), 735 "python_version": attr.string( 736 mandatory = True, 737 doc = """ 738The Python version the dependencies are targetting, in Major.Minor format 739(e.g., "3.11") or patch level granularity (e.g. "3.11.1"). 740 741If an interpreter isn't explicitly provided (using `python_interpreter` or 742`python_interpreter_target`), then the version specified here must have 743a corresponding `python.toolchain()` configured. 744""", 745 ), 746 "whl_modifications": attr.label_keyed_string_dict( 747 mandatory = False, 748 doc = """\ 749A dict of labels to wheel names that is typically generated by the whl_modifications. 750The labels are JSON config files describing the modifications. 751""", 752 ), 753 "_evaluate_markers_srcs": attr.label_list( 754 default = EVALUATE_MARKERS_SRCS, 755 doc = """\ 756The list of labels to use as SRCS for the marker evaluation code. This ensures that the 757code will be re-evaluated when any of files in the default changes. 758""", 759 ), 760 }, **ATTRS) 761 attrs.update(AUTH_ATTRS) 762 763 return attrs 764 765def _whl_mod_attrs(): 766 attrs = { 767 "additive_build_content": attr.string( 768 doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.", 769 ), 770 "additive_build_content_file": attr.label( 771 doc = """\ 772(label, optional): path to a BUILD file to add to the generated 773`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file 774arguments at the same time.""", 775 ), 776 "copy_executables": attr.string_dict( 777 doc = """\ 778(dict, optional): A mapping of `src` and `out` files for 779[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as 780executable.""", 781 ), 782 "copy_files": attr.string_dict( 783 doc = """\ 784(dict, optional): A mapping of `src` and `out` files for 785[@bazel_skylib//rules:copy_file.bzl][cf]""", 786 ), 787 "data": attr.string_list( 788 doc = """\ 789(list, optional): A list of labels to add as `data` dependencies to 790the generated `py_library` target.""", 791 ), 792 "data_exclude_glob": attr.string_list( 793 doc = """\ 794(list, optional): A list of exclude glob patterns to add as `data` to 795the generated `py_library` target.""", 796 ), 797 "hub_name": attr.string( 798 doc = """\ 799Name of the whl modification, hub we use this name to set the modifications for 800pip.parse. If you have different pip hubs you can use a different name, 801otherwise it is best practice to just use one. 802 803You cannot have the same `hub_name` in different modules. You can reuse the same 804name in the same module for different wheels that you put in the same hub, but you 805cannot have a child module that uses the same `hub_name`. 806""", 807 mandatory = True, 808 ), 809 "srcs_exclude_glob": attr.string_list( 810 doc = """\ 811(list, optional): A list of labels to add as `srcs` to the generated 812`py_library` target.""", 813 ), 814 "whl_name": attr.string( 815 doc = "The whl name that the modifications are used for.", 816 mandatory = True, 817 ), 818 } 819 return attrs 820 821# NOTE: the naming of 'override' is taken from the bzlmod native 822# 'archive_override', 'git_override' bzlmod functions. 823_override_tag = tag_class( 824 attrs = { 825 "file": attr.string( 826 doc = """\ 827The Python distribution file name which needs to be patched. This will be 828applied to all repositories that setup this distribution via the pip.parse tag 829class.""", 830 mandatory = True, 831 ), 832 "patch_strip": attr.int( 833 default = 0, 834 doc = """\ 835The number of leading path segments to be stripped from the file name in the 836patches.""", 837 ), 838 "patches": attr.label_list( 839 doc = """\ 840A list of patches to apply to the repository *after* 'whl_library' is extracted 841and BUILD.bazel file is generated.""", 842 mandatory = True, 843 ), 844 }, 845 doc = """\ 846Apply any overrides (e.g. patches) to a given Python distribution defined by 847other tags in this extension.""", 848) 849 850pypi = module_extension( 851 doc = """\ 852This extension is used to make dependencies from pip available. 853 854pip.parse: 855To use, call `pip.parse()` and specify `hub_name` and your requirements file. 856Dependencies will be downloaded and made available in a repo named after the 857`hub_name` argument. 858 859Each `pip.parse()` call configures a particular Python version. Multiple calls 860can be made to configure different Python versions, and will be grouped by 861the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy` 862to automatically resolve to different, Python version-specific, libraries. 863 864pip.whl_mods: 865This tag class is used to help create JSON files to describe modifications to 866the BUILD files for wheels. 867""", 868 implementation = _pip_impl, 869 tag_classes = { 870 "override": _override_tag, 871 "parse": tag_class( 872 attrs = _pip_parse_ext_attrs(), 873 doc = """\ 874This tag class is used to create a pip hub and all of the spokes that are part of that hub. 875This tag class reuses most of the attributes found in {bzl:obj}`pip_parse`. 876The exception is it does not use the arg 'repo_prefix'. We set the repository 877prefix for the user and the alias arg is always True in bzlmod. 878""", 879 ), 880 "whl_mods": tag_class( 881 attrs = _whl_mod_attrs(), 882 doc = """\ 883This tag class is used to create JSON file that are used when calling wheel_builder.py. These 884JSON files contain instructions on how to modify a wheel's project. Each of the attributes 885create different modifications based on the type of attribute. Previously to bzlmod these 886JSON files where referred to as annotations, and were renamed to whl_modifications in this 887extension. 888""", 889 ), 890 }, 891) 892 893def _whl_mods_repo_impl(rctx): 894 rctx.file("BUILD.bazel", "") 895 for whl_name, mods in rctx.attr.whl_mods.items(): 896 rctx.file("{}.json".format(whl_name), mods) 897 898_whl_mods_repo = repository_rule( 899 doc = """\ 900This rule creates json files based on the whl_mods attribute. 901""", 902 implementation = _whl_mods_repo_impl, 903 attrs = { 904 "whl_mods": attr.string_dict( 905 mandatory = True, 906 doc = "JSON endcoded string that is provided to wheel_builder.py", 907 ), 908 }, 909) 910