# Copyright 2022 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Create a repository to hold the toolchains. This follows guidance here: https://docs.bazel.build/versions/main/skylark/deploying.html#registering-toolchains The "complex computation" in our case is simply downloading large artifacts. This guidance tells us how to avoid that: we put the toolchain targets in the alias repository with only the toolchain attribute pointing into the platform-specific repositories. """ load( "//python:versions.bzl", "PLATFORMS", "WINDOWS_NAME", ) load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":text_util.bzl", "render") def get_repository_name(repository_workspace): dummy_label = "//:_" return str(repository_workspace.relative(dummy_label))[:-len(dummy_label)] or "@" def python_toolchain_build_file_content( prefix, python_version, set_python_version_constraint, user_repository_name, loaded_platforms): """Creates the content for toolchain definitions for a build file. Args: prefix: Python toolchain name prefixes python_version: Python versions for the toolchains set_python_version_constraint: string, "True" if the toolchain should have the Python version constraint added as a requirement for matching the toolchain, "False" if not. user_repository_name: names for the user repos loaded_platforms: {type}`struct` the list of platform structs defining the loaded platforms. It is as they are defined in `//python:versions.bzl`. Returns: build_content: Text containing toolchain definitions """ return "\n\n".join([ """\ py_toolchain_suite( user_repository_name = "{user_repository_name}_{platform}", prefix = "{prefix}{platform}", target_compatible_with = {compatible_with}, flag_values = {flag_values}, python_version = "{python_version}", set_python_version_constraint = "{set_python_version_constraint}", )""".format( compatible_with = render.indent(render.list(meta.compatible_with)).lstrip(), flag_values = render.indent(render.dict( meta.flag_values, key_repr = lambda x: repr(str(x)), # this is to correctly display labels )).lstrip(), platform = platform, set_python_version_constraint = set_python_version_constraint, user_repository_name = user_repository_name, prefix = prefix, python_version = python_version, ) for platform, meta in loaded_platforms.items() ]) def _toolchains_repo_impl(rctx): build_content = """\ # Generated by python/private/toolchains_repo.bzl # # These can be registered in the workspace file or passed to --extra_toolchains # flag. By default all these toolchains are registered by the # python_register_toolchains macro so you don't normally need to interact with # these targets. load("@{rules_python}//python/private:py_toolchain_suite.bzl", "py_toolchain_suite") """.format( rules_python = rctx.attr._rules_python_workspace.workspace_name, ) toolchains = python_toolchain_build_file_content( prefix = "", python_version = rctx.attr.python_version, set_python_version_constraint = str(rctx.attr.set_python_version_constraint), user_repository_name = rctx.attr.user_repository_name, loaded_platforms = { k: v for k, v in PLATFORMS.items() if k in rctx.attr.platforms }, ) rctx.file("BUILD.bazel", build_content + toolchains) toolchains_repo = repository_rule( _toolchains_repo_impl, doc = "Creates a repository with toolchain definitions for all known platforms " + "which can be registered or selected.", attrs = { "platforms": attr.string_list(doc = "List of platforms for which the toolchain definitions shall be created"), "python_version": attr.string(doc = "The Python version."), "set_python_version_constraint": attr.bool(doc = "if target_compatible_with for the toolchain should set the version constraint"), "user_repository_name": attr.string(doc = "what the user chose for the base name"), "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), }, ) def _toolchain_aliases_impl(rctx): # Base BUILD file for this repository. build_contents = """\ # Generated by python/private/toolchains_repo.bzl load("@rules_python//python/private:toolchain_aliases.bzl", "toolchain_aliases") package(default_visibility = ["//visibility:public"]) exports_files(["defs.bzl"]) PLATFORMS = [ {loaded_platforms} ] toolchain_aliases( name = "{py_repository}", platforms = PLATFORMS, ) """.format( py_repository = rctx.attr.user_repository_name, loaded_platforms = "\n".join([" \"{}\",".format(p) for p in rctx.attr.platforms]), ) rctx.file("BUILD.bazel", build_contents) # Expose a Starlark file so rules can know what host platform we used and where to find an interpreter # when using repository_ctx.path, which doesn't understand aliases. rctx.file("defs.bzl", content = """\ # Generated by python/private/toolchains_repo.bzl load( "{rules_python}//python/config_settings:transition.bzl", _py_binary = "py_binary", _py_test = "py_test", ) load( "{rules_python}//python/entry_points:py_console_script_binary.bzl", _py_console_script_binary = "py_console_script_binary", ) load("{rules_python}//python:pip.bzl", _compile_pip_requirements = "compile_pip_requirements") def py_binary(name, **kwargs): return _py_binary( name = name, python_version = "{python_version}", **kwargs ) def py_console_script_binary(name, **kwargs): return _py_console_script_binary( name = name, binary_rule = py_binary, **kwargs ) def py_test(name, **kwargs): return _py_test( name = name, python_version = "{python_version}", **kwargs ) def compile_pip_requirements(name, **kwargs): return _compile_pip_requirements( name = name, py_binary = py_binary, py_test = py_test, **kwargs ) """.format( python_version = rctx.attr.python_version, rules_python = get_repository_name(rctx.attr._rules_python_workspace), )) toolchain_aliases = repository_rule( _toolchain_aliases_impl, doc = """\ Creates a repository with a shorter name only referencing the python version, it contains a BUILD.bazel file declaring aliases to the host platform's targets and is a great fit for any usage related to setting up toolchains for build actions.""", attrs = { "platforms": attr.string_list( doc = "List of platforms for which aliases shall be created", ), "python_version": attr.string(doc = "The Python version."), "user_repository_name": attr.string( mandatory = True, doc = "The base name for all created repositories, like 'python38'.", ), "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), }, environ = [REPO_DEBUG_ENV_VAR], ) def _host_toolchain_impl(rctx): rctx.file("BUILD.bazel", """\ # Generated by python/private/toolchains_repo.bzl exports_files(["python"], visibility = ["//visibility:public"]) """) os_name = repo_utils.get_platforms_os_name(rctx) host_platform = _get_host_platform( rctx = rctx, logger = repo_utils.logger(rctx), python_version = rctx.attr.python_version, os_name = os_name, cpu_name = repo_utils.get_platforms_cpu_name(rctx), platforms = rctx.attr.platforms, ) repo = "@@{py_repository}_{host_platform}".format( py_repository = rctx.attr.name[:-len("_host")], host_platform = host_platform, ) rctx.report_progress("Symlinking interpreter files to the target platform") host_python_repo = rctx.path(Label("{repo}//:BUILD.bazel".format(repo = repo))) # The interpreter might not work on platfroms that don't have symlink support if # we just symlink the interpreter itself. rctx.symlink does a copy in such cases # so we can just attempt to symlink all of the directories in the host interpreter # repo, which should be faster than re-downloading it. for p in host_python_repo.dirname.readdir(): if p.basename in [ # ignore special files created by the repo rule automatically "BUILD.bazel", "MODULE.bazel", "REPO.bazel", "WORKSPACE", "WORKSPACE.bazel", "WORKSPACE.bzlmod", ]: continue # symlink works on all platforms that bazel supports, so it should work on # UNIX and Windows with and without symlink support. For better performance # users should enable the symlink startup option, however that requires admin # privileges. rctx.symlink(p, p.basename) is_windows = (os_name == WINDOWS_NAME) python_binary = "python.exe" if is_windows else "python" # Ensure that we can run the interpreter and check that we are not # using the host interpreter. python_tester_contents = """\ from pathlib import Path import sys python = Path(sys.executable) want_python = str(Path("{python}").resolve()) got_python = str(Path(sys.executable).resolve()) assert want_python == got_python, \ "Expected to use a different interpreter:\\nwant: '{{}}'\\n got: '{{}}'".format( want_python, got_python, ) """.format(repo = repo.strip("@"), python = python_binary) python_tester = rctx.path("python_tester.py") rctx.file(python_tester, python_tester_contents) repo_utils.execute_checked( rctx, op = "CheckHostInterpreter", arguments = [rctx.path(python_binary), python_tester], ) if not rctx.delete(python_tester): fail("Failed to delete the python tester") host_toolchain = repository_rule( _host_toolchain_impl, doc = """\ Creates a repository with a shorter name meant to be used in the repository_ctx, which needs to have `symlinks` for the interpreter. This is separate from the toolchain_aliases repo because referencing the `python` interpreter target from this repo causes an eager fetch of the toolchain for the host platform. """, attrs = { "platforms": attr.string_list(mandatory = True), "python_version": attr.string(mandatory = True), "_rule_name": attr.string(default = "host_toolchain"), "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), }, ) def _multi_toolchain_aliases_impl(rctx): rules_python = rctx.attr._rules_python_workspace.workspace_name for python_version, repository_name in rctx.attr.python_versions.items(): file = "{}/defs.bzl".format(python_version) rctx.file(file, content = """\ # Generated by python/private/toolchains_repo.bzl load( "@{repository_name}//:defs.bzl", _compile_pip_requirements = "compile_pip_requirements", _py_binary = "py_binary", _py_console_script_binary = "py_console_script_binary", _py_test = "py_test", ) compile_pip_requirements = _compile_pip_requirements py_binary = _py_binary py_console_script_binary = _py_console_script_binary py_test = _py_test """.format( repository_name = repository_name, )) rctx.file("{}/BUILD.bazel".format(python_version), "") pip_bzl = """\ # Generated by python/private/toolchains_repo.bzl load("@{rules_python}//python:pip.bzl", "pip_parse", _multi_pip_parse = "multi_pip_parse") def multi_pip_parse(name, requirements_lock, **kwargs): return _multi_pip_parse( name = name, python_versions = {python_versions}, requirements_lock = requirements_lock, minor_mapping = {minor_mapping}, **kwargs ) """.format( python_versions = rctx.attr.python_versions.keys(), minor_mapping = render.indent(render.dict(rctx.attr.minor_mapping), indent = " " * 8).lstrip(), rules_python = rules_python, ) rctx.file("pip.bzl", content = pip_bzl) rctx.file("BUILD.bazel", "") multi_toolchain_aliases = repository_rule( _multi_toolchain_aliases_impl, attrs = { "minor_mapping": attr.string_dict(doc = "The mapping between `X.Y` and `X.Y.Z` python version values"), "python_versions": attr.string_dict(doc = "The Python versions."), "_rules_python_workspace": attr.label(default = Label("//:WORKSPACE")), }, ) def sanitize_platform_name(platform): return platform.replace("-", "_") def _get_host_platform(*, rctx, logger, python_version, os_name, cpu_name, platforms): """Gets the host platform. Args: rctx: {type}`repository_ctx`. logger: {type}`struct`. python_version: {type}`string`. os_name: {type}`str` the host OS name. cpu_name: {type}`str` the host CPU name. platforms: {type}`list[str]` the list of loaded platforms. Returns: The host platform. """ candidates = [] for platform in platforms: meta = PLATFORMS[platform] if meta.os_name == os_name and meta.arch == cpu_name: candidates.append(platform) if len(candidates) == 1: return candidates[0] if candidates: env_var = "RULES_PYTHON_REPO_TOOLCHAIN_{}_{}_{}".format( python_version.replace(".", "_"), os_name.upper(), cpu_name.upper(), ) preference = repo_utils.getenv(rctx, env_var) if preference == None: logger.info("Consider using '{}' to select from one of the platforms: {}".format( env_var, candidates, )) elif preference not in candidates: return logger.fail("Please choose a preferred interpreter out of the following platforms: {}".format(candidates)) else: candidates = [preference] if candidates: return candidates[0] return logger.fail("Could not find a compatible 'host' python for '{os_name}', '{cpu_name}' from the loaded platforms: {platforms}".format( os_name = os_name, cpu_name = cpu_name, platforms = platforms, ))