1# Copyright 2024 The Bazel Authors. All rights reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""This file contains repository rules and macros to support toolchain registration. 16""" 17 18load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY", "PLATFORMS") 19load(":auth.bzl", "get_auth") 20load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") 21load(":text_util.bzl", "render") 22 23STANDALONE_INTERPRETER_FILENAME = "STANDALONE_INTERPRETER" 24 25def is_standalone_interpreter(rctx, python_interpreter_path, *, logger = None): 26 """Query a python interpreter target for whether or not it's a rules_rust provided toolchain 27 28 Args: 29 rctx: {type}`repository_ctx` The repository rule's context object. 30 python_interpreter_path: {type}`path` A path representing the interpreter. 31 logger: Optional logger to use for operations. 32 33 Returns: 34 {type}`bool` Whether or not the target is from a rules_python generated toolchain. 35 """ 36 37 # Only update the location when using a hermetic toolchain. 38 if not python_interpreter_path: 39 return False 40 41 # This is a rules_python provided toolchain. 42 return repo_utils.execute_unchecked( 43 rctx, 44 op = "IsStandaloneInterpreter", 45 arguments = [ 46 "ls", 47 "{}/{}".format( 48 python_interpreter_path.dirname, 49 STANDALONE_INTERPRETER_FILENAME, 50 ), 51 ], 52 logger = logger, 53 ).return_code == 0 54 55def _python_repository_impl(rctx): 56 if rctx.attr.distutils and rctx.attr.distutils_content: 57 fail("Only one of (distutils, distutils_content) should be set.") 58 if bool(rctx.attr.url) == bool(rctx.attr.urls): 59 fail("Exactly one of (url, urls) must be set.") 60 61 logger = repo_utils.logger(rctx) 62 63 platform = rctx.attr.platform 64 python_version = rctx.attr.python_version 65 python_version_info = python_version.split(".") 66 release_filename = rctx.attr.release_filename 67 version_suffix = "t" if FREETHREADED in release_filename else "" 68 python_short_version = "{0}.{1}{suffix}".format( 69 suffix = version_suffix, 70 *python_version_info 71 ) 72 urls = rctx.attr.urls or [rctx.attr.url] 73 auth = get_auth(rctx, urls) 74 75 if INSTALL_ONLY in release_filename: 76 rctx.download_and_extract( 77 url = urls, 78 sha256 = rctx.attr.sha256, 79 stripPrefix = rctx.attr.strip_prefix, 80 auth = auth, 81 ) 82 else: 83 rctx.download_and_extract( 84 url = urls, 85 sha256 = rctx.attr.sha256, 86 stripPrefix = rctx.attr.strip_prefix, 87 auth = auth, 88 ) 89 90 # Strip the things that are not present in the INSTALL_ONLY builds 91 # NOTE: if the dirs are not present, we will not fail here 92 rctx.delete("python/build") 93 rctx.delete("python/licenses") 94 rctx.delete("python/PYTHON.json") 95 96 patches = rctx.attr.patches 97 if patches: 98 for patch in patches: 99 rctx.patch(patch, strip = rctx.attr.patch_strip) 100 101 # Write distutils.cfg to the Python installation. 102 if "windows" in platform: 103 distutils_path = "Lib/distutils/distutils.cfg" 104 else: 105 distutils_path = "lib/python{}/distutils/distutils.cfg".format(python_short_version) 106 if rctx.attr.distutils: 107 rctx.file(distutils_path, rctx.read(rctx.attr.distutils)) 108 elif rctx.attr.distutils_content: 109 rctx.file(distutils_path, rctx.attr.distutils_content) 110 111 if "darwin" in platform and "osx" == repo_utils.get_platforms_os_name(rctx): 112 # Fix up the Python distribution's LC_ID_DYLIB field. 113 # It points to a build directory local to the GitHub Actions 114 # host machine used in the Python standalone build, which causes 115 # dyld lookup errors. To fix, set the full path to the dylib as 116 # it appears in the Bazel workspace as its LC_ID_DYLIB using 117 # the `install_name_tool` bundled with macOS. 118 dylib = "libpython{}.dylib".format(python_short_version) 119 repo_utils.execute_checked( 120 rctx, 121 op = "python_repository.FixUpDyldIdPath", 122 arguments = [repo_utils.which_checked(rctx, "install_name_tool"), "-id", "@rpath/{}".format(dylib), "lib/{}".format(dylib)], 123 logger = logger, 124 ) 125 126 # Make the Python installation read-only. This is to prevent issues due to 127 # pycs being generated at runtime: 128 # * The pycs are not deterministic (they contain timestamps) 129 # * Multiple processes trying to write the same pycs can result in errors. 130 if not rctx.attr.ignore_root_user_error: 131 if "windows" not in platform: 132 lib_dir = "lib" if "windows" not in platform else "Lib" 133 134 repo_utils.execute_checked( 135 rctx, 136 op = "python_repository.MakeReadOnly", 137 arguments = [repo_utils.which_checked(rctx, "chmod"), "-R", "ugo-w", lib_dir], 138 logger = logger, 139 ) 140 exec_result = repo_utils.execute_unchecked( 141 rctx, 142 op = "python_repository.TestReadOnly", 143 arguments = [repo_utils.which_checked(rctx, "touch"), "{}/.test".format(lib_dir)], 144 logger = logger, 145 ) 146 147 # The issue with running as root is the installation is no longer 148 # read-only, so the problems due to pyc can resurface. 149 if exec_result.return_code == 0: 150 stdout = repo_utils.execute_checked_stdout( 151 rctx, 152 op = "python_repository.GetUserId", 153 arguments = [repo_utils.which_checked(rctx, "id"), "-u"], 154 logger = logger, 155 ) 156 uid = int(stdout.strip()) 157 if uid == 0: 158 fail("The current user is root, please run as non-root when using the hermetic Python interpreter. See https://github.com/bazelbuild/rules_python/pull/713.") 159 else: 160 fail("The current user has CAP_DAC_OVERRIDE set, please drop this capability when using the hermetic Python interpreter. See https://github.com/bazelbuild/rules_python/pull/713.") 161 162 python_bin = "python.exe" if ("windows" in platform) else "bin/python3" 163 164 if "linux" in platform: 165 # Workaround around https://github.com/indygreg/python-build-standalone/issues/231 166 for url in urls: 167 head_and_release, _, _ = url.rpartition("/") 168 _, _, release = head_and_release.rpartition("/") 169 if not release.isdigit(): 170 # Maybe this is some custom toolchain, so skip this 171 break 172 173 if int(release) >= 20240224: 174 # Starting with this release the Linux toolchains have infinite symlink loop 175 # on host platforms that are not Linux. Delete the files no 176 # matter the host platform so that the cross-built artifacts 177 # are the same irrespective of the host platform we are 178 # building on. 179 # 180 # Link to the first affected release: 181 # https://github.com/indygreg/python-build-standalone/releases/tag/20240224 182 rctx.delete("share/terminfo") 183 break 184 185 glob_include = [] 186 glob_exclude = [] 187 if rctx.attr.ignore_root_user_error or "windows" in platform: 188 glob_exclude += [ 189 # These pycache files are created on first use of the associated python files. 190 # Exclude them from the glob because otherwise between the first time and second time a python toolchain is used," 191 # the definition of this filegroup will change, and depending rules will get invalidated." 192 # See https://github.com/bazelbuild/rules_python/issues/1008 for unconditionally adding these to toolchains so we can stop ignoring them." 193 "**/__pycache__/*.pyc", 194 "**/__pycache__/*.pyo", 195 ] 196 197 if "windows" in platform: 198 glob_include += [ 199 "*.exe", 200 "*.dll", 201 "DLLs/**", 202 "Lib/**", 203 "Scripts/**", 204 "tcl/**", 205 ] 206 else: 207 glob_include.append( 208 "lib/**", 209 ) 210 211 if "windows" in platform: 212 coverage_tool = None 213 else: 214 coverage_tool = rctx.attr.coverage_tool 215 216 build_content = """\ 217# Generated by python/private/python_repositories.bzl 218 219load("@rules_python//python/private:hermetic_runtime_repo_setup.bzl", "define_hermetic_runtime_toolchain_impl") 220 221package(default_visibility = ["//visibility:public"]) 222 223define_hermetic_runtime_toolchain_impl( 224 name = "define_runtime", 225 extra_files_glob_include = {extra_files_glob_include}, 226 extra_files_glob_exclude = {extra_files_glob_exclude}, 227 python_version = {python_version}, 228 python_bin = {python_bin}, 229 coverage_tool = {coverage_tool}, 230) 231""".format( 232 extra_files_glob_exclude = render.list(glob_exclude), 233 extra_files_glob_include = render.list(glob_include), 234 python_bin = render.str(python_bin), 235 python_version = render.str(rctx.attr.python_version), 236 coverage_tool = render.str(coverage_tool), 237 ) 238 rctx.delete("python") 239 rctx.symlink(python_bin, "python") 240 rctx.file(STANDALONE_INTERPRETER_FILENAME, "# File intentionally left blank. Indicates that this is an interpreter repo created by rules_python.") 241 rctx.file("BUILD.bazel", build_content) 242 243 attrs = { 244 "auth_patterns": rctx.attr.auth_patterns, 245 "coverage_tool": rctx.attr.coverage_tool, 246 "distutils": rctx.attr.distutils, 247 "distutils_content": rctx.attr.distutils_content, 248 "ignore_root_user_error": rctx.attr.ignore_root_user_error, 249 "name": rctx.attr.name, 250 "netrc": rctx.attr.netrc, 251 "patch_strip": rctx.attr.patch_strip, 252 "patches": rctx.attr.patches, 253 "platform": platform, 254 "python_version": python_version, 255 "release_filename": release_filename, 256 "sha256": rctx.attr.sha256, 257 "strip_prefix": rctx.attr.strip_prefix, 258 } 259 260 if rctx.attr.url: 261 attrs["url"] = rctx.attr.url 262 else: 263 attrs["urls"] = urls 264 265 return attrs 266 267python_repository = repository_rule( 268 _python_repository_impl, 269 doc = "Fetches the external tools needed for the Python toolchain.", 270 attrs = { 271 "auth_patterns": attr.string_dict( 272 doc = "Override mapping of hostnames to authorization patterns; mirrors the eponymous attribute from http_archive", 273 ), 274 "coverage_tool": attr.string( 275 doc = """ 276This is a target to use for collecting code coverage information from {rule}`py_binary` 277and {rule}`py_test` targets. 278 279The target is accepted as a string by the python_repository and evaluated within 280the context of the toolchain repository. 281 282For more information see {attr}`py_runtime.coverage_tool`. 283""", 284 ), 285 "distutils": attr.label( 286 allow_single_file = True, 287 doc = "A distutils.cfg file to be included in the Python installation. " + 288 "Either distutils or distutils_content can be specified, but not both.", 289 mandatory = False, 290 ), 291 "distutils_content": attr.string( 292 doc = "A distutils.cfg file content to be included in the Python installation. " + 293 "Either distutils or distutils_content can be specified, but not both.", 294 mandatory = False, 295 ), 296 "ignore_root_user_error": attr.bool( 297 default = False, 298 doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", 299 mandatory = False, 300 ), 301 "netrc": attr.string( 302 doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", 303 ), 304 "patch_strip": attr.int( 305 doc = """ 306Same as the --strip argument of Unix patch. 307 308:::{note} 309In the future the default value will be set to `0`, to mimic the well known 310function defaults (e.g. `single_version_override` for `MODULE.bazel` files. 311::: 312 313:::{versionadded} 0.36.0 314::: 315""", 316 default = 1, 317 mandatory = False, 318 ), 319 "patches": attr.label_list( 320 doc = "A list of patch files to apply to the unpacked interpreter", 321 mandatory = False, 322 ), 323 "platform": attr.string( 324 doc = "The platform name for the Python interpreter tarball.", 325 mandatory = True, 326 values = PLATFORMS.keys(), 327 ), 328 "python_version": attr.string( 329 doc = "The Python version.", 330 mandatory = True, 331 ), 332 "release_filename": attr.string( 333 doc = "The filename of the interpreter to be downloaded", 334 mandatory = True, 335 ), 336 "sha256": attr.string( 337 doc = "The SHA256 integrity hash for the Python interpreter tarball.", 338 mandatory = True, 339 ), 340 "strip_prefix": attr.string( 341 doc = "A directory prefix to strip from the extracted files.", 342 ), 343 "url": attr.string( 344 doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", 345 ), 346 "urls": attr.string_list( 347 doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", 348 ), 349 "_rule_name": attr.string(default = "python_repository"), 350 }, 351 environ = [REPO_DEBUG_ENV_VAR], 352) 353