1# Adapted with modifications from tensorflow/third_party/py/ 2"""Repository rule for Python autoconfiguration. 3 4`python_configure` depends on the following environment variables: 5 6 * `PYTHON2_BIN_PATH`: location of python binary. 7 * `PYTHON2_LIB_PATH`: Location of python libraries. 8""" 9 10_BAZEL_SH = "BAZEL_SH" 11_PYTHON2_BIN_PATH = "PYTHON2_BIN_PATH" 12_PYTHON2_LIB_PATH = "PYTHON2_LIB_PATH" 13_PYTHON3_BIN_PATH = "PYTHON3_BIN_PATH" 14_PYTHON3_LIB_PATH = "PYTHON3_LIB_PATH" 15 16_HEADERS_HELP = ( 17 "Are Python headers installed? Try installing python-dev or " + 18 "python3-dev on Debian-based systems. Try python-devel or python3-devel " + 19 "on Redhat-based systems." 20) 21 22def _tpl(repository_ctx, tpl, substitutions = {}, out = None): 23 if not out: 24 out = tpl 25 repository_ctx.template( 26 out, 27 Label("//third_party/py:%s.tpl" % tpl), 28 substitutions, 29 ) 30 31def _fail(msg): 32 """Output failure message when auto configuration fails.""" 33 red = "\033[0;31m" 34 no_color = "\033[0m" 35 fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg)) 36 37def _is_windows(repository_ctx): 38 """Returns true if the host operating system is windows.""" 39 os_name = repository_ctx.os.name.lower() 40 return os_name.find("windows") != -1 41 42def _execute( 43 repository_ctx, 44 cmdline, 45 error_msg = None, 46 error_details = None, 47 empty_stdout_fine = False): 48 """Executes an arbitrary shell command. 49 50 Args: 51 repository_ctx: the repository_ctx object 52 cmdline: list of strings, the command to execute 53 error_msg: string, a summary of the error if the command fails 54 error_details: string, details about the error or steps to fix it 55 empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise 56 it's an error 57 Return: 58 the result of repository_ctx.execute(cmdline) 59 """ 60 result = repository_ctx.execute(cmdline) 61 if result.stderr or not (empty_stdout_fine or result.stdout): 62 _fail("\n".join([ 63 error_msg.strip() if error_msg else "Repository command failed", 64 result.stderr.strip(), 65 error_details if error_details else "", 66 ])) 67 else: 68 return result 69 70def _read_dir(repository_ctx, src_dir): 71 """Returns a string with all files in a directory. 72 73 Finds all files inside a directory, traversing subfolders and following 74 symlinks. The returned string contains the full path of all files 75 separated by line breaks. 76 """ 77 if _is_windows(repository_ctx): 78 src_dir = src_dir.replace("/", "\\") 79 find_result = _execute( 80 repository_ctx, 81 ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], 82 empty_stdout_fine = True, 83 ) 84 85 # src_files will be used in genrule.outs where the paths must 86 # use forward slashes. 87 return find_result.stdout.replace("\\", "/") 88 else: 89 find_result = _execute( 90 repository_ctx, 91 ["find", src_dir, "-follow", "-type", "f"], 92 empty_stdout_fine = True, 93 ) 94 return find_result.stdout 95 96def _genrule(src_dir, genrule_name, command, outs): 97 """Returns a string with a genrule. 98 99 Genrule executes the given command and produces the given outputs. 100 """ 101 return ("genrule(\n" + ' name = "' + genrule_name + '",\n' + 102 " outs = [\n" + outs + "\n ],\n" + ' cmd = """\n' + 103 command + '\n """,\n' + ")\n") 104 105def _normalize_path(path): 106 """Returns a path with '/' and remove the trailing slash.""" 107 path = path.replace("\\", "/") 108 if path[-1] == "/": 109 path = path[:-1] 110 return path 111 112def _symlink_genrule_for_dir( 113 repository_ctx, 114 src_dir, 115 dest_dir, 116 genrule_name, 117 src_files = [], 118 dest_files = []): 119 """Returns a genrule to symlink(or copy if on Windows) a set of files. 120 121 If src_dir is passed, files will be read from the given directory; otherwise 122 we assume files are in src_files and dest_files 123 """ 124 if src_dir != None: 125 src_dir = _normalize_path(src_dir) 126 dest_dir = _normalize_path(dest_dir) 127 files = "\n".join( 128 sorted(_read_dir(repository_ctx, src_dir).splitlines()), 129 ) 130 131 # Create a list with the src_dir stripped to use for outputs. 132 dest_files = files.replace(src_dir, "").splitlines() 133 src_files = files.splitlines() 134 command = [] 135 outs = [] 136 for i in range(len(dest_files)): 137 if dest_files[i] != "": 138 # If we have only one file to link we do not want to use the dest_dir, as 139 # $(@D) will include the full path to the file. 140 dest = "$(@D)/" + dest_dir + dest_files[i] if len( 141 dest_files, 142 ) != 1 else "$(@D)/" + dest_files[i] 143 144 # On Windows, symlink is not supported, so we just copy all the files. 145 cmd = "cp -f" if _is_windows(repository_ctx) else "ln -s" 146 command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) 147 outs.append(' "' + dest_dir + dest_files[i] + '",') 148 return _genrule( 149 src_dir, 150 genrule_name, 151 " && ".join(command), 152 "\n".join(outs), 153 ) 154 155def _get_python_bin(repository_ctx, bin_path_key, default_bin_path): 156 """Gets the python bin path.""" 157 python_bin = repository_ctx.os.environ.get(bin_path_key, default_bin_path) 158 if not repository_ctx.path(python_bin).exists: 159 # It's a command, use 'which' to find its path. 160 python_bin_path = repository_ctx.which(python_bin) 161 else: 162 # It's a path, use it as it is. 163 python_bin_path = python_bin 164 if python_bin_path != None: 165 return str(python_bin_path) 166 _fail("Cannot find python in PATH, please make sure " + 167 "python is installed and add its directory in PATH, or --define " + 168 "%s='/something/else'.\nPATH=%s" % 169 (bin_path_key, repository_ctx.os.environ.get("PATH", ""))) 170 171def _get_bash_bin(repository_ctx): 172 """Gets the bash bin path.""" 173 bash_bin = repository_ctx.os.environ.get(_BAZEL_SH) 174 if bash_bin != None: 175 return bash_bin 176 else: 177 bash_bin_path = repository_ctx.which("bash") 178 if bash_bin_path != None: 179 return str(bash_bin_path) 180 else: 181 _fail( 182 "Cannot find bash in PATH, please make sure " + 183 "bash is installed and add its directory in PATH, or --define " + 184 "%s='/path/to/bash'.\nPATH=%s" % 185 (_BAZEL_SH, repository_ctx.os.environ.get("PATH", "")), 186 ) 187 188def _get_python_lib(repository_ctx, python_bin, lib_path_key): 189 """Gets the python lib path.""" 190 python_lib = repository_ctx.os.environ.get(lib_path_key) 191 if python_lib != None: 192 return python_lib 193 print_lib = ( 194 "<<END\n" + "from __future__ import print_function\n" + 195 "import site\n" + "import os\n" + "\n" + "try:\n" + 196 " input = raw_input\n" + "except NameError:\n" + " pass\n" + "\n" + 197 "python_paths = []\n" + "if os.getenv('PYTHONPATH') is not None:\n" + 198 " python_paths = os.getenv('PYTHONPATH').split(':')\n" + "try:\n" + 199 " library_paths = site.getsitepackages()\n" + 200 "except AttributeError:\n" + 201 " from distutils.sysconfig import get_python_lib\n" + 202 " library_paths = [get_python_lib()]\n" + 203 "all_paths = set(python_paths + library_paths)\n" + "paths = []\n" + 204 "for path in all_paths:\n" + " if os.path.isdir(path):\n" + 205 " paths.append(path)\n" + "if len(paths) >=1:\n" + 206 " print(paths[0])\n" + "END" 207 ) 208 cmd = "%s - %s" % (python_bin, print_lib) 209 result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) 210 return result.stdout.strip("\n") 211 212def _check_python_lib(repository_ctx, python_lib): 213 """Checks the python lib path.""" 214 cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) 215 result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) 216 if result.return_code == 1: 217 _fail("Invalid python library path: %s" % python_lib) 218 219def _check_python_bin(repository_ctx, python_bin, bin_path_key): 220 """Checks the python bin path.""" 221 cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) 222 result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) 223 if result.return_code == 1: 224 _fail("--define %s='%s' is not executable. Is it the python binary?" % 225 (bin_path_key, python_bin)) 226 227def _get_python_include(repository_ctx, python_bin): 228 """Gets the python include path.""" 229 result = _execute( 230 repository_ctx, 231 [ 232 python_bin, 233 "-c", 234 "from __future__ import print_function;" + 235 "from distutils import sysconfig;" + 236 "print(sysconfig.get_python_inc())", 237 ], 238 error_msg = "Problem getting python include path for {}.".format(python_bin), 239 error_details = ( 240 "Is the Python binary path set up right? " + "(See ./configure or " + 241 python_bin + ".) " + "Is distutils installed? " + 242 _HEADERS_HELP 243 ), 244 ) 245 include_path = result.stdout.splitlines()[0] 246 _execute( 247 repository_ctx, 248 [ 249 python_bin, 250 "-c", 251 "import os;" + 252 "main_header = os.path.join('{}', 'Python.h');".format(include_path) + 253 "assert os.path.exists(main_header), main_header + ' does not exist.'", 254 ], 255 error_msg = "Unable to find Python headers for {}".format(python_bin), 256 error_details = _HEADERS_HELP, 257 empty_stdout_fine = True, 258 ) 259 return include_path 260 261def _get_python_import_lib_name(repository_ctx, python_bin, bin_path_key): 262 """Get Python import library name (pythonXY.lib) on Windows.""" 263 result = _execute( 264 repository_ctx, 265 [ 266 python_bin, 267 "-c", 268 "import sys;" + 'print("python" + str(sys.version_info[0]) + ' + 269 ' str(sys.version_info[1]) + ".lib")', 270 ], 271 error_msg = "Problem getting python import library.", 272 error_details = ("Is the Python binary path set up right? " + 273 "(See ./configure or " + bin_path_key + ".) "), 274 ) 275 return result.stdout.splitlines()[0] 276 277def _create_single_version_package( 278 repository_ctx, 279 variety_name, 280 bin_path_key, 281 default_bin_path, 282 lib_path_key): 283 """Creates the repository containing files set up to build with Python.""" 284 python_bin = _get_python_bin(repository_ctx, bin_path_key, default_bin_path) 285 _check_python_bin(repository_ctx, python_bin, bin_path_key) 286 python_lib = _get_python_lib(repository_ctx, python_bin, lib_path_key) 287 _check_python_lib(repository_ctx, python_lib) 288 python_include = _get_python_include(repository_ctx, python_bin) 289 python_include_rule = _symlink_genrule_for_dir( 290 repository_ctx, 291 python_include, 292 "{}_include".format(variety_name), 293 "{}_include".format(variety_name), 294 ) 295 python_import_lib_genrule = "" 296 297 # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib 298 # See https://docs.python.org/3/extending/windows.html 299 if _is_windows(repository_ctx): 300 python_include = _normalize_path(python_include) 301 python_import_lib_name = _get_python_import_lib_name( 302 repository_ctx, 303 python_bin, 304 bin_path_key, 305 ) 306 python_import_lib_src = python_include.rsplit( 307 "/", 308 1, 309 )[0] + "/libs/" + python_import_lib_name 310 python_import_lib_genrule = _symlink_genrule_for_dir( 311 repository_ctx, 312 None, 313 "", 314 "{}_import_lib".format(variety_name), 315 [python_import_lib_src], 316 [python_import_lib_name], 317 ) 318 _tpl( 319 repository_ctx, 320 "variety", 321 { 322 "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, 323 "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, 324 "%{VARIETY_NAME}": variety_name, 325 }, 326 out = "{}/BUILD".format(variety_name), 327 ) 328 329def _python_autoconf_impl(repository_ctx): 330 """Implementation of the python_autoconf repository rule.""" 331 _create_single_version_package( 332 repository_ctx, 333 "_python2", 334 _PYTHON2_BIN_PATH, 335 "python", 336 _PYTHON2_LIB_PATH, 337 ) 338 _create_single_version_package( 339 repository_ctx, 340 "_python3", 341 _PYTHON3_BIN_PATH, 342 "python3", 343 _PYTHON3_LIB_PATH, 344 ) 345 _tpl(repository_ctx, "BUILD") 346 347python_configure = repository_rule( 348 implementation = _python_autoconf_impl, 349 environ = [ 350 _BAZEL_SH, 351 _PYTHON2_BIN_PATH, 352 _PYTHON2_LIB_PATH, 353 _PYTHON3_BIN_PATH, 354 _PYTHON3_LIB_PATH, 355 ], 356 attrs = { 357 "_build_tpl": attr.label( 358 default = Label("//third_party/py:BUILD.tpl"), 359 allow_single_file = True, 360 ), 361 "_variety_tpl": attr.label( 362 default = Label("//third_party/py:variety.tpl"), 363 allow_single_file = True, 364 ), 365 }, 366) 367"""Detects and configures the local Python. 368 369It is expected that the system have both a working Python 2 and python 3 370installation 371 372Add the following to your WORKSPACE FILE: 373 374```python 375python_configure(name = "local_config_python") 376``` 377 378Args: 379 name: A unique name for this workspace rule. 380""" 381