1# Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""configure script to get build parameters from user.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import argparse 22import errno 23import os 24import platform 25import re 26import subprocess 27import sys 28 29# pylint: disable=g-import-not-at-top 30try: 31 from shutil import which 32except ImportError: 33 from distutils.spawn import find_executable as which 34# pylint: enable=g-import-not-at-top 35 36_DEFAULT_CUDA_VERSION = '10' 37_DEFAULT_CUDNN_VERSION = '7' 38_DEFAULT_TENSORRT_VERSION = '6' 39_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0' 40 41_TF_OPENCL_VERSION = '1.2' 42_DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp' 43_DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include' 44_SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16, 17, 18] 45 46_DEFAULT_PROMPT_ASK_ATTEMPTS = 10 47 48_TF_BAZELRC_FILENAME = '.tf_configure.bazelrc' 49_TF_WORKSPACE_ROOT = '' 50_TF_BAZELRC = '' 51_TF_CURRENT_BAZEL_VERSION = None 52_TF_MIN_BAZEL_VERSION = '1.2.1' 53_TF_MAX_BAZEL_VERSION = '1.2.1' 54 55NCCL_LIB_PATHS = [ 56 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', '' 57] 58 59# List of files to configure when building Bazel on Apple platforms. 60APPLE_BAZEL_FILES = [ 61 'tensorflow/lite/experimental/ios/BUILD', 62 'tensorflow/lite/experimental/objc/BUILD', 63 'tensorflow/lite/experimental/swift/BUILD', 64 'tensorflow/lite/tools/benchmark/experimental/ios/BUILD' 65] 66 67# List of files to move when building for iOS. 68IOS_FILES = [ 69 'tensorflow/lite/experimental/objc/TensorFlowLiteObjC.podspec', 70 'tensorflow/lite/experimental/swift/TensorFlowLiteSwift.podspec', 71] 72 73 74class UserInputError(Exception): 75 pass 76 77 78def is_windows(): 79 return platform.system() == 'Windows' 80 81 82def is_linux(): 83 return platform.system() == 'Linux' 84 85 86def is_macos(): 87 return platform.system() == 'Darwin' 88 89 90def is_ppc64le(): 91 return platform.machine() == 'ppc64le' 92 93 94def is_cygwin(): 95 return platform.system().startswith('CYGWIN_NT') 96 97 98def get_input(question): 99 try: 100 try: 101 answer = raw_input(question) 102 except NameError: 103 answer = input(question) # pylint: disable=bad-builtin 104 except EOFError: 105 answer = '' 106 return answer 107 108 109def symlink_force(target, link_name): 110 """Force symlink, equivalent of 'ln -sf'. 111 112 Args: 113 target: items to link to. 114 link_name: name of the link. 115 """ 116 try: 117 os.symlink(target, link_name) 118 except OSError as e: 119 if e.errno == errno.EEXIST: 120 os.remove(link_name) 121 os.symlink(target, link_name) 122 else: 123 raise e 124 125 126def sed_in_place(filename, old, new): 127 """Replace old string with new string in file. 128 129 Args: 130 filename: string for filename. 131 old: string to replace. 132 new: new string to replace to. 133 """ 134 with open(filename, 'r') as f: 135 filedata = f.read() 136 newdata = filedata.replace(old, new) 137 with open(filename, 'w') as f: 138 f.write(newdata) 139 140 141def write_to_bazelrc(line): 142 with open(_TF_BAZELRC, 'a') as f: 143 f.write(line + '\n') 144 145 146def write_action_env_to_bazelrc(var_name, var): 147 write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var))) 148 149 150def run_shell(cmd, allow_non_zero=False, stderr=None): 151 if stderr is None: 152 stderr = sys.stdout 153 if allow_non_zero: 154 try: 155 output = subprocess.check_output(cmd, stderr=stderr) 156 except subprocess.CalledProcessError as e: 157 output = e.output 158 else: 159 output = subprocess.check_output(cmd, stderr=stderr) 160 return output.decode('UTF-8').strip() 161 162 163def cygpath(path): 164 """Convert path from posix to windows.""" 165 return os.path.abspath(path).replace('\\', '/') 166 167 168def get_python_path(environ_cp, python_bin_path): 169 """Get the python site package paths.""" 170 python_paths = [] 171 if environ_cp.get('PYTHONPATH'): 172 python_paths = environ_cp.get('PYTHONPATH').split(':') 173 try: 174 stderr = open(os.devnull, 'wb') 175 library_paths = run_shell([ 176 python_bin_path, '-c', 177 'import site; print("\\n".join(site.getsitepackages()))' 178 ], 179 stderr=stderr).split('\n') 180 except subprocess.CalledProcessError: 181 library_paths = [ 182 run_shell([ 183 python_bin_path, '-c', 184 'from distutils.sysconfig import get_python_lib;' 185 'print(get_python_lib())' 186 ]) 187 ] 188 189 all_paths = set(python_paths + library_paths) 190 191 paths = [] 192 for path in all_paths: 193 if os.path.isdir(path): 194 paths.append(path) 195 return paths 196 197 198def get_python_major_version(python_bin_path): 199 """Get the python major version.""" 200 return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])']) 201 202 203def setup_python(environ_cp): 204 """Setup python related env variables.""" 205 # Get PYTHON_BIN_PATH, default is the current running python. 206 default_python_bin_path = sys.executable 207 ask_python_bin_path = ('Please specify the location of python. [Default is ' 208 '%s]: ') % default_python_bin_path 209 while True: 210 python_bin_path = get_from_env_or_user_or_default(environ_cp, 211 'PYTHON_BIN_PATH', 212 ask_python_bin_path, 213 default_python_bin_path) 214 # Check if the path is valid 215 if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK): 216 break 217 elif not os.path.exists(python_bin_path): 218 print('Invalid python path: %s cannot be found.' % python_bin_path) 219 else: 220 print('%s is not executable. Is it the python binary?' % python_bin_path) 221 environ_cp['PYTHON_BIN_PATH'] = '' 222 223 # Convert python path to Windows style before checking lib and version 224 if is_windows() or is_cygwin(): 225 python_bin_path = cygpath(python_bin_path) 226 227 # Get PYTHON_LIB_PATH 228 python_lib_path = environ_cp.get('PYTHON_LIB_PATH') 229 if not python_lib_path: 230 python_lib_paths = get_python_path(environ_cp, python_bin_path) 231 if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1': 232 python_lib_path = python_lib_paths[0] 233 else: 234 print('Found possible Python library paths:\n %s' % 235 '\n '.join(python_lib_paths)) 236 default_python_lib_path = python_lib_paths[0] 237 python_lib_path = get_input( 238 'Please input the desired Python library path to use. ' 239 'Default is [%s]\n' % python_lib_paths[0]) 240 if not python_lib_path: 241 python_lib_path = default_python_lib_path 242 environ_cp['PYTHON_LIB_PATH'] = python_lib_path 243 244 python_major_version = get_python_major_version(python_bin_path) 245 if python_major_version == '2': 246 write_to_bazelrc('build --host_force_python=PY2') 247 248 # Convert python path to Windows style before writing into bazel.rc 249 if is_windows() or is_cygwin(): 250 python_lib_path = cygpath(python_lib_path) 251 252 # Set-up env variables used by python_configure.bzl 253 write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path) 254 write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path) 255 write_to_bazelrc('build --python_path=\"%s"' % python_bin_path) 256 environ_cp['PYTHON_BIN_PATH'] = python_bin_path 257 258 # If choosen python_lib_path is from a path specified in the PYTHONPATH 259 # variable, need to tell bazel to include PYTHONPATH 260 if environ_cp.get('PYTHONPATH'): 261 python_paths = environ_cp.get('PYTHONPATH').split(':') 262 if python_lib_path in python_paths: 263 write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH')) 264 265 # Write tools/python_bin_path.sh 266 with open( 267 os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'), 268 'w') as f: 269 f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path) 270 271 272def reset_tf_configure_bazelrc(): 273 """Reset file that contains customized config settings.""" 274 open(_TF_BAZELRC, 'w').close() 275 276 277def cleanup_makefile(): 278 """Delete any leftover BUILD files from the Makefile build. 279 280 These files could interfere with Bazel parsing. 281 """ 282 makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow', 283 'contrib', 'makefile', 'downloads') 284 if os.path.isdir(makefile_download_dir): 285 for root, _, filenames in os.walk(makefile_download_dir): 286 for f in filenames: 287 if f.endswith('BUILD'): 288 os.remove(os.path.join(root, f)) 289 290 291def get_var(environ_cp, 292 var_name, 293 query_item, 294 enabled_by_default, 295 question=None, 296 yes_reply=None, 297 no_reply=None): 298 """Get boolean input from user. 299 300 If var_name is not set in env, ask user to enable query_item or not. If the 301 response is empty, use the default. 302 303 Args: 304 environ_cp: copy of the os.environ. 305 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". 306 query_item: string for feature related to the variable, e.g. "CUDA for 307 Nvidia GPUs". 308 enabled_by_default: boolean for default behavior. 309 question: optional string for how to ask for user input. 310 yes_reply: optional string for reply when feature is enabled. 311 no_reply: optional string for reply when feature is disabled. 312 313 Returns: 314 boolean value of the variable. 315 316 Raises: 317 UserInputError: if an environment variable is set, but it cannot be 318 interpreted as a boolean indicator, assume that the user has made a 319 scripting error, and will continue to provide invalid input. 320 Raise the error to avoid infinitely looping. 321 """ 322 if not question: 323 question = 'Do you wish to build TensorFlow with %s support?' % query_item 324 if not yes_reply: 325 yes_reply = '%s support will be enabled for TensorFlow.' % query_item 326 if not no_reply: 327 no_reply = 'No %s' % yes_reply 328 329 yes_reply += '\n' 330 no_reply += '\n' 331 332 if enabled_by_default: 333 question += ' [Y/n]: ' 334 else: 335 question += ' [y/N]: ' 336 337 var = environ_cp.get(var_name) 338 if var is not None: 339 var_content = var.strip().lower() 340 true_strings = ('1', 't', 'true', 'y', 'yes') 341 false_strings = ('0', 'f', 'false', 'n', 'no') 342 if var_content in true_strings: 343 var = True 344 elif var_content in false_strings: 345 var = False 346 else: 347 raise UserInputError( 348 'Environment variable %s must be set as a boolean indicator.\n' 349 'The following are accepted as TRUE : %s.\n' 350 'The following are accepted as FALSE: %s.\n' 351 'Current value is %s.' % 352 (var_name, ', '.join(true_strings), ', '.join(false_strings), var)) 353 354 while var is None: 355 user_input_origin = get_input(question) 356 user_input = user_input_origin.strip().lower() 357 if user_input == 'y': 358 print(yes_reply) 359 var = True 360 elif user_input == 'n': 361 print(no_reply) 362 var = False 363 elif not user_input: 364 if enabled_by_default: 365 print(yes_reply) 366 var = True 367 else: 368 print(no_reply) 369 var = False 370 else: 371 print('Invalid selection: %s' % user_input_origin) 372 return var 373 374 375def set_build_var(environ_cp, 376 var_name, 377 query_item, 378 option_name, 379 enabled_by_default, 380 bazel_config_name=None): 381 """Set if query_item will be enabled for the build. 382 383 Ask user if query_item will be enabled. Default is used if no input is given. 384 Set subprocess environment variable and write to .bazelrc if enabled. 385 386 Args: 387 environ_cp: copy of the os.environ. 388 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". 389 query_item: string for feature related to the variable, e.g. "CUDA for 390 Nvidia GPUs". 391 option_name: string for option to define in .bazelrc. 392 enabled_by_default: boolean for default behavior. 393 bazel_config_name: Name for Bazel --config argument to enable build feature. 394 """ 395 396 var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default))) 397 environ_cp[var_name] = var 398 if var == '1': 399 write_to_bazelrc('build:%s --define %s=true' % 400 (bazel_config_name, option_name)) 401 write_to_bazelrc('build --config=%s' % bazel_config_name) 402 elif bazel_config_name is not None: 403 # TODO(mikecase): Migrate all users of configure.py to use --config Bazel 404 # options and not to set build configs through environment variables. 405 write_to_bazelrc('build:%s --define %s=true' % 406 (bazel_config_name, option_name)) 407 408 409def set_action_env_var(environ_cp, 410 var_name, 411 query_item, 412 enabled_by_default, 413 question=None, 414 yes_reply=None, 415 no_reply=None, 416 bazel_config_name=None): 417 """Set boolean action_env variable. 418 419 Ask user if query_item will be enabled. Default is used if no input is given. 420 Set environment variable and write to .bazelrc. 421 422 Args: 423 environ_cp: copy of the os.environ. 424 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". 425 query_item: string for feature related to the variable, e.g. "CUDA for 426 Nvidia GPUs". 427 enabled_by_default: boolean for default behavior. 428 question: optional string for how to ask for user input. 429 yes_reply: optional string for reply when feature is enabled. 430 no_reply: optional string for reply when feature is disabled. 431 bazel_config_name: adding config to .bazelrc instead of action_env. 432 """ 433 var = int( 434 get_var(environ_cp, var_name, query_item, enabled_by_default, question, 435 yes_reply, no_reply)) 436 437 if not bazel_config_name: 438 write_action_env_to_bazelrc(var_name, var) 439 elif var: 440 write_to_bazelrc('build --config=%s' % bazel_config_name) 441 environ_cp[var_name] = str(var) 442 443 444def convert_version_to_int(version): 445 """Convert a version number to a integer that can be used to compare. 446 447 Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The 448 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored. 449 450 Args: 451 version: a version to be converted 452 453 Returns: 454 An integer if converted successfully, otherwise return None. 455 """ 456 version = version.split('-')[0] 457 version_segments = version.split('.') 458 # Treat "0.24" as "0.24.0" 459 if len(version_segments) == 2: 460 version_segments.append('0') 461 for seg in version_segments: 462 if not seg.isdigit(): 463 return None 464 465 version_str = ''.join(['%03d' % int(seg) for seg in version_segments]) 466 return int(version_str) 467 468 469def check_bazel_version(min_version, max_version): 470 """Check installed bazel version is between min_version and max_version. 471 472 Args: 473 min_version: string for minimum bazel version (must exist!). 474 max_version: string for maximum bazel version (must exist!). 475 476 Returns: 477 The bazel version detected. 478 """ 479 if which('bazel') is None: 480 print('Cannot find bazel. Please install bazel.') 481 sys.exit(0) 482 curr_version = run_shell( 483 ['bazel', '--batch', '--bazelrc=/dev/null', 'version']) 484 485 for line in curr_version.split('\n'): 486 if 'Build label: ' in line: 487 curr_version = line.split('Build label: ')[1] 488 break 489 490 min_version_int = convert_version_to_int(min_version) 491 curr_version_int = convert_version_to_int(curr_version) 492 max_version_int = convert_version_to_int(max_version) 493 494 # Check if current bazel version can be detected properly. 495 if not curr_version_int: 496 print('WARNING: current bazel installation is not a release version.') 497 print('Make sure you are running at least bazel %s' % min_version) 498 return curr_version 499 500 print('You have bazel %s installed.' % curr_version) 501 502 if curr_version_int < min_version_int: 503 print('Please upgrade your bazel installation to version %s or higher to ' 504 'build TensorFlow!' % min_version) 505 sys.exit(1) 506 if (curr_version_int > max_version_int and 507 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ): 508 print('Please downgrade your bazel installation to version %s or lower to ' 509 'build TensorFlow! To downgrade: download the installer for the old ' 510 'version (from https://github.com/bazelbuild/bazel/releases) then ' 511 'run the installer.' % max_version) 512 sys.exit(1) 513 return curr_version 514 515 516def set_cc_opt_flags(environ_cp): 517 """Set up architecture-dependent optimization flags. 518 519 Also append CC optimization flags to bazel.rc.. 520 521 Args: 522 environ_cp: copy of the os.environ. 523 """ 524 if is_ppc64le(): 525 # gcc on ppc64le does not support -march, use mcpu instead 526 default_cc_opt_flags = '-mcpu=native' 527 elif is_windows(): 528 default_cc_opt_flags = '/arch:AVX' 529 else: 530 default_cc_opt_flags = '-march=native -Wno-sign-compare' 531 question = ('Please specify optimization flags to use during compilation when' 532 ' bazel option "--config=opt" is specified [Default is %s]: ' 533 ) % default_cc_opt_flags 534 cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS', 535 question, default_cc_opt_flags) 536 for opt in cc_opt_flags.split(): 537 write_to_bazelrc('build:opt --copt=%s' % opt) 538 # It should be safe on the same build host. 539 if not is_ppc64le() and not is_windows(): 540 write_to_bazelrc('build:opt --host_copt=-march=native') 541 write_to_bazelrc('build:opt --define with_default_optimizations=true') 542 543 544def set_tf_cuda_clang(environ_cp): 545 """set TF_CUDA_CLANG action_env. 546 547 Args: 548 environ_cp: copy of the os.environ. 549 """ 550 question = 'Do you want to use clang as CUDA compiler?' 551 yes_reply = 'Clang will be used as CUDA compiler.' 552 no_reply = 'nvcc will be used as CUDA compiler.' 553 set_action_env_var( 554 environ_cp, 555 'TF_CUDA_CLANG', 556 None, 557 False, 558 question=question, 559 yes_reply=yes_reply, 560 no_reply=no_reply, 561 bazel_config_name='cuda_clang') 562 563 564def set_tf_download_clang(environ_cp): 565 """Set TF_DOWNLOAD_CLANG action_env.""" 566 question = 'Do you wish to download a fresh release of clang? (Experimental)' 567 yes_reply = 'Clang will be downloaded and used to compile tensorflow.' 568 no_reply = 'Clang will not be downloaded.' 569 set_action_env_var( 570 environ_cp, 571 'TF_DOWNLOAD_CLANG', 572 None, 573 False, 574 question=question, 575 yes_reply=yes_reply, 576 no_reply=no_reply, 577 bazel_config_name='download_clang') 578 579 580def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var, 581 var_default): 582 """Get var_name either from env, or user or default. 583 584 If var_name has been set as environment variable, use the preset value, else 585 ask for user input. If no input is provided, the default is used. 586 587 Args: 588 environ_cp: copy of the os.environ. 589 var_name: string for name of environment variable, e.g. "TF_NEED_CUDA". 590 ask_for_var: string for how to ask for user input. 591 var_default: default value string. 592 593 Returns: 594 string value for var_name 595 """ 596 var = environ_cp.get(var_name) 597 if not var: 598 var = get_input(ask_for_var) 599 print('\n') 600 if not var: 601 var = var_default 602 return var 603 604 605def set_clang_cuda_compiler_path(environ_cp): 606 """Set CLANG_CUDA_COMPILER_PATH.""" 607 default_clang_path = which('clang') or '' 608 ask_clang_path = ('Please specify which clang should be used as device and ' 609 'host compiler. [Default is %s]: ') % default_clang_path 610 611 while True: 612 clang_cuda_compiler_path = get_from_env_or_user_or_default( 613 environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path, 614 default_clang_path) 615 if os.path.exists(clang_cuda_compiler_path): 616 break 617 618 # Reset and retry 619 print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path) 620 environ_cp['CLANG_CUDA_COMPILER_PATH'] = '' 621 622 # Set CLANG_CUDA_COMPILER_PATH 623 environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path 624 write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH', 625 clang_cuda_compiler_path) 626 627 628def prompt_loop_or_load_from_env(environ_cp, 629 var_name, 630 var_default, 631 ask_for_var, 632 check_success, 633 error_msg, 634 suppress_default_error=False, 635 resolve_symlinks=False, 636 n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS): 637 """Loop over user prompts for an ENV param until receiving a valid response. 638 639 For the env param var_name, read from the environment or verify user input 640 until receiving valid input. When done, set var_name in the environ_cp to its 641 new value. 642 643 Args: 644 environ_cp: (Dict) copy of the os.environ. 645 var_name: (String) string for name of environment variable, e.g. "TF_MYVAR". 646 var_default: (String) default value string. 647 ask_for_var: (String) string for how to ask for user input. 648 check_success: (Function) function that takes one argument and returns a 649 boolean. Should return True if the value provided is considered valid. May 650 contain a complex error message if error_msg does not provide enough 651 information. In that case, set suppress_default_error to True. 652 error_msg: (String) String with one and only one '%s'. Formatted with each 653 invalid response upon check_success(input) failure. 654 suppress_default_error: (Bool) Suppress the above error message in favor of 655 one from the check_success function. 656 resolve_symlinks: (Bool) Translate symbolic links into the real filepath. 657 n_ask_attempts: (Integer) Number of times to query for valid input before 658 raising an error and quitting. 659 660 Returns: 661 [String] The value of var_name after querying for input. 662 663 Raises: 664 UserInputError: if a query has been attempted n_ask_attempts times without 665 success, assume that the user has made a scripting error, and will 666 continue to provide invalid input. Raise the error to avoid infinitely 667 looping. 668 """ 669 default = environ_cp.get(var_name) or var_default 670 full_query = '%s [Default is %s]: ' % ( 671 ask_for_var, 672 default, 673 ) 674 675 for _ in range(n_ask_attempts): 676 val = get_from_env_or_user_or_default(environ_cp, var_name, full_query, 677 default) 678 if check_success(val): 679 break 680 if not suppress_default_error: 681 print(error_msg % val) 682 environ_cp[var_name] = '' 683 else: 684 raise UserInputError('Invalid %s setting was provided %d times in a row. ' 685 'Assuming to be a scripting mistake.' % 686 (var_name, n_ask_attempts)) 687 688 if resolve_symlinks and os.path.islink(val): 689 val = os.path.realpath(val) 690 environ_cp[var_name] = val 691 return val 692 693 694def create_android_ndk_rule(environ_cp): 695 """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule.""" 696 if is_windows() or is_cygwin(): 697 default_ndk_path = cygpath('%s/Android/Sdk/ndk-bundle' % 698 environ_cp['APPDATA']) 699 elif is_macos(): 700 default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] 701 else: 702 default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME'] 703 704 def valid_ndk_path(path): 705 return (os.path.exists(path) and 706 os.path.exists(os.path.join(path, 'source.properties'))) 707 708 android_ndk_home_path = prompt_loop_or_load_from_env( 709 environ_cp, 710 var_name='ANDROID_NDK_HOME', 711 var_default=default_ndk_path, 712 ask_for_var='Please specify the home path of the Android NDK to use.', 713 check_success=valid_ndk_path, 714 error_msg=('The path %s or its child file "source.properties" ' 715 'does not exist.')) 716 write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path) 717 write_action_env_to_bazelrc( 718 'ANDROID_NDK_API_LEVEL', 719 get_ndk_api_level(environ_cp, android_ndk_home_path)) 720 721 722def create_android_sdk_rule(environ_cp): 723 """Set Android variables and write Android SDK WORKSPACE rule.""" 724 if is_windows() or is_cygwin(): 725 default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA']) 726 elif is_macos(): 727 default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME'] 728 else: 729 default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME'] 730 731 def valid_sdk_path(path): 732 return (os.path.exists(path) and 733 os.path.exists(os.path.join(path, 'platforms')) and 734 os.path.exists(os.path.join(path, 'build-tools'))) 735 736 android_sdk_home_path = prompt_loop_or_load_from_env( 737 environ_cp, 738 var_name='ANDROID_SDK_HOME', 739 var_default=default_sdk_path, 740 ask_for_var='Please specify the home path of the Android SDK to use.', 741 check_success=valid_sdk_path, 742 error_msg=('Either %s does not exist, or it does not contain the ' 743 'subdirectories "platforms" and "build-tools".')) 744 745 platforms = os.path.join(android_sdk_home_path, 'platforms') 746 api_levels = sorted(os.listdir(platforms)) 747 api_levels = [x.replace('android-', '') for x in api_levels] 748 749 def valid_api_level(api_level): 750 return os.path.exists( 751 os.path.join(android_sdk_home_path, 'platforms', 752 'android-' + api_level)) 753 754 android_api_level = prompt_loop_or_load_from_env( 755 environ_cp, 756 var_name='ANDROID_API_LEVEL', 757 var_default=api_levels[-1], 758 ask_for_var=('Please specify the Android SDK API level to use. ' 759 '[Available levels: %s]') % api_levels, 760 check_success=valid_api_level, 761 error_msg='Android-%s is not present in the SDK path.') 762 763 build_tools = os.path.join(android_sdk_home_path, 'build-tools') 764 versions = sorted(os.listdir(build_tools)) 765 766 def valid_build_tools(version): 767 return os.path.exists( 768 os.path.join(android_sdk_home_path, 'build-tools', version)) 769 770 android_build_tools_version = prompt_loop_or_load_from_env( 771 environ_cp, 772 var_name='ANDROID_BUILD_TOOLS_VERSION', 773 var_default=versions[-1], 774 ask_for_var=('Please specify an Android build tools version to use. ' 775 '[Available versions: %s]') % versions, 776 check_success=valid_build_tools, 777 error_msg=('The selected SDK does not have build-tools version %s ' 778 'available.')) 779 780 write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION', 781 android_build_tools_version) 782 write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level) 783 write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path) 784 785 786def get_ndk_api_level(environ_cp, android_ndk_home_path): 787 """Gets the appropriate NDK API level to use for the provided Android NDK path.""" 788 789 # First check to see if we're using a blessed version of the NDK. 790 properties_path = '%s/source.properties' % android_ndk_home_path 791 if is_windows() or is_cygwin(): 792 properties_path = cygpath(properties_path) 793 with open(properties_path, 'r') as f: 794 filedata = f.read() 795 796 revision = re.search(r'Pkg.Revision = (\d+)', filedata) 797 if revision: 798 ndk_version = revision.group(1) 799 else: 800 raise Exception('Unable to parse NDK revision.') 801 if int(ndk_version) not in _SUPPORTED_ANDROID_NDK_VERSIONS: 802 print('WARNING: The NDK version in %s is %s, which is not ' 803 'supported by Bazel (officially supported versions: %s). Please use ' 804 'another version. Compiling Android targets may result in confusing ' 805 'errors.\n' % 806 (android_ndk_home_path, ndk_version, _SUPPORTED_ANDROID_NDK_VERSIONS)) 807 808 # Now grab the NDK API level to use. Note that this is different from the 809 # SDK API level, as the NDK API level is effectively the *min* target SDK 810 # version. 811 platforms = os.path.join(android_ndk_home_path, 'platforms') 812 api_levels = sorted(os.listdir(platforms)) 813 api_levels = [ 814 x.replace('android-', '') for x in api_levels if 'android-' in x 815 ] 816 817 def valid_api_level(api_level): 818 return os.path.exists( 819 os.path.join(android_ndk_home_path, 'platforms', 820 'android-' + api_level)) 821 822 android_ndk_api_level = prompt_loop_or_load_from_env( 823 environ_cp, 824 var_name='ANDROID_NDK_API_LEVEL', 825 var_default='21', # 21 is required for ARM64 support. 826 ask_for_var=('Please specify the (min) Android NDK API level to use. ' 827 '[Available levels: %s]') % api_levels, 828 check_success=valid_api_level, 829 error_msg='Android-%s is not present in the NDK path.') 830 831 return android_ndk_api_level 832 833 834def set_gcc_host_compiler_path(environ_cp): 835 """Set GCC_HOST_COMPILER_PATH.""" 836 default_gcc_host_compiler_path = which('gcc') or '' 837 cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH') 838 839 if os.path.islink(cuda_bin_symlink): 840 # os.readlink is only available in linux 841 default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink) 842 843 gcc_host_compiler_path = prompt_loop_or_load_from_env( 844 environ_cp, 845 var_name='GCC_HOST_COMPILER_PATH', 846 var_default=default_gcc_host_compiler_path, 847 ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.', 848 check_success=os.path.exists, 849 resolve_symlinks=True, 850 error_msg='Invalid gcc path. %s cannot be found.', 851 ) 852 853 write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path) 854 855 856def reformat_version_sequence(version_str, sequence_count): 857 """Reformat the version string to have the given number of sequences. 858 859 For example: 860 Given (7, 2) -> 7.0 861 (7.0.1, 2) -> 7.0 862 (5, 1) -> 5 863 (5.0.3.2, 1) -> 5 864 865 Args: 866 version_str: String, the version string. 867 sequence_count: int, an integer. 868 869 Returns: 870 string, reformatted version string. 871 """ 872 v = version_str.split('.') 873 if len(v) < sequence_count: 874 v = v + (['0'] * (sequence_count - len(v))) 875 876 return '.'.join(v[:sequence_count]) 877 878 879def set_tf_cuda_paths(environ_cp): 880 """Set TF_CUDA_PATHS.""" 881 ask_cuda_paths = ( 882 'Please specify the comma-separated list of base paths to look for CUDA ' 883 'libraries and headers. [Leave empty to use the default]: ') 884 tf_cuda_paths = get_from_env_or_user_or_default(environ_cp, 'TF_CUDA_PATHS', 885 ask_cuda_paths, '') 886 if tf_cuda_paths: 887 environ_cp['TF_CUDA_PATHS'] = tf_cuda_paths 888 889 890def set_tf_cuda_version(environ_cp): 891 """Set TF_CUDA_VERSION.""" 892 ask_cuda_version = ( 893 'Please specify the CUDA SDK version you want to use. ' 894 '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION 895 tf_cuda_version = get_from_env_or_user_or_default(environ_cp, 896 'TF_CUDA_VERSION', 897 ask_cuda_version, 898 _DEFAULT_CUDA_VERSION) 899 environ_cp['TF_CUDA_VERSION'] = tf_cuda_version 900 901 902def set_tf_cudnn_version(environ_cp): 903 """Set TF_CUDNN_VERSION.""" 904 ask_cudnn_version = ( 905 'Please specify the cuDNN version you want to use. ' 906 '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION 907 tf_cudnn_version = get_from_env_or_user_or_default(environ_cp, 908 'TF_CUDNN_VERSION', 909 ask_cudnn_version, 910 _DEFAULT_CUDNN_VERSION) 911 environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version 912 913 914def is_cuda_compatible(lib, cuda_ver, cudnn_ver): 915 """Check compatibility between given library and cudnn/cudart libraries.""" 916 ldd_bin = which('ldd') or '/usr/bin/ldd' 917 ldd_out = run_shell([ldd_bin, lib], True) 918 ldd_out = ldd_out.split(os.linesep) 919 cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$') 920 cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$') 921 cudnn = None 922 cudart = None 923 cudnn_ok = True # assume no cudnn dependency by default 924 cuda_ok = True # assume no cuda dependency by default 925 for line in ldd_out: 926 if 'libcudnn.so' in line: 927 cudnn = cudnn_pattern.search(line) 928 cudnn_ok = False 929 elif 'libcudart.so' in line: 930 cudart = cuda_pattern.search(line) 931 cuda_ok = False 932 if cudnn and len(cudnn.group(1)): 933 cudnn = convert_version_to_int(cudnn.group(1)) 934 if cudart and len(cudart.group(1)): 935 cudart = convert_version_to_int(cudart.group(1)) 936 if cudnn is not None: 937 cudnn_ok = (cudnn == cudnn_ver) 938 if cudart is not None: 939 cuda_ok = (cudart == cuda_ver) 940 return cudnn_ok and cuda_ok 941 942 943def set_tf_tensorrt_version(environ_cp): 944 """Set TF_TENSORRT_VERSION.""" 945 if not is_linux(): 946 raise ValueError('Currently TensorRT is only supported on Linux platform.') 947 948 if not int(environ_cp.get('TF_NEED_TENSORRT', False)): 949 return 950 951 ask_tensorrt_version = ( 952 'Please specify the TensorRT version you want to use. ' 953 '[Leave empty to default to TensorRT %s]: ') % _DEFAULT_TENSORRT_VERSION 954 tf_tensorrt_version = get_from_env_or_user_or_default( 955 environ_cp, 'TF_TENSORRT_VERSION', ask_tensorrt_version, 956 _DEFAULT_TENSORRT_VERSION) 957 environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version 958 959 960def set_tf_nccl_version(environ_cp): 961 """Set TF_NCCL_VERSION.""" 962 if not is_linux(): 963 raise ValueError('Currently NCCL is only supported on Linux platform.') 964 965 if 'TF_NCCL_VERSION' in environ_cp: 966 return 967 968 ask_nccl_version = ( 969 'Please specify the locally installed NCCL version you want to use. ' 970 '[Leave empty to use http://github.com/nvidia/nccl]: ') 971 tf_nccl_version = get_from_env_or_user_or_default(environ_cp, 972 'TF_NCCL_VERSION', 973 ask_nccl_version, '') 974 environ_cp['TF_NCCL_VERSION'] = tf_nccl_version 975 976 977def get_native_cuda_compute_capabilities(environ_cp): 978 """Get native cuda compute capabilities. 979 980 Args: 981 environ_cp: copy of the os.environ. 982 983 Returns: 984 string of native cuda compute capabilities, separated by comma. 985 """ 986 device_query_bin = os.path.join( 987 environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery') 988 if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK): 989 try: 990 output = run_shell(device_query_bin).split('\n') 991 pattern = re.compile('[0-9]*\\.[0-9]*') 992 output = [pattern.search(x) for x in output if 'Capability' in x] 993 output = ','.join(x.group() for x in output if x is not None) 994 except subprocess.CalledProcessError: 995 output = '' 996 else: 997 output = '' 998 return output 999 1000 1001def set_tf_cuda_compute_capabilities(environ_cp): 1002 """Set TF_CUDA_COMPUTE_CAPABILITIES.""" 1003 while True: 1004 native_cuda_compute_capabilities = get_native_cuda_compute_capabilities( 1005 environ_cp) 1006 if not native_cuda_compute_capabilities: 1007 default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES 1008 else: 1009 default_cuda_compute_capabilities = native_cuda_compute_capabilities 1010 1011 ask_cuda_compute_capabilities = ( 1012 'Please specify a list of comma-separated ' 1013 'CUDA compute capabilities you want to ' 1014 'build with.\nYou can find the compute ' 1015 'capability of your device at: ' 1016 'https://developer.nvidia.com/cuda-gpus.\nPlease' 1017 ' note that each additional compute ' 1018 'capability significantly increases your ' 1019 'build time and binary size, and that ' 1020 'TensorFlow only supports compute ' 1021 'capabilities >= 3.5 [Default is: %s]: ' % 1022 default_cuda_compute_capabilities) 1023 tf_cuda_compute_capabilities = get_from_env_or_user_or_default( 1024 environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES', 1025 ask_cuda_compute_capabilities, default_cuda_compute_capabilities) 1026 # Check whether all capabilities from the input is valid 1027 all_valid = True 1028 # Remove all whitespace characters before splitting the string 1029 # that users may insert by accident, as this will result in error 1030 tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split()) 1031 for compute_capability in tf_cuda_compute_capabilities.split(','): 1032 m = re.match('[0-9]+.[0-9]+', compute_capability) 1033 if not m: 1034 print('Invalid compute capability: %s' % compute_capability) 1035 all_valid = False 1036 else: 1037 ver = float(m.group(0)) 1038 if ver < 3.0: 1039 print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 ' 1040 'and higher. Please re-specify the list of compute ' 1041 'capabilities excluding version %s.' % ver) 1042 all_valid = False 1043 if ver < 3.5: 1044 print('WARNING: XLA does not support CUDA compute capabilities ' 1045 'lower than 3.5. Disable XLA when running on older GPUs.') 1046 1047 if all_valid: 1048 break 1049 1050 # Reset and Retry 1051 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = '' 1052 1053 # Set TF_CUDA_COMPUTE_CAPABILITIES 1054 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities 1055 write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES', 1056 tf_cuda_compute_capabilities) 1057 1058 1059def set_other_cuda_vars(environ_cp): 1060 """Set other CUDA related variables.""" 1061 # If CUDA is enabled, always use GPU during build and test. 1062 if environ_cp.get('TF_CUDA_CLANG') == '1': 1063 write_to_bazelrc('build --config=cuda_clang') 1064 else: 1065 write_to_bazelrc('build --config=cuda') 1066 1067 1068def set_host_cxx_compiler(environ_cp): 1069 """Set HOST_CXX_COMPILER.""" 1070 default_cxx_host_compiler = which('g++') or '' 1071 1072 host_cxx_compiler = prompt_loop_or_load_from_env( 1073 environ_cp, 1074 var_name='HOST_CXX_COMPILER', 1075 var_default=default_cxx_host_compiler, 1076 ask_for_var=('Please specify which C++ compiler should be used as the ' 1077 'host C++ compiler.'), 1078 check_success=os.path.exists, 1079 error_msg='Invalid C++ compiler path. %s cannot be found.', 1080 ) 1081 1082 write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler) 1083 1084 1085def set_host_c_compiler(environ_cp): 1086 """Set HOST_C_COMPILER.""" 1087 default_c_host_compiler = which('gcc') or '' 1088 1089 host_c_compiler = prompt_loop_or_load_from_env( 1090 environ_cp, 1091 var_name='HOST_C_COMPILER', 1092 var_default=default_c_host_compiler, 1093 ask_for_var=('Please specify which C compiler should be used as the host ' 1094 'C compiler.'), 1095 check_success=os.path.exists, 1096 error_msg='Invalid C compiler path. %s cannot be found.', 1097 ) 1098 1099 write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler) 1100 1101 1102def set_computecpp_toolkit_path(environ_cp): 1103 """Set COMPUTECPP_TOOLKIT_PATH.""" 1104 1105 def toolkit_exists(toolkit_path): 1106 """Check if a computecpp toolkit path is valid.""" 1107 if is_linux(): 1108 sycl_rt_lib_path = 'lib/libComputeCpp.so' 1109 else: 1110 sycl_rt_lib_path = '' 1111 1112 sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path) 1113 exists = os.path.exists(sycl_rt_lib_path_full) 1114 if not exists: 1115 print('Invalid SYCL %s library path. %s cannot be found' % 1116 (_TF_OPENCL_VERSION, sycl_rt_lib_path_full)) 1117 return exists 1118 1119 computecpp_toolkit_path = prompt_loop_or_load_from_env( 1120 environ_cp, 1121 var_name='COMPUTECPP_TOOLKIT_PATH', 1122 var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH, 1123 ask_for_var=( 1124 'Please specify the location where ComputeCpp for SYCL %s is ' 1125 'installed.' % _TF_OPENCL_VERSION), 1126 check_success=toolkit_exists, 1127 error_msg='Invalid SYCL compiler path. %s cannot be found.', 1128 suppress_default_error=True) 1129 1130 write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH', 1131 computecpp_toolkit_path) 1132 1133 1134def set_trisycl_include_dir(environ_cp): 1135 """Set TRISYCL_INCLUDE_DIR.""" 1136 1137 ask_trisycl_include_dir = ('Please specify the location of the triSYCL ' 1138 'include directory. (Use --config=sycl_trisycl ' 1139 'when building with Bazel) ' 1140 '[Default is %s]: ') % ( 1141 _DEFAULT_TRISYCL_INCLUDE_DIR) 1142 1143 while True: 1144 trisycl_include_dir = get_from_env_or_user_or_default( 1145 environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir, 1146 _DEFAULT_TRISYCL_INCLUDE_DIR) 1147 if os.path.exists(trisycl_include_dir): 1148 break 1149 1150 print('Invalid triSYCL include directory, %s cannot be found' % 1151 (trisycl_include_dir)) 1152 1153 # Set TRISYCL_INCLUDE_DIR 1154 environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir 1155 write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir) 1156 1157 1158def system_specific_test_config(env): 1159 """Add default build and test flags required for TF tests to bazelrc.""" 1160 write_to_bazelrc('test --flaky_test_attempts=3') 1161 write_to_bazelrc('test --test_size_filters=small,medium') 1162 1163 # Each instance of --test_tag_filters or --build_tag_filters overrides all 1164 # previous instances, so we need to build up a complete list and write a 1165 # single list of filters for the .bazelrc file. 1166 1167 # Filters to use with both --test_tag_filters and --build_tag_filters 1168 test_and_build_filters = ['-benchmark-test', '-no_oss'] 1169 # Additional filters for --test_tag_filters beyond those in 1170 # test_and_build_filters 1171 test_only_filters = ['-oss_serial'] 1172 if is_windows(): 1173 test_and_build_filters.append('-no_windows') 1174 if env.get('TF_NEED_CUDA', None) == '1': 1175 test_and_build_filters += ['-no_windows_gpu', '-no_gpu'] 1176 else: 1177 test_and_build_filters.append('-gpu') 1178 elif is_macos(): 1179 test_and_build_filters += ['-gpu', '-nomac', '-no_mac'] 1180 elif is_linux(): 1181 if env.get('TF_NEED_CUDA', None) == '1': 1182 test_and_build_filters.append('-no_gpu') 1183 write_to_bazelrc('test --test_env=LD_LIBRARY_PATH') 1184 else: 1185 test_and_build_filters.append('-gpu') 1186 1187 # Disable tests with "v1only" tag in "v2" Bazel config, but not in "v1" config 1188 write_to_bazelrc('test:v1 --test_tag_filters=%s' % 1189 ','.join(test_and_build_filters + test_only_filters)) 1190 write_to_bazelrc('test:v1 --build_tag_filters=%s' % 1191 ','.join(test_and_build_filters)) 1192 write_to_bazelrc( 1193 'test:v2 --test_tag_filters=%s' % 1194 ','.join(test_and_build_filters + test_only_filters + ['-v1only'])) 1195 write_to_bazelrc('test:v2 --build_tag_filters=%s' % 1196 ','.join(test_and_build_filters + ['-v1only'])) 1197 1198 1199def set_system_libs_flag(environ_cp): 1200 syslibs = environ_cp.get('TF_SYSTEM_LIBS', '') 1201 if syslibs: 1202 if ',' in syslibs: 1203 syslibs = ','.join(sorted(syslibs.split(','))) 1204 else: 1205 syslibs = ','.join(sorted(syslibs.split())) 1206 write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs) 1207 1208 if 'PREFIX' in environ_cp: 1209 write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX']) 1210 if 'LIBDIR' in environ_cp: 1211 write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR']) 1212 if 'INCLUDEDIR' in environ_cp: 1213 write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR']) 1214 1215 1216def is_reduced_optimize_huge_functions_available(environ_cp): 1217 """Check to see if the system supports /d2ReducedOptimizeHugeFunctions. 1218 1219 The above compiler flag is a new compiler flag introduced to the Visual Studio 1220 compiler in version 16.4 (available in Visual Studio 2019, Preview edition 1221 only, as of 2019-11-19). TensorFlow needs this flag to massively reduce 1222 compile times, but until 16.4 is officially released, we can't depend on it. 1223 1224 See also https://groups.google.com/a/tensorflow.org/d/topic/build/SsW98Eo7l3o/discussion 1225 1226 Because it's very annoying to check this manually (to check the MSVC installed 1227 versions, you need to use the registry, and it's not clear if Bazel will be 1228 using that install version anyway), we expect enviroments who know they may 1229 use this flag to export TF_VC_VERSION=16.4 1230 1231 TODO(angerson, gunan): Remove this function when TensorFlow's minimum VS 1232 version is upgraded to 16.4. 1233 1234 Arguments: 1235 environ_cp: Environment of the current execution 1236 1237 Returns: 1238 boolean, whether or not /d2ReducedOptimizeHugeFunctions is available on this 1239 machine. 1240 """ 1241 return float(environ_cp.get('TF_VC_VERSION', '0')) >= 16.4 1242 1243 1244def set_windows_build_flags(environ_cp): 1245 """Set Windows specific build options.""" 1246 if is_reduced_optimize_huge_functions_available(environ_cp): 1247 write_to_bazelrc( 1248 'build --copt=/d2ReducedOptimizeHugeFunctions --host_copt=/d2ReducedOptimizeHugeFunctions' 1249 ) 1250 1251 if get_var( 1252 environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline', 1253 True, ('Would you like to override eigen strong inline for some C++ ' 1254 'compilation to reduce the compilation time?'), 1255 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, ' 1256 'some compilations could take more than 20 mins.'): 1257 # Due to a known MSVC compiler issue 1258 # https://github.com/tensorflow/tensorflow/issues/10521 1259 # Overriding eigen strong inline speeds up the compiling of 1260 # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes, 1261 # but this also hurts the performance. Let users decide what they want. 1262 write_to_bazelrc('build --define=override_eigen_strong_inline=true') 1263 1264 1265def config_info_line(name, help_text): 1266 """Helper function to print formatted help text for Bazel config options.""" 1267 print('\t--config=%-12s\t# %s' % (name, help_text)) 1268 1269 1270def configure_ios(): 1271 """Configures TensorFlow for iOS builds. 1272 1273 This function will only be executed if `is_macos()` is true. 1274 """ 1275 if not is_macos(): 1276 return 1277 for filepath in APPLE_BAZEL_FILES: 1278 existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple') 1279 renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath) 1280 symlink_force(existing_filepath, renamed_filepath) 1281 for filepath in IOS_FILES: 1282 filename = os.path.basename(filepath) 1283 new_filepath = os.path.join(_TF_WORKSPACE_ROOT, filename) 1284 symlink_force(filepath, new_filepath) 1285 1286 1287def validate_cuda_config(environ_cp): 1288 """Run find_cuda_config.py and return cuda_toolkit_path, or None.""" 1289 1290 def maybe_encode_env(env): 1291 """Encodes unicode in env to str on Windows python 2.x.""" 1292 if not is_windows() or sys.version_info[0] != 2: 1293 return env 1294 for k, v in env.items(): 1295 if isinstance(k, unicode): 1296 k = k.encode('ascii') 1297 if isinstance(v, unicode): 1298 v = v.encode('ascii') 1299 env[k] = v 1300 return env 1301 1302 cuda_libraries = ['cuda', 'cudnn'] 1303 if is_linux(): 1304 if int(environ_cp.get('TF_NEED_TENSORRT', False)): 1305 cuda_libraries.append('tensorrt') 1306 if environ_cp.get('TF_NCCL_VERSION', None): 1307 cuda_libraries.append('nccl') 1308 1309 proc = subprocess.Popen( 1310 [environ_cp['PYTHON_BIN_PATH'], 'third_party/gpus/find_cuda_config.py'] + 1311 cuda_libraries, 1312 stdout=subprocess.PIPE, 1313 env=maybe_encode_env(environ_cp)) 1314 1315 if proc.wait(): 1316 # Errors from find_cuda_config.py were sent to stderr. 1317 print('Asking for detailed CUDA configuration...\n') 1318 return False 1319 1320 config = dict( 1321 tuple(line.decode('ascii').rstrip().split(': ')) for line in proc.stdout) 1322 1323 print('Found CUDA %s in:' % config['cuda_version']) 1324 print(' %s' % config['cuda_library_dir']) 1325 print(' %s' % config['cuda_include_dir']) 1326 1327 print('Found cuDNN %s in:' % config['cudnn_version']) 1328 print(' %s' % config['cudnn_library_dir']) 1329 print(' %s' % config['cudnn_include_dir']) 1330 1331 if 'tensorrt_version' in config: 1332 print('Found TensorRT %s in:' % config['tensorrt_version']) 1333 print(' %s' % config['tensorrt_library_dir']) 1334 print(' %s' % config['tensorrt_include_dir']) 1335 1336 if config.get('nccl_version', None): 1337 print('Found NCCL %s in:' % config['nccl_version']) 1338 print(' %s' % config['nccl_library_dir']) 1339 print(' %s' % config['nccl_include_dir']) 1340 1341 print('\n') 1342 1343 environ_cp['CUDA_TOOLKIT_PATH'] = config['cuda_toolkit_path'] 1344 return True 1345 1346 1347def main(): 1348 global _TF_WORKSPACE_ROOT 1349 global _TF_BAZELRC 1350 global _TF_CURRENT_BAZEL_VERSION 1351 1352 parser = argparse.ArgumentParser() 1353 parser.add_argument( 1354 '--workspace', 1355 type=str, 1356 default=os.path.abspath(os.path.dirname(__file__)), 1357 help='The absolute path to your active Bazel workspace.') 1358 args = parser.parse_args() 1359 1360 _TF_WORKSPACE_ROOT = args.workspace 1361 _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME) 1362 1363 # Make a copy of os.environ to be clear when functions and getting and setting 1364 # environment variables. 1365 environ_cp = dict(os.environ) 1366 1367 current_bazel_version = check_bazel_version(_TF_MIN_BAZEL_VERSION, 1368 _TF_MAX_BAZEL_VERSION) 1369 _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version) 1370 1371 reset_tf_configure_bazelrc() 1372 1373 cleanup_makefile() 1374 setup_python(environ_cp) 1375 1376 if is_windows(): 1377 environ_cp['TF_NEED_OPENCL_SYCL'] = '0' 1378 environ_cp['TF_NEED_COMPUTECPP'] = '0' 1379 environ_cp['TF_NEED_OPENCL'] = '0' 1380 environ_cp['TF_CUDA_CLANG'] = '0' 1381 environ_cp['TF_NEED_TENSORRT'] = '0' 1382 # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on 1383 # Windows. 1384 environ_cp['TF_DOWNLOAD_CLANG'] = '0' 1385 environ_cp['TF_NEED_MPI'] = '0' 1386 environ_cp['TF_SET_ANDROID_WORKSPACE'] = '0' 1387 1388 if is_macos(): 1389 environ_cp['TF_NEED_TENSORRT'] = '0' 1390 else: 1391 environ_cp['TF_CONFIGURE_IOS'] = '0' 1392 1393 xla_enabled_by_default = is_linux() or is_macos() 1394 set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support', 1395 xla_enabled_by_default, 'xla') 1396 1397 set_action_env_var( 1398 environ_cp, 1399 'TF_NEED_OPENCL_SYCL', 1400 'OpenCL SYCL', 1401 False, 1402 bazel_config_name='sycl') 1403 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': 1404 set_host_cxx_compiler(environ_cp) 1405 set_host_c_compiler(environ_cp) 1406 set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True) 1407 if environ_cp.get('TF_NEED_COMPUTECPP') == '1': 1408 set_computecpp_toolkit_path(environ_cp) 1409 else: 1410 set_trisycl_include_dir(environ_cp) 1411 1412 set_action_env_var( 1413 environ_cp, 'TF_NEED_ROCM', 'ROCm', False, bazel_config_name='rocm') 1414 if (environ_cp.get('TF_NEED_ROCM') == '1' and 1415 'LD_LIBRARY_PATH' in environ_cp and 1416 environ_cp.get('LD_LIBRARY_PATH') != '1'): 1417 write_action_env_to_bazelrc('LD_LIBRARY_PATH', 1418 environ_cp.get('LD_LIBRARY_PATH')) 1419 1420 environ_cp['TF_NEED_CUDA'] = str( 1421 int(get_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False))) 1422 if (environ_cp.get('TF_NEED_CUDA') == '1' and 1423 'TF_CUDA_CONFIG_REPO' not in environ_cp): 1424 1425 set_action_env_var( 1426 environ_cp, 1427 'TF_NEED_TENSORRT', 1428 'TensorRT', 1429 False, 1430 bazel_config_name='tensorrt') 1431 1432 environ_save = dict(environ_cp) 1433 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): 1434 1435 if validate_cuda_config(environ_cp): 1436 cuda_env_names = [ 1437 'TF_CUDA_VERSION', 1438 'TF_CUBLAS_VERSION', 1439 'TF_CUDNN_VERSION', 1440 'TF_TENSORRT_VERSION', 1441 'TF_NCCL_VERSION', 1442 'TF_CUDA_PATHS', 1443 # Items below are for backwards compatibility when not using 1444 # TF_CUDA_PATHS. 1445 'CUDA_TOOLKIT_PATH', 1446 'CUDNN_INSTALL_PATH', 1447 'NCCL_INSTALL_PATH', 1448 'NCCL_HDR_PATH', 1449 'TENSORRT_INSTALL_PATH' 1450 ] 1451 # Note: set_action_env_var above already writes to bazelrc. 1452 for name in cuda_env_names: 1453 if name in environ_cp: 1454 write_action_env_to_bazelrc(name, environ_cp[name]) 1455 break 1456 1457 # Restore settings changed below if CUDA config could not be validated. 1458 environ_cp = dict(environ_save) 1459 1460 set_tf_cuda_version(environ_cp) 1461 set_tf_cudnn_version(environ_cp) 1462 if is_linux(): 1463 set_tf_tensorrt_version(environ_cp) 1464 set_tf_nccl_version(environ_cp) 1465 1466 set_tf_cuda_paths(environ_cp) 1467 1468 else: 1469 raise UserInputError( 1470 'Invalid CUDA setting were provided %d ' 1471 'times in a row. Assuming to be a scripting mistake.' % 1472 _DEFAULT_PROMPT_ASK_ATTEMPTS) 1473 1474 set_tf_cuda_compute_capabilities(environ_cp) 1475 if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get( 1476 'LD_LIBRARY_PATH') != '1': 1477 write_action_env_to_bazelrc('LD_LIBRARY_PATH', 1478 environ_cp.get('LD_LIBRARY_PATH')) 1479 1480 set_tf_cuda_clang(environ_cp) 1481 if environ_cp.get('TF_CUDA_CLANG') == '1': 1482 # Ask whether we should download the clang toolchain. 1483 set_tf_download_clang(environ_cp) 1484 if environ_cp.get('TF_DOWNLOAD_CLANG') != '1': 1485 # Set up which clang we should use as the cuda / host compiler. 1486 set_clang_cuda_compiler_path(environ_cp) 1487 else: 1488 # Use downloaded LLD for linking. 1489 write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld') 1490 else: 1491 # Set up which gcc nvcc should use as the host compiler 1492 # No need to set this on Windows 1493 if not is_windows(): 1494 set_gcc_host_compiler_path(environ_cp) 1495 set_other_cuda_vars(environ_cp) 1496 else: 1497 # CUDA not required. Ask whether we should download the clang toolchain and 1498 # use it for the CPU build. 1499 set_tf_download_clang(environ_cp) 1500 1501 # SYCL / ROCm / CUDA are mutually exclusive. 1502 # At most 1 GPU platform can be configured. 1503 gpu_platform_count = 0 1504 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': 1505 gpu_platform_count += 1 1506 if environ_cp.get('TF_NEED_ROCM') == '1': 1507 gpu_platform_count += 1 1508 if environ_cp.get('TF_NEED_CUDA') == '1': 1509 gpu_platform_count += 1 1510 if gpu_platform_count >= 2: 1511 raise UserInputError('SYCL / CUDA / ROCm are mututally exclusive. ' 1512 'At most 1 GPU platform can be configured.') 1513 1514 set_cc_opt_flags(environ_cp) 1515 set_system_libs_flag(environ_cp) 1516 if is_windows(): 1517 set_windows_build_flags(environ_cp) 1518 1519 if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False, 1520 ('Would you like to interactively configure ./WORKSPACE for ' 1521 'Android builds?'), 'Searching for NDK and SDK installations.', 1522 'Not configuring the WORKSPACE for Android builds.'): 1523 create_android_ndk_rule(environ_cp) 1524 create_android_sdk_rule(environ_cp) 1525 1526 system_specific_test_config(os.environ) 1527 1528 set_action_env_var(environ_cp, 'TF_CONFIGURE_IOS', 'iOS', False) 1529 if environ_cp.get('TF_CONFIGURE_IOS') == '1': 1530 configure_ios() 1531 1532 print('Preconfigured Bazel build configs. You can use any of the below by ' 1533 'adding "--config=<>" to your build command. See .bazelrc for more ' 1534 'details.') 1535 config_info_line('mkl', 'Build with MKL support.') 1536 config_info_line('monolithic', 'Config for mostly static monolithic build.') 1537 config_info_line('ngraph', 'Build with Intel nGraph support.') 1538 config_info_line('numa', 'Build with NUMA support.') 1539 config_info_line( 1540 'dynamic_kernels', 1541 '(Experimental) Build kernels into separate shared objects.') 1542 config_info_line('v2', 'Build TensorFlow 2.x instead of 1.x.') 1543 1544 print('Preconfigured Bazel build configs to DISABLE default on features:') 1545 config_info_line('noaws', 'Disable AWS S3 filesystem support.') 1546 config_info_line('nogcp', 'Disable GCP support.') 1547 config_info_line('nohdfs', 'Disable HDFS support.') 1548 config_info_line('nonccl', 'Disable NVIDIA NCCL support.') 1549 1550 1551if __name__ == '__main__': 1552 main() 1553